8#if !defined(ALIB_C20_MODULES) || ((ALIB_C20_MODULES != 0) && (ALIB_C20_MODULES != 1))
9# error "Symbol ALIB_C20_MODULES has to be given to the compiler as either 0 or 1"
45 for(
auto& op :
compiler.UnaryOperators )
47 ALIB_ASSERT_ERROR( !unaryOperators.Contains(op),
"EXPR",
48 "Doubly defined unary operator symbol '{}'.", op )
50 unaryOperators.EmplaceUnique(op);
52 operatorChars[it]= true;
55 for(
auto& op :
compiler.AlphabeticUnaryOperatorAliases )
57 ALIB_ASSERT_ERROR( !unaryOperators.Contains(op.first),
"EXPR",
58 "Doubly defined unary operator symbol '{}'.", op.first )
60 unaryOperators.EmplaceUnique(op.first);
61 if( !isalpha( op.first.CharAtStart() ) )
62 for( auto it : op.first )
63 operatorChars[it]= true;
67 for(
auto& op :
compiler.BinaryOperators )
69 ALIB_ASSERT_ERROR( !binaryOperators.Contains(op.first),
"EXPR",
70 "Doubly defined binary operator symbol '{}'.", op.first )
71 if( op.first == A_CHAR(
"[]") )
73 syntaxTokens[u8'[
']= true;
74 syntaxTokens[u8']
']= true;
78 binaryOperators.EmplaceUnique(op.first);
79 for( auto it : op.first )
80 operatorChars[it]= true;
84 for( auto& op : compiler.AlphabeticBinaryOperatorAliases )
86 ALIB_ASSERT_ERROR( !binaryOperators.Contains(op.first), "EXPR",
87 "Doubly defined binary operator symbol '{}
'.", op.first )
89 ALIB_DBG( auto originalOp= )
90 compiler.BinaryOperators.Find( op.second );
91 ALIB_ASSERT_ERROR( originalOp != compiler.BinaryOperators.end(), "EXPR",
92 "Alias '{}
' defined for unknown operator '{}
'.",
95 binaryOperators.EmplaceUnique(op.first);
96 if( !isalpha( op.first.CharAtStart() ) )
97 for( auto it : op.first )
98 operatorChars[it]= true;
102// #################################################################################################
104// #################################################################################################
105void ParserImpl::NextToken()
108 tokPosition= expression.Length() - scanner.Length();
110 if( scanner.IsEmpty() )
116 character first= scanner.CharAtStart<NC>();
118 //------------------------------ Syntax Tokens ------------------------------
119 if( syntaxTokens[first] )
121 token= Tokens(first);
122 scanner.ConsumeChar();
126 //------------------------------ Symbolic operators ------------------------------
127 // read up to 3 operator characters
128 if( operatorChars[first] )
130 integer operatorLength= 1;
131 scanner.ConsumeChar();
132 if( operatorChars[scanner.CharAtStart() ] )
134 scanner.ConsumeChar();
137 if( operatorChars[scanner.CharAtStart() ] )
139 scanner.ConsumeChar();
144 token= Tokens::SymbolicOp;
145 tokString= String( expression.Buffer() + tokPosition, operatorLength );
147 // special treatment for Elvis with spaces "? :"
148 if( tokString == A_CHAR("?") && compiler.BinaryOperators.Contains( A_CHAR("?:") ) )
150 // patch existing token and return
151 Substring backup= scanner;
152 if( scanner.TrimStart().CharAtStart() == ':
' )
154 tokString= A_CHAR("?:");
155 scanner.ConsumeChar();
163 //------------------------------ alphabetic operators ------------------------------
164 if( isalpha( first ) )
167 while( len < scanner.Length() && ( isalpha( scanner[len] ) || scanner[len] == '_
' ) )
169 tokString= scanner.Substring<NC>( 0, len );
170 auto hashCode= tokString.HashcodeIgnoreCase();
174 decltype(unaryOperators)::Iterator it;
175 if( (it= unaryOperators .Find( tokString, hashCode )) != unaryOperators.end()
176 && ( HasBits(compiler.CfgCompilation, Compilation::AlphabeticOperatorsIgnoreCase)
177 || tokString.Equals<NC>( it.Value() ) ) )
179 scanner.ConsumeChars<NC>( tokString.Length() );
180 token= Tokens::AlphaUnOp;
187 decltype(binaryOperators)::Iterator it;
188 if( (it= binaryOperators .Find( tokString, hashCode )) != binaryOperators.end()
189 && ( HasBits(compiler.CfgCompilation, Compilation::AlphabeticOperatorsIgnoreCase)
190 || tokString.Equals<NC>( it.Value() ) ) )
192 scanner.ConsumeChars<NC>( tokString.Length() );
193 token= Tokens::AlphaBinOp;
200 //------------------------------ Identifiers ------------------------------
201 if( isalpha( first ) || first == '_
' )
203 integer endOfIdent= 0;
205 while( ++endOfIdent < scanner.Length()
206 && ( isalnum( next= scanner[endOfIdent] )
209 token= Tokens::Identifier;
210 tokString= String( expression.Buffer() + tokPosition, endOfIdent );
211 scanner.ConsumeChars<NC>( endOfIdent );
215 //------------------------------ numbers ------------------------------
216 if( isdigit( first ) )
218 integer endOfDecPart= 0;
220 while( ++endOfDecPart < scanner.Length()
221 && ( isdigit( next= scanner[endOfDecPart] )
222 || ( HasBits(numberFormat->Flags, NumberFormatFlags::ReadGroupChars) && next== numberFormat->ThousandsGroupChar ) )
227 if( next == numberFormat->DecimalPointChar
230 || scanner.Substring( endOfDecPart ).StartsWith( numberFormat->ExponentSeparator ) )
233 auto oldStart= scanner.Buffer();
235 scanner.ConsumeFloat( value, numberFormat );
236 token = Tokens::LitFloat;
239 String numberParsed( oldStart, scanner.Buffer() - oldStart );
240 tokLiteralHint= numberParsed.IndexOf('e
') > 0
241 || numberParsed.IndexOf('E
') > 0
242 || numberParsed.IndexOf( numberFormat->ExponentSeparator ) > 0
243 ? ASTLiteral::NFHint::Scientific
244 : ASTLiteral::NFHint::NONE;
250 tokLiteralHint= ASTLiteral::NFHint::NONE;
251 if( numberFormat->HexLiteralPrefix.IsNotEmpty()
252 && scanner.StartsWith( numberFormat->HexLiteralPrefix ) ) tokLiteralHint= ASTLiteral::NFHint::Hexadecimal;
253 else if( numberFormat->OctLiteralPrefix.IsNotEmpty()
254 && scanner.StartsWith( numberFormat->OctLiteralPrefix ) ) tokLiteralHint= ASTLiteral::NFHint::Octal;
255 else if( numberFormat->BinLiteralPrefix.IsNotEmpty()
256 && scanner.StartsWith( numberFormat->BinLiteralPrefix ) ) tokLiteralHint= ASTLiteral::NFHint::Binary;
259 scanner.ConsumeInt( value, numberFormat );
260 token= Tokens::LitInteger;
267 //------------------------------ Strings ------------------------------
270 bool lastWasSlash= false;
271 scanner.ConsumeChar<NC>();
273 while( (next= scanner.ConsumeChar()) != '\0' )
275 if( next == '\\' ) { lastWasSlash= true; continue; }
276 if( next == '\"' && !lastWasSlash ) break;
282 Exception e( ALIB_CALLER_NULLED, Exceptions::SyntaxErrorExpectation,
283 EXPRESSIONS.GetResource("EE4") );
284 e.Add ( ALIB_CALLER_NULLED, Exceptions::ExpressionInfo,
285 expression, expression.Length() - scanner.Length() );
289 String quoted( expression.Buffer() + tokPosition + 1,
290 expression.Length() - scanner.Length() - tokPosition -2 );
291 token = Tokens::LitString;
292 tokString.Allocate(compileTimeAllocator, String1K(quoted) << Escape( lang::Switch::Off ) );
296 // -------- unrecognized token ---------
297 Exception e( ALIB_CALLER_NULLED, Exceptions::SyntaxError );
298 e.Add ( ALIB_CALLER_NULLED, Exceptions::ExpressionInfo, expression, expression.Length() - scanner.Length() );
304// #################################################################################################
306// #################################################################################################
307#define Start parseConditional
309detail::AST* ParserImpl::Parse( const String& exprString, NumberFormat* nf )
311 if( exprString.IsEmpty() )
312 throw Exception( ALIB_CALLER, Exceptions::EmptyExpressionString );
314 expression = exprString;
316 ASTs = compileTimeAllocator().New<StdVectorMono<AST*>>( compileTimeAllocator );
323//ALIB_DBG( lexer.DbgListTokens(); )
328 // if tokens remain, an "operator" would be expected
329 if( token != Tokens::EOT )
331 Exception e( ALIB_CALLER_NULLED, Exceptions::SyntaxErrorExpectation, EXPRESSIONS.GetResource("EE5") );
332 e.Add ( ALIB_CALLER_NULLED, Exceptions::ExpressionInfo, expression, tokPosition );
340AST* ParserImpl::parseConditional()
342 // parse lhs as simple
343 push( parseBinary() ); // Q
345 integer qmPosition= tokPosition;
348 if( token == Tokens::SymbolicOp && tokString == A_CHAR("?") )
351 push( Start() ); // T
354 if( token != Tokens::SymbolicOp || tokString != A_CHAR(":") )
356 Exception e( ALIB_CALLER_NULLED, Exceptions::SyntaxErrorExpectation, EXPRESSIONS.GetResource("EE6") );
357 e.Add ( ALIB_CALLER_NULLED, Exceptions::ExpressionInfo, expression, tokPosition );
360 integer colonPosition= tokPosition;
367 return compileTimeAllocator().New<ASTConditional>(Q, T, F, qmPosition, colonPosition );
370 // was no conditional
374AST* ParserImpl::parseBinary()
376 // parse lhs as simple
377 push( parseSimple() );
380 integer position= tokPosition;
384 binOp= getBinaryOp();
388 // rhs is braced? -> lhs becomes <lhs op rhs> and we start over
389 if( token == Tokens::BraceOpen )
391 replace( compileTimeAllocator().New<ASTBinaryOp>(binOp, top(), parseSimple(), position ) );
392 position= tokPosition;
398 // check if tokens remain
399 if( token == Tokens::EOT )
401 Exception e( ALIB_CALLER_NULLED, Exceptions::SyntaxErrorExpectation, EXPRESSIONS.GetResource("EE7") );
402 e.Add ( ALIB_CALLER_NULLED, Exceptions::ExpressionInfo, expression, tokPosition );
407 AST* rhs= push( parseBinary() );
409 int binOpPrecedence= compiler.GetBinaryOperatorPrecedence( binOp );
411 ASTBinaryOp* parent = nullptr;
412 while( replace->NodeType == AST::Types::BinaryOp
413 && compiler.GetBinaryOperatorPrecedence(dynamic_cast<ASTBinaryOp*>(replace)->Operator) <= binOpPrecedence )
415 parent = dynamic_cast<ASTBinaryOp*>(replace);
416 replace= parent->Lhs;
421 if( parent == nullptr )
422 return compileTimeAllocator().New<ASTBinaryOp>( binOp, lhs, rhs, position );
424 // insert binary at lhs of deepest equal-level binary found.
425 // Its current lhs becomes its new lhs-child's rhs.
426 parent->Lhs= compileTimeAllocator().New<ASTBinaryOp>( binOp, lhs, parent->Lhs, position );
433 if( token == Tokens::BraceOpen )
438 if( token != Tokens::BraceClose )
440 Exception e( ALIB_CALLER_NULLED, Exceptions::SyntaxErrorExpectation, EXPRESSIONS.GetResource(
"EE1"));
441 e.Add ( ALIB_CALLER_NULLED, Exceptions::ExpressionInfo, expression, tokPosition );
445 replace( parseSubscript( top() ) );
452 String unOp= getUnaryOp();
453 if( unOp.IsNotNull() )
455 push( compileTimeAllocator().New<ASTUnaryOp>(unOp, parseSimple(), position ) );
456 replace( parseSubscript( top() ) );
462 if( token == Tokens::LitInteger ) { push(compileTimeAllocator().New<ASTLiteral>(tokInteger, position, tokLiteralHint ) ); NextToken(); replace( parseSubscript(top()) );
return pop(); }
463 if( token == Tokens::LitFloat ) { push(compileTimeAllocator().New<ASTLiteral>(tokFloat , position, tokLiteralHint ) ); NextToken(); replace( parseSubscript(top()) );
return pop(); }
464 if( token == Tokens::LitString ) { push(compileTimeAllocator().New<ASTLiteral>(
String(compileTimeAllocator, tokString), position )); NextToken(); replace( parseSubscript(top()) );
return pop(); }
465 if( token == Tokens::Identifier || token == Tokens::AlphaBinOp )
471 if( token == Tokens::BraceOpen )
473 ASTFunction* astFunction= compileTimeAllocator().New<ASTFunction>( name, position, compileTimeAllocator );
478 if( token == Tokens::BraceClose )
483 astFunction->Arguments.emplace_back( Start() );
485 if( token == Tokens::Comma )
488 if( token != Tokens::BraceClose )
496 replace( parseSubscript( astFunction ) );
502 replace( parseSubscript( push(compileTimeAllocator().New<ASTIdentifier>(
String(compileTimeAllocator, name), position ) ) ) );
507 if( token == Tokens::EOT )
514 if( token == Tokens::BraceClose )
521 if( token == Tokens::SubscriptOpen || token == Tokens::SubscriptClose )
528 if( token == Tokens::Comma )
535 ALIB_ERROR(
"EXPR",
"Internal error. This should never happen." )
575 for(
integer partialRead= 1 ; partialRead <=
tokString.Length() ; ++partialRead )
599 return alphabeticOperator;
615 for (
integer partialRead =
tokString.Length(); partialRead > 0; --partialRead )
640 return alphabeticOperator;
const String & GetResource(const NString &name)
Exception & Add(const lang::CallerInfo &ci, TEnum type, TArgs &&... args)
Tokens token
The actual token type.
ALIB_DLL String getUnaryOp()
HashSet< MonoAllocator, String, alib::hash_string_ignore_case< character >, alib::equal_to_string_ignore_case< character > > unaryOperators
Compiler & compiler
The compiler that this parser works for.
AST * parseSubscript(AST *function)
ALIB_DLL String getBinaryOp()
BitSet< 256 > syntaxTokens
String expression
The given expression to parse.
integer tokPosition
The position of the token in expression.
String tokString
String value of token (if applicable).
ParserImpl(Compiler &compiler, MonoAllocator &allocator)
BitSet< 256 > operatorChars
@ SubscriptClose
A closing subscript brace.
@ AlphaBinOp
An alphabetic binary operator.
@ SymbolicOp
A symbolic operator. Can be unary or binary.
@ AlphaUnOp
An alphabetic unary operator.
@ SubscriptOpen
An opening subscript brace.
HashSet< MonoAllocator, String, alib::hash_string_ignore_case< character >, alib::equal_to_string_ignore_case< character > > binaryOperators
void NextToken()
This is the "scanner" or "lexer" method.
MonoAllocator & compileTimeAllocator
#define ALIB_CALLER_NULLED
#define ALIB_ERROR(domain,...)
@ UnknownBinaryOperatorSymbol
Unknown binary operator symbol found when parsing expression string.
@ UnknownUnaryOperatorSymbol
Unknown unary operator symbol found when parsing expression string.
@ SyntaxErrorExpectation
Syntax error with concrete information about what the parser expected at given position.
constexpr String NULL_STRING
A nulled string of the default character type.
lang::integer integer
Type alias in namespace alib.
monomem::TMonoAllocator< lang::HeapAllocator > MonoAllocator
exceptions::Exception Exception
Type alias in namespace alib.
strings::TString< character > String
Type alias in namespace alib.
expressions::ExpressionsCamp EXPRESSIONS
The singleton instance of ALib Camp class ExpressionsCamp.
strings::TSubstring< character > Substring
Type alias in namespace alib.
Abstract syntax tree node representing binary operators.