From 102b68fc27c8a25b75805bfbe7852210dca2b542 Mon Sep 17 00:00:00 2001 From: Eatswap Date: Sat, 10 Dec 2022 19:16:19 +0800 Subject: [PATCH] feat: update tokens --- DragonLisp.l | 75 ++++++++++++++++++++++++++++++++---------------- DragonLisp.y | 80 +++++++++++++--------------------------------------- token.h | 25 +++++++++------- 3 files changed, 85 insertions(+), 95 deletions(-) diff --git a/DragonLisp.l b/DragonLisp.l index cced544..5765fe6 100644 --- a/DragonLisp.l +++ b/DragonLisp.l @@ -20,8 +20,9 @@ using token = DragonLisp::DLParser::token; %option yyclass="DragonLisp::DLScanner" %option verbose backup warn noyywrap c++ nounistd debug noline +float [+-]?[0-9]*[.][0-9]+([eE][+-][0-9]+)? +int [+-]?(0[xX][0-9A-Fa-f]*|0[0-7]+|[1-9][0-9]*) id [a-zA-Z_][a-zA-Z_0-9]* -int [0-9]+ blank [ \t\v\r] comment ;.*$ string \"[^\r\n]*\" @@ -47,32 +48,65 @@ string \"[^\r\n]*\" {blank}+ { loc->step(); std::printf("Skipping blank\n"); + return token::TOKEN_SPACE; }; \n+ { loc->lines(yyleng); loc->step(); std::printf("Skipping newline\n"); + return token::TOKEN_SPACE; } +{float} { + errno = 0; + char* seq_end_ptr = nullptr; + double n = strtod(yytext, &seq_end_str); + if (errno == ERANGE) + throw DragonLisp::DLParser::syntax_error(*loc, "Float out of range: " + std::string(yytext)); + if (seq_end_ptr - yytext < yyleng) + throw DragonLisp::DLParser::syntax_error(*loc, "Invalid float scanned: [" + std::string(yytext, seq_end_ptr) + "], but provided [" + std::string(yytext) + "]"); + yylval->emplace(n); + std::printf("Scanned float: %lf\n", n); + return TOKEN::TOKEN_FLOAT; +} + +{int} { + errno = 0; + char* seq_end_ptr = nullptr; + int64_t n = strtoll(yytext, &seq_end_ptr, 0); + if (errno == ERANGE) + throw DragonLisp::DLParser::syntax_error(*loc, "Integer out of range: " + std::string(yytext)); + if (seq_end_ptr - yytext < yyleng) + throw DragonLisp::DLParser::syntax_error(*loc, "Invalid integer scanned: [" + std::string(yytext, seq_end_ptr) + "], but provided [" + std::string(yytext) + "]"); + yylval->emplace(n); + std::printf("Scanned integer: %lld\n", n); + return token::TOKEN_INTEGER; +}; + "<=" { std::printf("Scanned <=\n"); - return token::TOKEN_LE; + return token::TOKEN_LESS_EQUAL; }; ">=" { std::printf("Scanned >=\n"); - return token::TOKEN_GE; + return token::TOKEN_GREATER_EQUAL; }; "<" { std::printf("Scanned <\n"); - return token::TOKEN_LT; + return token::TOKEN_LESS; }; ">" { std::printf("Scanned >\n"); - return token::TOKEN_GT; + return token::TOKEN_GREATER; +}; + +"/=" { + std::printf("Scanned !=\n"); + return token::TOKEN_NOT_EQUAL; }; "=" { @@ -80,11 +114,6 @@ string \"[^\r\n]*\" return token::TOKEN_EQUAL; }; -"/=" { - std::printf("Scanned !=\n"); - return token::TOKEN_NE; -}; - "(" { std::printf("Scanned (\n"); return token::TOKEN_LPAREN; @@ -107,28 +136,26 @@ string \"[^\r\n]*\" "*" { std::printf("Scanned *\n"); - return token::TOKEN_STAR; + return token::TOKEN_MULTIPLY; }; "/" { std::printf("Scanned /\n"); - return token::TOKEN_SLASH; + return token::TOKEN_DIVIDE; }; -{int} { - errno = 0; - int64_t n = strtoll(yytext, nullptr, 10); - if (errno) - throw DragonLisp::DLParser::syntax_error(*loc, "Invalid integer provided: " + std::string(yytext)); - yylval->emplace(n); - std::printf("Scanned integer: %lld\n", n); - return token::TOKEN_NUMBER; +{id} { + std::printf("Scanned identifier: %s\n", yytext); + yylval->emplace(std::string(yytext, yyleng)); + return token::TOKEN_IDENTIFIER; }; -{id} { /* return DragonLisp::DLParser::make_IDENTIFIER(yytext, *loc); */ } +. { + throw DragonLisp::DLParser::syntax_error(*loc, "Invalid character: " + std::string(yytext)); +}; -. { throw DragonLisp::DLParser::syntax_error(*loc, "Invalid character: " + std::string(yytext)); } - -<> { yyterminate(); } +<> { + yyterminate(); +}; %% diff --git a/DragonLisp.y b/DragonLisp.y index 77b791d..b4ff58a 100644 --- a/DragonLisp.y +++ b/DragonLisp.y @@ -41,78 +41,36 @@ namespace DragonLisp { %define parse.assert %token - LE "<=" - GE ">=" - EQUAL "=" - NE "/=" - LT "<" - GT ">" - LPAREN "(" - RPAREN ")" - PLUS "+" - MINUS "-" - STAR "*" - SLASH "/" + LESS_EQUAL "<=" + GREATER_EQUAL ">=" + LESS "<" + GREATER ">" + NOT_EQUAL "/=" + EQUAL "=" + LPAREN "(" + RPAREN ")" + PLUS "+" + MINUS "-" + MULTIPLY "*" + DIVIDE "/" + SPACE ; -%token END 0 "EOF" -%token NUMBER "number" -%token STRING "string" - -%type basic_operator -%type arithmetic_operator -%type comparsion_operator +%token END 0 "EOF" +%token FLOAT "float" +%token INTEGER "integer" +%token STRING "string" +%token IDENTIFIER "identifier" %define parse.error verbose %% S - : R END - -R - : - | R S-Expr - | STRING { std::cout << "Scanned string -> " << $1 << std::endl; } - -S-Expr - : LPAREN basic_operator NUMBER NUMBER RPAREN { - std::printf("Operator -> %d, LHS -> %lld, RHS -> %lld\n", int($2), $3, $4); - std::printf("This is S-Expr!\n"); - } -; - -basic_operator - : arithmetic_operator { $$ = $1; } - | comparsion_operator { $$ = $1; } -; - -arithmetic_operator - : PLUS { $$ = Token::PLUS; std::printf("I am plus +\n"); } - | MINUS { $$ = Token::MINUS; std::printf("I am minus -\n"); } - | STAR { $$ = Token::MULTIPLY; std::printf("I am star *\n"); } - | SLASH { $$ = Token::DIVIDE; std::printf("I am slash /\n"); } -; - -comparsion_operator - : LE { $$ = Token::LE; } - | LT { $$ = Token::LT; } - | GE { $$ = Token::GE; } - | GT { $$ = Token::GT; } - | EQUAL { $$ = Token::EQUAL; } - | NE { $$ = Token::NE; } + : END %% void DragonLisp::DLParser::error(const location_type& l, const std::string& msg) { std::cerr << "Error: " << msg << " at " << l << "\n"; } - -DragonLisp::DLParser::symbol_type make_NUMBER(const std::string& s, const DragonLisp::DLParser::location_type& loc) { - try { - int n = std::stoi(s); - return DragonLisp::DLParser::make_NUMBER(n, loc); - } catch (...) { - throw DragonLisp::DLParser::syntax_error(loc, "Invalid integer provided: " + s); - } -} diff --git a/token.h b/token.h index 4fb9e0f..3dfc45b 100644 --- a/token.h +++ b/token.h @@ -4,17 +4,22 @@ namespace DragonLisp { enum Token { - LE, - GE, - EQUAL, - NE, - LT, - GT, - PLUS, - MINUS, - MULTIPLY, - DIVIDE, STRING, + SPACE, + FLOAT, + INTEGER, + LESS_EQUAL, + GREATER_EQUAL, + LESS, + GREATER, + NOT_EQUAL, + EQUAL, + LPAREN, + RPAREN, + PLUS, + MINUS, + MULTIPLY, + DIVIDE, }; }