#ifndef __NASAL_LEXER_H__ #define __NASAL_LEXER_H__ #define IS_IDENTIFIER_HEAD(c) ((c=='_')||('a'<=c && c<='z')||('A'<=c&&c<='Z')) #define IS_IDENTIFIER_BODY(c) ((c=='_')||('a'<=c && c<='z')||('A'<=c&&c<='Z')||('0'<=c&&c<='9')) #define IS_HEX_NUMBER(c) (('0'<=c&&c<='9')||('a'<=c&&c<='f')||('A'<=c && c<='F')) #define IS_OCT_NUMEBR(c) ('0'<=c&&c<='7') #define IS_DIGIT(c) ('0'<=c&&c<='9') #define IS_STRING_HEAD(c) (c=='\''||c=='\"') // single operators have only one character #define IS_SINGLE_OPRATOR(c) (c=='('||c==')'||c=='['||c==']'||c=='{'||c=='}'||c==','||c==';'||c=='|'||c==':'||\ c=='?'||c=='`'||c=='&'||c=='@'||c=='%'||c=='$'||c=='^'||c=='\\') // calculation operators may have two chars, for example: += -= *= /= ~= != == >= <= #define IS_CALC_OPERATOR(c) (c=='='||c=='+'||c=='-'||c=='*'||c=='!'||c=='/'||c=='<'||c=='>'||c=='~') #define IS_NOTE_HEAD(c) (c=='#') #ifndef TOKEN_TABLE_SIZE #define TOKEN_TABLE_SIZE 45 struct token_table { std::string str; int tok_type; }tok_tbl[TOKEN_TABLE_SIZE]= { {"for" ,tok_for }, {"forindex",tok_forindex }, {"foreach" ,tok_foreach }, {"while" ,tok_while }, {"var" ,tok_var }, {"func" ,tok_func }, {"break" ,tok_break }, {"continue",tok_continue }, {"return" ,tok_return }, {"if" ,tok_if }, {"elsif" ,tok_elsif }, {"else" ,tok_else }, {"nil" ,tok_nil }, {"(" ,tok_left_curve }, {")" ,tok_right_curve }, {"[" ,tok_left_bracket }, {"]" ,tok_right_bracket}, {"{" ,tok_left_brace }, {"}" ,tok_right_brace }, {";" ,tok_semi }, {"and" ,tok_and }, {"or" ,tok_or }, {"," ,tok_comma }, {"." ,tok_dot }, {"..." ,tok_ellipsis }, {"?" ,tok_quesmark }, {":" ,tok_colon }, {"+" ,tok_add }, {"-" ,tok_sub }, {"*" ,tok_mult }, {"/" ,tok_div }, {"~" ,tok_link }, {"!" ,tok_not }, {"=" ,tok_equal }, {"+=" ,tok_add_equal }, {"-=" ,tok_sub_equal }, {"*=" ,tok_mult_equal }, {"/=" ,tok_div_equal }, {"~=" ,tok_link_equal }, {"==" ,tok_cmp_equal }, {"!=" ,tok_cmp_not_equal}, {"<" ,tok_less_than }, {">" ,tok_greater_than }, {"<=" ,tok_less_equal }, {">=" ,tok_greater_equal}, }; #endif struct token { int line; int type; std::string str; }; class nasal_lexer { private: int error; std::vector token_list; std::string identifier_gen(std::vector&,int&,int&); void generate_number_error(int,std::string); std::string number_gen(std::vector&,int&,int&); std::string string_gen(std::vector&,int&,int&); public: void clear(); void scanner(std::vector&); void print_token(); int get_error(); std::vector& get_token_list(); }; void nasal_lexer::clear() { token_list.clear(); return; } std::string nasal_lexer::identifier_gen(std::vector& res,int& ptr,int& line) { int res_size=res.size(); std::string token_str=""; while(ptr> [lexer] line "<& res,int& ptr,int& line) { int res_size=res.size(); bool scientific_notation=false;// numbers like 1e8 are scientific_notation std::string token_str=""; // generate hex number if(res[ptr]=='0' && ptr+1 0|[1~9][0~9]*(.[0~9]*)(e|E(+|-)0|[1~9][0~9]*) if(ptr=res_size) { generate_number_error(line,token_str); return "0"; } while(ptr=res_size) { generate_number_error(line,token_str); return "0"; } if(ptr=res_size) { generate_number_error(line,token_str); return "0"; } if(ptr& res,int& ptr,int& line) { int res_size=res.size(); std::string token_str=""; char str_begin=res[ptr++]; if(ptr>=res_size) return token_str; while(ptr=res_size) { ++error; std::cout<<">> [lexer] line "<& res) { error=0; token_list.clear(); int line=1,ptr=0,res_size=res.size(); std::string token_str; while(ptr=res_size) break; if(IS_IDENTIFIER_HEAD(res[ptr])) { token_str=identifier_gen(res,ptr,line); token new_token; new_token.line=line; new_token.str=token_str; new_token.type=0; for(int i=0;i> [lexer] line "<& nasal_lexer::get_token_list() { return token_list; } #endif