#ifndef __NASAL_LEXER_H__ #define __NASAL_LEXER_H__ #define ID(c) ((c=='_')||('a'<=c && c<='z')||('A'<=c&&c<='Z')) #define HEX(c) (('0'<=c&&c<='9')||('a'<=c&&c<='f')||('A'<=c && c<='F')) #define OCT(c) ('0'<=c&&c<='7') #define DIGIT(c) ('0'<=c&&c<='9') #define STR(c) (c=='\''||c=='\"'||c=='`') // single operators have only one character #define SINGLE_OPERATOR(c) (c=='('||c==')'||c=='['||c==']'||c=='{'||c=='}'||c==','||c==';'||c=='|'||c==':'||\ c=='?'||c=='`'||c=='&'||c=='@'||c=='%'||c=='$'||c=='^'||c=='\\') // calculation operators may have two chars, for example: += -= *= /= ~= != == >= <= #define CALC_OPERATOR(c) (c=='='||c=='+'||c=='-'||c=='*'||c=='!'||c=='/'||c=='<'||c=='>'||c=='~') #define NOTE(c) (c=='#') enum token_type { tok_null=0,// null token default token type tok_num, // number basic token type tok_str, // string basic token type tok_id, // identifier basic token type tok_for,tok_forindex,tok_foreach,tok_while, tok_var,tok_func,tok_break,tok_continue, tok_ret,tok_if,tok_elsif,tok_else,tok_nil, tok_lcurve,tok_rcurve, tok_lbracket,tok_rbracket, tok_lbrace,tok_rbrace, tok_semi,tok_and,tok_or,tok_comma,tok_dot,tok_ellipsis,tok_quesmark, tok_colon,tok_add,tok_sub,tok_mult,tok_div,tok_link,tok_not, tok_eq, tok_addeq,tok_subeq,tok_multeq,tok_diveq,tok_lnkeq, tok_cmpeq,tok_neq,tok_less,tok_leq,tok_grt,tok_geq, tok_eof // end of token list }; struct { const char* str; const uint32_t tok_type; }token_table[]= { {"for" ,tok_for }, {"forindex",tok_forindex }, {"foreach" ,tok_foreach }, {"while" ,tok_while }, {"var" ,tok_var }, {"func" ,tok_func }, {"break" ,tok_break }, {"continue",tok_continue }, {"return" ,tok_ret }, {"if" ,tok_if }, {"elsif" ,tok_elsif }, {"else" ,tok_else }, {"nil" ,tok_nil }, {"(" ,tok_lcurve }, {")" ,tok_rcurve }, {"[" ,tok_lbracket }, {"]" ,tok_rbracket }, {"{" ,tok_lbrace }, {"}" ,tok_rbrace }, {";" ,tok_semi }, {"and" ,tok_and }, {"or" ,tok_or }, {"," ,tok_comma }, {"." ,tok_dot }, {"..." ,tok_ellipsis }, {"?" ,tok_quesmark }, {":" ,tok_colon }, {"+" ,tok_add }, {"-" ,tok_sub }, {"*" ,tok_mult }, {"/" ,tok_div }, {"~" ,tok_link }, {"!" ,tok_not }, {"=" ,tok_eq }, {"+=" ,tok_addeq }, {"-=" ,tok_subeq }, {"*=" ,tok_multeq }, {"/=" ,tok_diveq }, {"~=" ,tok_lnkeq }, {"==" ,tok_cmpeq }, {"!=" ,tok_neq }, {"<" ,tok_less }, {"<=" ,tok_leq }, {">" ,tok_grt }, {">=" ,tok_geq }, {nullptr ,0 } }; struct token { uint32_t line; uint32_t type; std::string str; token(uint32_t l=0,uint32_t t=tok_null,std::string s=""){line=l;type=t;str=s;} }; class nasal_lexer { private: uint32_t error; uint32_t line; uint32_t ptr; size_t size; std::string code; std::string res; std::vector tokens; uint32_t get_type(const std::string&); void die(const char*); void open(const std::string&); std::string id_gen(); std::string num_gen(); std::string str_gen(); public: uint32_t err(){return error;} void scan(const std::string&); void print(); const std::vector& get_tokens() const {return tokens;} }; void nasal_lexer::open(const std::string& file) { error=0; std::ifstream fin(file,std::ios::binary); if(fin.fail()) { ++error; std::cout<<"[lexer] cannot open file <"<.\n"; } std::stringstream ss; ss< [0~9][0~9]*(.[0~9]*)(e|E(+|-)0|[1~9][0~9]*) std::string str=""; while(ptr=size) die("get EOF when generating string."); code+=res[ptr-1]; if(begin=='`' && str.length()!=1) die("\'`\' is used for string that includes one character."); return str; } void nasal_lexer::scan(const std::string& file) { open(file); tokens.clear(); line=1; ptr=0; code=""; size=res.size(); std::string str; while(ptr=size) break; if(ID(res[ptr])) { str=id_gen(); uint32_t type=get_type(str); tokens.push_back({line,type?type:tok_id,str}); } else if(DIGIT(res[ptr])) tokens.push_back({line,tok_num,num_gen()}); else if(STR(res[ptr])) tokens.push_back({line,tok_str,str_gen()}); else if(SINGLE_OPERATOR(res[ptr])) { str=res[ptr]; code+=res[ptr]; uint32_t type=get_type(str); if(!type) die("incorrect operator."); tokens.push_back({line,type,str}); ++ptr; } else if(res[ptr]=='.') { str="."; if(ptr+2