diff --git a/version3.0/main.cpp b/version3.0/main.cpp index 1964486..9025e7e 100644 --- a/version3.0/main.cpp +++ b/version3.0/main.cpp @@ -11,6 +11,7 @@ void logo() } nasal_resource resource; +nasal_lexer lexer; std::string command; int main() { @@ -74,7 +75,11 @@ int main() resource.print_file(); else if(command=="lex") { - ; + lexer.scanner(resource.get_file()); + if(!lexer.get_error()) + lexer.print_token(); + else + std::cout<<">> [lexer] error occurred,stop.\n"; } else if(command=="par") { diff --git a/version3.0/nasal.h b/version3.0/nasal.h index 96e6474..69e817b 100644 --- a/version3.0/nasal.h +++ b/version3.0/nasal.h @@ -16,7 +16,9 @@ #include #include "nasal_enum.h" +#include "nasal_misc.h" #include "nasal_resource.h" #include "nasal_lexer.h" +#include "nasal_parse.h" #endif \ No newline at end of file diff --git a/version3.0/nasal_enum.h b/version3.0/nasal_enum.h index a0e13d6..b7ed275 100644 --- a/version3.0/nasal_enum.h +++ b/version3.0/nasal_enum.h @@ -5,7 +5,7 @@ enum token_type { tok_null=0, - tok_number,tok_string, + tok_number,tok_string,tok_identifier, tok_for,tok_forindex,tok_foreach,tok_while, tok_var,tok_func,tok_break,tok_continue, tok_return,tok_if,tok_elsif,tok_else,tok_nil, diff --git a/version3.0/nasal_lexer.h b/version3.0/nasal_lexer.h index 6fda5ae..889b9f3 100644 --- a/version3.0/nasal_lexer.h +++ b/version3.0/nasal_lexer.h @@ -79,10 +79,16 @@ struct token class nasal_lexer { private: + int error; std::vector token_list; public: + std::string identifier_gen(std::vector&,int&,int&); + std::string number_gen(std::vector&,int&,int&); + std::string string_gen(std::vector&,int&,int&); void delete_tokens(); void scanner(std::vector&); + void print_token(); + int get_error(); }; void nasal_lexer::delete_tokens() @@ -91,9 +97,230 @@ void nasal_lexer::delete_tokens() return; } +std::string nasal_lexer::identifier_gen(std::vector& res,int& ptr,int& line) +{ + int res_size=res.size(); + std::string token_str=""; + while(ptr& res,int& ptr,int& line) +{ + int res_size=res.size(); + bool scientific_notation=false;// numbers like 1e8 are scientific_notation + std::string token_str=""; + while(ptr> [lexer] line "<& res,int& ptr,int& line) +{ + int res_size=res.size(); + std::string token_str=""; + char str_begin=res[ptr]; + ++ptr; + if(ptr>=res_size) return token_str; + while(ptr=res_size) + { + ++error; + std::cout<<">> [lexer] line "<& res) { + error=0; + token_list.clear(); + int line=1,ptr=0,res_size=res.size(); + std::string token_str; + while(ptr=res_size) break; + if(IS_IDENTIFIER_HEAD(res[ptr])) + { + token_str=identifier_gen(res,ptr,line); + token new_token; + new_token.line=line; + new_token.str=token_str; + new_token.type=0; + for(int i=0;i> [lexer] line "<> [lexer] complete scanning. "<'7') + return false; + return true; +} +inline bool check_dec_string(std::string str,int len) +{ + int dot_cnt=0; + if(str[0]=='.') return false; + if(str[0]=='0' && ('0'<=str[1] && str[1]<='9')) return false; + int i=0; + for(;i'9' ) return false; + } + if(str[i]=='e' || str[i]=='E') + { + ++i; + if(i==len) return false; + if(str[i]=='-') + { + ++i; + if(i==len) return false; + } + for(;i'9') + return false; + } + if(dot_cnt>1) return false; + return true; +} + +bool check_numerable_string(std::string str) +{ + int len=str.length(); + if(!len) return false; + if(str[0]=='-' && len>1) + { + std::string tmp=""; + for(int i=1;i2 && str[0]=='0' && str[1]=='x') + return check_hex_string(str,len); + else if(len>2 && str[0]=='0' && str[1]=='o') + return check_oct_string(str,len); + else if('0'<=str[0] && str[0]<='9') + return check_dec_string(str,len); + return false; +} + +/* + trans_string_to_number: + convert string to number +*/ +inline double hex_to_double(std::string str,int len) +{ + double ret=0; + double num_pow=1; + for(int i=len-1;i>1;--i) + { + if('0'<=str[i] && str[i]<='9') + ret+=num_pow*(str[i]-'0'); + else if('a'<=str[i] && str[i]<='f') + ret+=num_pow*(str[i]-'a'+10); + else if('A'<=str[i] && str[i]<='F') + ret+=num_pow*(str[i]-'A'+10); + num_pow*=16; + } + return ret; +} +inline double oct_to_double(std::string str,int len) +{ + double ret=0; + double num_pow=1; + for(int i=len-1;i>1;--i) + { + ret+=num_pow*(str[i]-'0'); + num_pow*=8; + } + return ret; +} +inline double dec_to_double(std::string str,int len) +{ + double ret=0; + int i=0; + for(;i1) + { + is_negative=true; + std::string tmp=""; + for(int i=1;i2 && str[0]=='0' && str[1]=='x') + ret_num=hex_to_double(str,len); + else if(len>2 && str[0]=='0' && str[1]=='o') + ret_num=oct_to_double(str,len); + else if('0'<=str[0] && str[0]<='9') + ret_num=dec_to_double(str,len); + return is_negative?-ret_num:ret_num; +} + +/* + trans_number_to_string: + convert number to string +*/ +std::string trans_number_to_string(double number) +{ + std::string trans_num_string=""; + if(number<0) + { + trans_num_string+='-'; + number=-number; + } + double integer_bit=1; + while(number>=integer_bit) + integer_bit*=10; + integer_bit/=10; + while(integer_bit!=0.1) + { + trans_num_string+=(char)('0'+(int(number/integer_bit))); + number-=(double)(int(number/integer_bit))*integer_bit; + integer_bit/=10; + } + if(number!=0) + trans_num_string+='.'; + while(number!=0) + { + trans_num_string+=(char)('0'+int(number*10)); + number*=10; + number-=(double)(int(number)); + } + return trans_num_string; +} + +/* + prt_hex: + transform int to hex format and print it out (std::cout) +*/ +void prt_hex(const int ptr) +{ + char hex[9]; + hex[8]=0; + int tmp_plc=ptr; + if(tmp_plc<0) + { + tmp_plc=-tmp_plc; + std::cout<<"-0x"; + } + else + std::cout<<"0x"; + /* + int: 00000000 00000000 00000000 00000000 + int: 0x00 00 00 00 + example: + a=0x13 57 9b df + a=00010011 01010111 10011011 11011111 + a & 0x00 00 00 0f: + 00010011 01010111 10011011 11011111 + and 00000000 00000000 00000000 00001111 + --------------------------------------- + 00000000 00000000 00000000 00001111 + a>>=4: + 00000001 00110101 01111001 10111101 + a & 0x00 00 00 0f + 00000001 00110101 01111001 10111101 + and 00000000 00000000 00000000 00001111 + --------------------------------------- + 00000000 00000000 00000000 00001101 + then convert 0~15 to 0~9 a~f + */ + for(int j=7;j>=0;--j) + { + int tmp=(tmp_plc & 0x0000000f); + hex[j]=tmp<10? (char)('0'+tmp):(char)('a'+tmp-10); + tmp_plc>>=4; + } + std::cout<