📝 rename lexical tokens

This commit is contained in:
ValKmjolnir 2024-05-21 23:31:21 +08:00
parent 2c851613ce
commit 77a14699f4
9 changed files with 564 additions and 489 deletions

View File

@ -34,7 +34,8 @@ public:
tcgetattr(0, &init_termios);
new_termios = init_termios;
new_termios.c_lflag &= ~(ICANON|ECHO|ECHONL|ECHOE);
// vmin=0 is nonblock input, but in wsl there is a bug that will block input
// vmin = 0 is nonblock input,
// but in wsl1 there is a bug that will block input
// so we use fcntl to write the nonblock input
new_termios.c_cc[VMIN] = 1;
new_termios.c_cc[VTIME] = 0;

View File

@ -239,9 +239,9 @@ void codegen::func_gen(function* node) {
}
}
usize newf=code.size();
const auto newf = code.size();
emit(op_newf, 0, node->get_location());
usize lsize=code.size();
const auto lsize = code.size();
emit(op_intl, 0, node->get_location());
// add special keyword 'me' into symbol table

View File

@ -380,7 +380,7 @@ void gc::context_change(nas_co* co) {
}
void gc::context_reserve() {
// pc=0 means this coroutine is finished
// pc = 0 means this coroutine is finished
cort->status = running_context->pc?
nas_co::status::suspended:
nas_co::status::dead;

View File

@ -96,7 +96,7 @@ void lexer::open(const std::string& file) {
}
tok lexer::get_type(const std::string& str) {
return token_mapper.count(str)? token_mapper.at(str):tok::null;
return token_mapper.count(str)? token_mapper.at(str):tok::tk_null;
}
std::string lexer::utf8_gen() {
@ -152,7 +152,8 @@ token lexer::id_gen() {
tok type = get_type(str);
return {
{begin_line, begin_column, line, column, filename},
(type!=tok::null)? type:tok::id, str
(type!=tok::tk_null)? type:tok::tk_id,
str
};
}
@ -174,7 +175,11 @@ token lexer::num_gen() {
"invalid number `"+str+"`"
);
}
return {{begin_line, begin_column, line, column, filename}, tok::num, str};
return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
str
};
} else if (ptr+1<res.size() && res[ptr]=='0' && res[ptr+1]=='o') { // generate oct number
std::string str = "0o";
ptr += 2;
@ -193,7 +198,11 @@ token lexer::num_gen() {
"invalid number `"+str+"`"
);
}
return {{begin_line, begin_column, line, column, filename}, tok::num, str};
return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
str
};
}
// generate dec number
// dec number -> [0~9][0~9]*(.[0~9]*)(e|E(+|-)0|[1~9][0~9]*)
@ -213,7 +222,11 @@ token lexer::num_gen() {
{begin_line, begin_column, line, column, filename},
"invalid number `"+str+"`"
);
return {{begin_line, begin_column, line, column, filename}, tok::num, "0"};
return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
"0"
};
}
}
if (ptr<res.size() && (res[ptr]=='e' || res[ptr]=='E')) {
@ -231,11 +244,19 @@ token lexer::num_gen() {
{begin_line, begin_column, line, column, filename},
"invalid number `"+str+"`"
);
return {{begin_line, begin_column, line, column, filename}, tok::num, "0"};
return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
"0"
};
}
}
column += str.length();
return {{begin_line, begin_column, line, column, filename}, tok::num, str};
return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
str
};
}
token lexer::str_gen() {
@ -283,7 +304,11 @@ token lexer::str_gen() {
{begin_line, begin_column, line, column, filename},
"get EOF when generating string"
);
return {{begin_line, begin_column, line, column, filename}, tok::str, str};
return {
{begin_line, begin_column, line, column, filename},
tok::tk_str,
str
};
}
++column;
@ -294,7 +319,11 @@ token lexer::str_gen() {
"\'`\' is used for string including one character"
);
}
return {{begin_line, begin_column, line, column, filename}, tok::str, str};
return {
{begin_line, begin_column, line, column, filename},
tok::tk_str,
str
};
}
token lexer::single_opr() {
@ -303,7 +332,7 @@ token lexer::single_opr() {
std::string str(1, res[ptr]);
++column;
tok type = get_type(str);
if (type==tok::null) {
if (type==tok::tk_null) {
err.err("lexer",
{begin_line, begin_column, line, column, filename},
"invalid operator `"+str+"`"
@ -380,10 +409,14 @@ const error& lexer::scan(const std::string& file) {
}
if (toks.size()) {
// eof token's location is the last token's location
toks.push_back({toks.back().loc, tok::eof, "<eof>"});
toks.push_back({toks.back().loc, tok::tk_eof, "<eof>"});
} else {
// if token sequence is empty, generate a default location
toks.push_back({{line, column, line, column, filename}, tok::eof, "<eof>"});
toks.push_back({
{line, column, line, column, filename},
tok::tk_eof,
"<eof>"
});
}
res = "";
return err;

View File

@ -16,66 +16,66 @@
namespace nasal {
enum class tok:u32 {
null=0, // null token (default token type)
num, // number literal
str, // string literal
id, // identifier
tktrue, // keyword true
tkfalse, // keyword false
use, // keyword use
rfor, // loop keyword for
forindex, // loop keyword forindex
foreach, // loop keyword foreach
rwhile, // loop keyword while
var, // keyword for definition
func, // keyword for definition of function
brk, // loop keyword break
cont, // loop keyword continue
ret, // function keyword return
rif, // condition expression keyword if
elsif, // condition expression keyword elsif
relse, // condition expression keyword else
nil, // nil literal
lcurve, // (
rcurve, // )
lbracket, // [
rbracket, // ]
lbrace, // {
rbrace, // }
semi, // ;
opand, // operator and
opor, // operator or
comma, // ,
dot, // .
ellipsis, // ...
quesmark, // ?
colon, // :
add, // operator +
sub, // operator -
mult, // operator *
div, // operator /
floater, // operator ~ and binary operator ~
btand, // bitwise operator &
btor, // bitwise operator |
btxor, // bitwise operator ^
opnot, // operator !
eq, // operator =
addeq, // operator +=
subeq, // operator -=
multeq, // operator *=
diveq, // operator /=
lnkeq, // operator ~=
btandeq, // operator &=
btoreq, // operator |=
btxoreq, // operator ^=
cmpeq, // operator ==
neq, // operator !=
less, // operator <
leq, // operator <=
grt, // operator >
geq, // operator >=
eof // <eof> end of token list
enum class tok {
tk_null = 0, // null token (default token type)
tk_num, // number literal
tk_str, // string literal
tk_id, // identifier
tk_true, // keyword true
tk_false, // keyword false
tk_use, // keyword use
tk_for, // loop keyword for
tk_forindex, // loop keyword forindex
tk_foreach, // loop keyword foreach
tk_while, // loop keyword while
tk_var, // keyword for definition
tk_func, // keyword for definition of function
tk_brk, // loop keyword break
tk_cont, // loop keyword continue
tk_ret, // function keyword return
tk_if, // condition expression keyword if
tk_elsif, // condition expression keyword elsif
tk_else, // condition expression keyword else
tk_nil, // nil literal
tk_lcurve, // (
tk_rcurve, // )
tk_lbracket, // [
tk_rbracket, // ]
tk_lbrace, // {
tk_rbrace, // }
tk_semi, // ;
tk_and, // operator and
tk_or, // operator or
tk_comma, // ,
tk_dot, // .
tk_ellipsis, // ...
tk_quesmark, // ?
tk_colon, // :
tk_add, // operator +
tk_sub, // operator -
tk_mult, // operator *
tk_div, // operator /
tk_floater, // operator ~ and binary operator ~
tk_btand, // bitwise operator &
tk_btor, // bitwise operator |
tk_btxor, // bitwise operator ^
tk_not, // operator !
tk_eq, // operator =
tk_addeq, // operator +=
tk_subeq, // operator -=
tk_multeq, // operator *=
tk_diveq, // operator /=
tk_lnkeq, // operator ~=
tk_btandeq, // operator &=
tk_btoreq, // operator |=
tk_btxoreq, // operator ^=
tk_cmpeq, // operator ==
tk_neq, // operator !=
tk_less, // operator <
tk_leq, // operator <=
tk_grt, // operator >
tk_geq, // operator >=
tk_eof // <eof> end of token list
};
struct token {
@ -99,60 +99,60 @@ private:
std::vector<token> toks;
const std::unordered_map<std::string, tok> token_mapper = {
{"use" ,tok::use },
{"true" ,tok::tktrue },
{"false" ,tok::tkfalse },
{"for" ,tok::rfor },
{"forindex",tok::forindex},
{"foreach" ,tok::foreach },
{"while" ,tok::rwhile },
{"var" ,tok::var },
{"func" ,tok::func },
{"break" ,tok::brk },
{"continue",tok::cont },
{"return" ,tok::ret },
{"if" ,tok::rif },
{"elsif" ,tok::elsif },
{"else" ,tok::relse },
{"nil" ,tok::nil },
{"(" ,tok::lcurve },
{")" ,tok::rcurve },
{"[" ,tok::lbracket},
{"]" ,tok::rbracket},
{"{" ,tok::lbrace },
{"}" ,tok::rbrace },
{";" ,tok::semi },
{"and" ,tok::opand },
{"or" ,tok::opor },
{"," ,tok::comma },
{"." ,tok::dot },
{"..." ,tok::ellipsis},
{"?" ,tok::quesmark},
{":" ,tok::colon },
{"+" ,tok::add },
{"-" ,tok::sub },
{"*" ,tok::mult },
{"/" ,tok::div },
{"~" ,tok::floater },
{"&" ,tok::btand },
{"|" ,tok::btor },
{"^" ,tok::btxor },
{"!" ,tok::opnot },
{"=" ,tok::eq },
{"+=" ,tok::addeq },
{"-=" ,tok::subeq },
{"*=" ,tok::multeq },
{"/=" ,tok::diveq },
{"~=" ,tok::lnkeq },
{"&=" ,tok::btandeq },
{"|=" ,tok::btoreq },
{"^=" ,tok::btxoreq },
{"==" ,tok::cmpeq },
{"!=" ,tok::neq },
{"<" ,tok::less },
{"<=" ,tok::leq },
{">" ,tok::grt },
{">=" ,tok::geq }
{"use" , tok::tk_use },
{"true" , tok::tk_true },
{"false" , tok::tk_false },
{"for" , tok::tk_for },
{"forindex", tok::tk_forindex},
{"foreach" , tok::tk_foreach },
{"while" , tok::tk_while },
{"var" , tok::tk_var },
{"func" , tok::tk_func },
{"break" , tok::tk_brk },
{"continue", tok::tk_cont },
{"return" , tok::tk_ret },
{"if" , tok::tk_if },
{"elsif" , tok::tk_elsif },
{"else" , tok::tk_else },
{"nil" , tok::tk_nil },
{"(" , tok::tk_lcurve },
{")" , tok::tk_rcurve },
{"[" , tok::tk_lbracket},
{"]" , tok::tk_rbracket},
{"{" , tok::tk_lbrace },
{"}" , tok::tk_rbrace },
{";" , tok::tk_semi },
{"and" , tok::tk_and },
{"or" , tok::tk_or },
{"," , tok::tk_comma },
{"." , tok::tk_dot },
{"..." , tok::tk_ellipsis},
{"?" , tok::tk_quesmark},
{":" , tok::tk_colon },
{"+" , tok::tk_add },
{"-" , tok::tk_sub },
{"*" , tok::tk_mult },
{"/" , tok::tk_div },
{"~" , tok::tk_floater },
{"&" , tok::tk_btand },
{"|" , tok::tk_btor },
{"^" , tok::tk_btxor },
{"!" , tok::tk_not },
{"=" , tok::tk_eq },
{"+=" , tok::tk_addeq },
{"-=" , tok::tk_subeq },
{"*=" , tok::tk_multeq },
{"/=" , tok::tk_diveq },
{"~=" , tok::tk_lnkeq },
{"&=" , tok::tk_btandeq },
{"|=" , tok::tk_btoreq },
{"^=" , tok::tk_btxoreq },
{"==" , tok::tk_cmpeq },
{"!=" , tok::tk_neq },
{"<" , tok::tk_less },
{"<=" , tok::tk_leq },
{">" , tok::tk_grt },
{">=" , tok::tk_geq }
};
tok get_type(const std::string&);

File diff suppressed because it is too large Load Diff

View File

@ -16,66 +16,68 @@ class parse {
private:
u64 ptr;
u64 in_func; // count function block
u64 in_loop; // count loop block
u64 in_func_depth; // count function block
u64 in_loop_depth; // count loop block
const token* toks;
code_block* root;
error err;
private:
const std::unordered_map<tok, std::string> tokname {
{tok::use ,"use" },
{tok::rfor ,"for" },
{tok::forindex,"forindex"},
{tok::foreach ,"foreach" },
{tok::rwhile ,"while" },
{tok::var ,"var" },
{tok::func ,"func" },
{tok::brk ,"break" },
{tok::cont ,"continue"},
{tok::ret ,"return" },
{tok::rif ,"if" },
{tok::elsif ,"elsif" },
{tok::relse ,"else" },
{tok::nil ,"nil" },
{tok::lcurve ,"(" },
{tok::rcurve ,")" },
{tok::lbracket,"[" },
{tok::rbracket,"]" },
{tok::lbrace ,"{" },
{tok::rbrace ,"}" },
{tok::semi ,";" },
{tok::opand ,"and" },
{tok::opor ,"or" },
{tok::comma ,"," },
{tok::dot ,"." },
{tok::ellipsis,"..." },
{tok::quesmark,"?" },
{tok::colon ,":" },
{tok::add ,"+" },
{tok::sub ,"-" },
{tok::mult ,"*" },
{tok::div ,"/" },
{tok::floater ,"~" },
{tok::btand ,"&" },
{tok::btor ,"|" },
{tok::btxor ,"^" },
{tok::opnot ,"!" },
{tok::eq ,"=" },
{tok::addeq ,"+=" },
{tok::subeq ,"-=" },
{tok::multeq ,"*=" },
{tok::diveq ,"/=" },
{tok::lnkeq ,"~=" },
{tok::btandeq ,"&=" },
{tok::btoreq ,"|=" },
{tok::btxoreq ,"^=" },
{tok::cmpeq ,"==" },
{tok::neq ,"!=" },
{tok::less ,"<" },
{tok::leq ,"<=" },
{tok::grt ,">" },
{tok::geq ,">=" }
const std::unordered_map<tok, std::string> tokname = {
{tok::tk_true , "true" },
{tok::tk_false , "false" },
{tok::tk_use , "use" },
{tok::tk_for , "for" },
{tok::tk_forindex, "forindex"},
{tok::tk_foreach , "foreach" },
{tok::tk_while , "while" },
{tok::tk_var , "var" },
{tok::tk_func , "func" },
{tok::tk_brk , "break" },
{tok::tk_cont , "continue"},
{tok::tk_ret , "return" },
{tok::tk_if , "if" },
{tok::tk_elsif , "elsif" },
{tok::tk_else , "else" },
{tok::tk_nil , "nil" },
{tok::tk_lcurve , "(" },
{tok::tk_rcurve , ")" },
{tok::tk_lbracket, "[" },
{tok::tk_rbracket, "]" },
{tok::tk_lbrace , "{" },
{tok::tk_rbrace , "}" },
{tok::tk_semi , ";" },
{tok::tk_and , "and" },
{tok::tk_or , "or" },
{tok::tk_comma , "," },
{tok::tk_dot , "." },
{tok::tk_ellipsis, "..." },
{tok::tk_quesmark, "?" },
{tok::tk_colon , ":" },
{tok::tk_add , "+" },
{tok::tk_sub , "-" },
{tok::tk_mult , "*" },
{tok::tk_div , "/" },
{tok::tk_floater , "~" },
{tok::tk_btand , "&" },
{tok::tk_btor , "|" },
{tok::tk_btxor , "^" },
{tok::tk_not , "!" },
{tok::tk_eq , "=" },
{tok::tk_addeq , "+=" },
{tok::tk_subeq , "-=" },
{tok::tk_multeq , "*=" },
{tok::tk_diveq , "/=" },
{tok::tk_lnkeq , "~=" },
{tok::tk_btandeq , "&=" },
{tok::tk_btoreq , "|=" },
{tok::tk_btxoreq , "^=" },
{tok::tk_cmpeq , "==" },
{tok::tk_neq , "!=" },
{tok::tk_less , "<" },
{tok::tk_leq , "<=" },
{tok::tk_grt , ">" },
{tok::tk_geq , ">=" }
};
private:
@ -151,7 +153,8 @@ public:
}
public:
parse(): ptr(0), in_func(0), in_loop(0), toks(nullptr), root(nullptr) {}
parse(): ptr(0), in_func_depth(0), in_loop_depth(0),
toks(nullptr), root(nullptr) {}
~parse() {delete root;}
const error& compile(const lexer&);
static void easter_egg();

View File

@ -50,6 +50,7 @@ protected:
/* limited mode, will not load unsafe system api if switched on */
bool flag_limited_mode = false;
protected:
/* vm initializing function */
void vm_init_enrty(const std::vector<std::string>&,
const std::vector<f64>&,
@ -60,6 +61,7 @@ protected:
const std::vector<std::string>&);
void context_and_global_init();
protected:
/* debug functions */
bool verbose = false;
void value_info(var&);
@ -190,13 +192,24 @@ public:
const std::vector<std::string>&); // get command line arguments
/* set detail report info flag */
void set_detail_report_info(bool flag) {verbose = flag;}
void set_detail_report_info(bool flag) {
verbose = flag;
}
/* set repl mode flag */
void set_repl_mode_flag(bool flag) {is_repl_mode = flag;}
void set_repl_mode_flag(bool flag) {
is_repl_mode = flag;
}
/* set repl output flag */
void set_allow_repl_output_flag(bool flag) {allow_repl_output = flag;}
void set_allow_repl_output_flag(bool flag) {
allow_repl_output = flag;
}
/* set limit mode flag */
void set_limit_mode_flag(bool flag) {flag_limited_mode = flag;}
void set_limit_mode_flag(bool flag) {
flag_limited_mode = flag;
}
};
inline bool vm::cond(var& val) {

View File

@ -47,12 +47,12 @@ bool repl::check_need_more_input() {
i64 in_brace = 0;
for(const auto& t : nasal_lexer->result()) {
switch(t.type) {
case tok::lcurve: ++in_curve; break;
case tok::rcurve: --in_curve; break;
case tok::lbracket: ++in_bracket; break;
case tok::rbracket: --in_bracket; break;
case tok::lbrace: ++in_brace; break;
case tok::rbrace: --in_brace; break;
case tok::tk_lcurve: ++in_curve; break;
case tok::tk_rcurve: --in_curve; break;
case tok::tk_lbracket: ++in_bracket; break;
case tok::tk_rbracket: --in_bracket; break;
case tok::tk_lbrace: ++in_brace; break;
case tok::tk_rbrace: --in_brace; break;
default: break;
}
}