xxq250
/
Nasal-Interpreter
mirror of https://github.com/ValKmjolnir/Nasal-Interpreter.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity

📝 rename lexical tokens

This commit is contained in:
ValKmjolnir 2024-05-21 23:31:21 +08:00
parent 2c851613ce
commit 77a14699f4
9 changed files with 564 additions and 489 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
module/keyboard.cpp
View File

@ -34,7 +34,8 @@ public:
tcgetattr(0, &init_termios); tcgetattr(0, &init_termios);
new_termios = init_termios; new_termios = init_termios;
new_termios.c_lflag &= ~(ICANON|ECHO|ECHONL|ECHOE); new_termios.c_lflag &= ~(ICANON|ECHO|ECHONL|ECHOE);
// vmin=0 is nonblock input, but in wsl there is a bug that will block input // vmin = 0 is nonblock input,
// but in wsl1 there is a bug that will block input
// so we use fcntl to write the nonblock input // so we use fcntl to write the nonblock input
new_termios.c_cc[VMIN] = 1; new_termios.c_cc[VMIN] = 1;
new_termios.c_cc[VTIME] = 0; new_termios.c_cc[VTIME] = 0;

4
src/nasal_codegen.cpp
View File

@ -239,9 +239,9 @@ void codegen::func_gen(function* node) {
} }
} }
usize newf=code.size(); const auto newf = code.size();
emit(op_newf, 0, node->get_location()); emit(op_newf, 0, node->get_location());
usize lsize=code.size(); const auto lsize = code.size();
emit(op_intl, 0, node->get_location()); emit(op_intl, 0, node->get_location());
// add special keyword 'me' into symbol table // add special keyword 'me' into symbol table

2
src/nasal_gc.cpp
View File

@ -380,7 +380,7 @@ void gc::context_change(nas_co* co) {
} }
void gc::context_reserve() { void gc::context_reserve() {
// pc=0 means this coroutine is finished // pc = 0 means this coroutine is finished
cort->status = running_context->pc? cort->status = running_context->pc?
nas_co::status::suspended: nas_co::status::suspended:
nas_co::status::dead; nas_co::status::dead;

57
src/nasal_lexer.cpp
View File

@ -96,7 +96,7 @@ void lexer::open(const std::string& file) {
} }
tok lexer::get_type(const std::string& str) { tok lexer::get_type(const std::string& str) {
return token_mapper.count(str)? token_mapper.at(str):tok::null; return token_mapper.count(str)? token_mapper.at(str):tok::tk_null;
} }
std::string lexer::utf8_gen() { std::string lexer::utf8_gen() {
@ -152,7 +152,8 @@ token lexer::id_gen() {
tok type = get_type(str); tok type = get_type(str);
return { return {
{begin_line, begin_column, line, column, filename}, {begin_line, begin_column, line, column, filename},
(type!=tok::null)? type:tok::id, str (type!=tok::tk_null)? type:tok::tk_id,
str
}; };
} }
@ -174,7 +175,11 @@ token lexer::num_gen() {
"invalid number `"+str+"`" "invalid number `"+str+"`"
); );
} }
return {{begin_line, begin_column, line, column, filename}, tok::num, str}; return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
str
};
} else if (ptr+1<res.size() && res[ptr]=='0' && res[ptr+1]=='o') { // generate oct number } else if (ptr+1<res.size() && res[ptr]=='0' && res[ptr+1]=='o') { // generate oct number
std::string str = "0o"; std::string str = "0o";
ptr += 2; ptr += 2;
@ -193,7 +198,11 @@ token lexer::num_gen() {
"invalid number `"+str+"`" "invalid number `"+str+"`"
); );
} }
return {{begin_line, begin_column, line, column, filename}, tok::num, str}; return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
str
};
} }
// generate dec number // generate dec number
// dec number -> [0~9][0~9]*(.[0~9]*)(e|E(+|-)0|[1~9][0~9]*) // dec number -> [0~9][0~9]*(.[0~9]*)(e|E(+|-)0|[1~9][0~9]*)
@ -213,7 +222,11 @@ token lexer::num_gen() {
{begin_line, begin_column, line, column, filename}, {begin_line, begin_column, line, column, filename},
"invalid number `"+str+"`" "invalid number `"+str+"`"
); );
return {{begin_line, begin_column, line, column, filename}, tok::num, "0"}; return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
"0"
};
} }
} }
if (ptr<res.size() && (res[ptr]=='e' || res[ptr]=='E')) { if (ptr<res.size() && (res[ptr]=='e' || res[ptr]=='E')) {
@ -231,11 +244,19 @@ token lexer::num_gen() {
{begin_line, begin_column, line, column, filename}, {begin_line, begin_column, line, column, filename},
"invalid number `"+str+"`" "invalid number `"+str+"`"
); );
return {{begin_line, begin_column, line, column, filename}, tok::num, "0"}; return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
"0"
};
} }
} }
column += str.length(); column += str.length();
return {{begin_line, begin_column, line, column, filename}, tok::num, str}; return {
{begin_line, begin_column, line, column, filename},
tok::tk_num,
str
};
} }
token lexer::str_gen() { token lexer::str_gen() {
@ -283,7 +304,11 @@ token lexer::str_gen() {
{begin_line, begin_column, line, column, filename}, {begin_line, begin_column, line, column, filename},
"get EOF when generating string" "get EOF when generating string"
); );
return {{begin_line, begin_column, line, column, filename}, tok::str, str}; return {
{begin_line, begin_column, line, column, filename},
tok::tk_str,
str
};
} }
++column; ++column;
@ -294,7 +319,11 @@ token lexer::str_gen() {
"\'`\' is used for string including one character" "\'`\' is used for string including one character"
); );
} }
return {{begin_line, begin_column, line, column, filename}, tok::str, str}; return {
{begin_line, begin_column, line, column, filename},
tok::tk_str,
str
};
} }
token lexer::single_opr() { token lexer::single_opr() {
@ -303,7 +332,7 @@ token lexer::single_opr() {
std::string str(1, res[ptr]); std::string str(1, res[ptr]);
++column; ++column;
tok type = get_type(str); tok type = get_type(str);
if (type==tok::null) { if (type==tok::tk_null) {
err.err("lexer", err.err("lexer",
{begin_line, begin_column, line, column, filename}, {begin_line, begin_column, line, column, filename},
"invalid operator `"+str+"`" "invalid operator `"+str+"`"
@ -380,10 +409,14 @@ const error& lexer::scan(const std::string& file) {
} }
if (toks.size()) { if (toks.size()) {
// eof token's location is the last token's location // eof token's location is the last token's location
toks.push_back({toks.back().loc, tok::eof, "<eof>"}); toks.push_back({toks.back().loc, tok::tk_eof, "<eof>"});
} else { } else {
// if token sequence is empty, generate a default location // if token sequence is empty, generate a default location
toks.push_back({{line, column, line, column, filename}, tok::eof, "<eof>"}); toks.push_back({
{line, column, line, column, filename},
tok::tk_eof,
"<eof>"
});
} }
res = ""; res = "";
return err; return err;

228
src/nasal_lexer.h
View File

@ -16,66 +16,66 @@
namespace nasal { namespace nasal {
enum class tok:u32 { enum class tok {
null=0, // null token (default token type) tk_null = 0, // null token (default token type)
num, // number literal tk_num, // number literal
str, // string literal tk_str, // string literal
id, // identifier tk_id, // identifier
tktrue, // keyword true tk_true, // keyword true
tkfalse, // keyword false tk_false, // keyword false
use, // keyword use tk_use, // keyword use
rfor, // loop keyword for tk_for, // loop keyword for
forindex, // loop keyword forindex tk_forindex, // loop keyword forindex
foreach, // loop keyword foreach tk_foreach, // loop keyword foreach
rwhile, // loop keyword while tk_while, // loop keyword while
var, // keyword for definition tk_var, // keyword for definition
func, // keyword for definition of function tk_func, // keyword for definition of function
brk, // loop keyword break tk_brk, // loop keyword break
cont, // loop keyword continue tk_cont, // loop keyword continue
ret, // function keyword return tk_ret, // function keyword return
rif, // condition expression keyword if tk_if, // condition expression keyword if
elsif, // condition expression keyword elsif tk_elsif, // condition expression keyword elsif
relse, // condition expression keyword else tk_else, // condition expression keyword else
nil, // nil literal tk_nil, // nil literal
lcurve, // ( tk_lcurve, // (
rcurve, // ) tk_rcurve, // )
lbracket, // [ tk_lbracket, // [
rbracket, // ] tk_rbracket, // ]
lbrace, // { tk_lbrace, // {
rbrace, // } tk_rbrace, // }
semi, // ; tk_semi, // ;
opand, // operator and tk_and, // operator and
opor, // operator or tk_or, // operator or
comma, // , tk_comma, // ,
dot, // . tk_dot, // .
ellipsis, // ... tk_ellipsis, // ...
quesmark, // ? tk_quesmark, // ?
colon, // : tk_colon, // :
add, // operator + tk_add, // operator +
sub, // operator - tk_sub, // operator -
mult, // operator * tk_mult, // operator *
div, // operator / tk_div, // operator /
floater, // operator ~ and binary operator ~ tk_floater, // operator ~ and binary operator ~
btand, // bitwise operator & tk_btand, // bitwise operator &
btor, // bitwise operator | tk_btor, // bitwise operator |
btxor, // bitwise operator ^ tk_btxor, // bitwise operator ^
opnot, // operator ! tk_not, // operator !
eq, // operator = tk_eq, // operator =
addeq, // operator += tk_addeq, // operator +=
subeq, // operator -= tk_subeq, // operator -=
multeq, // operator *= tk_multeq, // operator *=
diveq, // operator /= tk_diveq, // operator /=
lnkeq, // operator ~= tk_lnkeq, // operator ~=
btandeq, // operator &= tk_btandeq, // operator &=
btoreq, // operator |= tk_btoreq, // operator |=
btxoreq, // operator ^= tk_btxoreq, // operator ^=
cmpeq, // operator == tk_cmpeq, // operator ==
neq, // operator != tk_neq, // operator !=
less, // operator < tk_less, // operator <
leq, // operator <= tk_leq, // operator <=
grt, // operator > tk_grt, // operator >
geq, // operator >= tk_geq, // operator >=
eof // <eof> end of token list tk_eof // <eof> end of token list
}; };
struct token { struct token {
@ -99,60 +99,60 @@ private:
std::vector<token> toks; std::vector<token> toks;
const std::unordered_map<std::string, tok> token_mapper = { const std::unordered_map<std::string, tok> token_mapper = {
{"use" ,tok::use }, {"use" , tok::tk_use },
{"true" ,tok::tktrue }, {"true" , tok::tk_true },
{"false" ,tok::tkfalse }, {"false" , tok::tk_false },
{"for" ,tok::rfor }, {"for" , tok::tk_for },
{"forindex",tok::forindex}, {"forindex", tok::tk_forindex},
{"foreach" ,tok::foreach }, {"foreach" , tok::tk_foreach },
{"while" ,tok::rwhile }, {"while" , tok::tk_while },
{"var" ,tok::var }, {"var" , tok::tk_var },
{"func" ,tok::func }, {"func" , tok::tk_func },
{"break" ,tok::brk }, {"break" , tok::tk_brk },
{"continue",tok::cont }, {"continue", tok::tk_cont },
{"return" ,tok::ret }, {"return" , tok::tk_ret },
{"if" ,tok::rif }, {"if" , tok::tk_if },
{"elsif" ,tok::elsif }, {"elsif" , tok::tk_elsif },
{"else" ,tok::relse }, {"else" , tok::tk_else },
{"nil" ,tok::nil }, {"nil" , tok::tk_nil },
{"(" ,tok::lcurve }, {"(" , tok::tk_lcurve },
{")" ,tok::rcurve }, {")" , tok::tk_rcurve },
{"[" ,tok::lbracket}, {"[" , tok::tk_lbracket},
{"]" ,tok::rbracket}, {"]" , tok::tk_rbracket},
{"{" ,tok::lbrace }, {"{" , tok::tk_lbrace },
{"}" ,tok::rbrace }, {"}" , tok::tk_rbrace },
{";" ,tok::semi }, {";" , tok::tk_semi },
{"and" ,tok::opand }, {"and" , tok::tk_and },
{"or" ,tok::opor }, {"or" , tok::tk_or },
{"," ,tok::comma }, {"," , tok::tk_comma },
{"." ,tok::dot }, {"." , tok::tk_dot },
{"..." ,tok::ellipsis}, {"..." , tok::tk_ellipsis},
{"?" ,tok::quesmark}, {"?" , tok::tk_quesmark},
{":" ,tok::colon }, {":" , tok::tk_colon },
{"+" ,tok::add }, {"+" , tok::tk_add },
{"-" ,tok::sub }, {"-" , tok::tk_sub },
{"*" ,tok::mult }, {"*" , tok::tk_mult },
{"/" ,tok::div }, {"/" , tok::tk_div },
{"~" ,tok::floater }, {"~" , tok::tk_floater },
{"&" ,tok::btand }, {"&" , tok::tk_btand },
{"|" ,tok::btor }, {"|" , tok::tk_btor },
{"^" ,tok::btxor }, {"^" , tok::tk_btxor },
{"!" ,tok::opnot }, {"!" , tok::tk_not },
{"=" ,tok::eq }, {"=" , tok::tk_eq },
{"+=" ,tok::addeq }, {"+=" , tok::tk_addeq },
{"-=" ,tok::subeq }, {"-=" , tok::tk_subeq },
{"*=" ,tok::multeq }, {"*=" , tok::tk_multeq },
{"/=" ,tok::diveq }, {"/=" , tok::tk_diveq },
{"~=" ,tok::lnkeq }, {"~=" , tok::tk_lnkeq },
{"&=" ,tok::btandeq }, {"&=" , tok::tk_btandeq },
{"|=" ,tok::btoreq }, {"|=" , tok::tk_btoreq },
{"^=" ,tok::btxoreq }, {"^=" , tok::tk_btxoreq },
{"==" ,tok::cmpeq }, {"==" , tok::tk_cmpeq },
{"!=" ,tok::neq }, {"!=" , tok::tk_neq },
{"<" ,tok::less }, {"<" , tok::tk_less },
{"<=" ,tok::leq }, {"<=" , tok::tk_leq },
{">" ,tok::grt }, {">" , tok::tk_grt },
{">=" ,tok::geq } {">=" , tok::tk_geq }
}; };
tok get_type(const std::string&); tok get_type(const std::string&);

611
src/nasal_parse.cpp
View File

File diff suppressed because it is too large Load Diff

115
src/nasal_parse.h
View File

@ -16,66 +16,68 @@ class parse {
private: private:
u64 ptr; u64 ptr;
u64 in_func; // count function block u64 in_func_depth; // count function block
u64 in_loop; // count loop block u64 in_loop_depth; // count loop block
const token* toks; const token* toks;
code_block* root; code_block* root;
error err; error err;
private: private:
const std::unordered_map<tok, std::string> tokname { const std::unordered_map<tok, std::string> tokname = {
{tok::use ,"use" }, {tok::tk_true , "true" },
{tok::rfor ,"for" }, {tok::tk_false , "false" },
{tok::forindex,"forindex"}, {tok::tk_use , "use" },
{tok::foreach ,"foreach" }, {tok::tk_for , "for" },
{tok::rwhile ,"while" }, {tok::tk_forindex, "forindex"},
{tok::var ,"var" }, {tok::tk_foreach , "foreach" },
{tok::func ,"func" }, {tok::tk_while , "while" },
{tok::brk ,"break" }, {tok::tk_var , "var" },
{tok::cont ,"continue"}, {tok::tk_func , "func" },
{tok::ret ,"return" }, {tok::tk_brk , "break" },
{tok::rif ,"if" }, {tok::tk_cont , "continue"},
{tok::elsif ,"elsif" }, {tok::tk_ret , "return" },
{tok::relse ,"else" }, {tok::tk_if , "if" },
{tok::nil ,"nil" }, {tok::tk_elsif , "elsif" },
{tok::lcurve ,"(" }, {tok::tk_else , "else" },
{tok::rcurve ,")" }, {tok::tk_nil , "nil" },
{tok::lbracket,"[" }, {tok::tk_lcurve , "(" },
{tok::rbracket,"]" }, {tok::tk_rcurve , ")" },
{tok::lbrace ,"{" }, {tok::tk_lbracket, "[" },
{tok::rbrace ,"}" }, {tok::tk_rbracket, "]" },
{tok::semi ,";" }, {tok::tk_lbrace , "{" },
{tok::opand ,"and" }, {tok::tk_rbrace , "}" },
{tok::opor ,"or" }, {tok::tk_semi , ";" },
{tok::comma ,"," }, {tok::tk_and , "and" },
{tok::dot ,"." }, {tok::tk_or , "or" },
{tok::ellipsis,"..." }, {tok::tk_comma , "," },
{tok::quesmark,"?" }, {tok::tk_dot , "." },
{tok::colon ,":" }, {tok::tk_ellipsis, "..." },
{tok::add ,"+" }, {tok::tk_quesmark, "?" },
{tok::sub ,"-" }, {tok::tk_colon , ":" },
{tok::mult ,"*" }, {tok::tk_add , "+" },
{tok::div ,"/" }, {tok::tk_sub , "-" },
{tok::floater ,"~" }, {tok::tk_mult , "*" },
{tok::btand ,"&" }, {tok::tk_div , "/" },
{tok::btor ,"|" }, {tok::tk_floater , "~" },
{tok::btxor ,"^" }, {tok::tk_btand , "&" },
{tok::opnot ,"!" }, {tok::tk_btor , "|" },
{tok::eq ,"=" }, {tok::tk_btxor , "^" },
{tok::addeq ,"+=" }, {tok::tk_not , "!" },
{tok::subeq ,"-=" }, {tok::tk_eq , "=" },
{tok::multeq ,"*=" }, {tok::tk_addeq , "+=" },
{tok::diveq ,"/=" }, {tok::tk_subeq , "-=" },
{tok::lnkeq ,"~=" }, {tok::tk_multeq , "*=" },
{tok::btandeq ,"&=" }, {tok::tk_diveq , "/=" },
{tok::btoreq ,"|=" }, {tok::tk_lnkeq , "~=" },
{tok::btxoreq ,"^=" }, {tok::tk_btandeq , "&=" },
{tok::cmpeq ,"==" }, {tok::tk_btoreq , "|=" },
{tok::neq ,"!=" }, {tok::tk_btxoreq , "^=" },
{tok::less ,"<" }, {tok::tk_cmpeq , "==" },
{tok::leq ,"<=" }, {tok::tk_neq , "!=" },
{tok::grt ,">" }, {tok::tk_less , "<" },
{tok::geq ,">=" } {tok::tk_leq , "<=" },
{tok::tk_grt , ">" },
{tok::tk_geq , ">=" }
}; };
private: private:
@ -151,7 +153,8 @@ public:
} }
public: public:
parse(): ptr(0), in_func(0), in_loop(0), toks(nullptr), root(nullptr) {} parse(): ptr(0), in_func_depth(0), in_loop_depth(0),
toks(nullptr), root(nullptr) {}
~parse() {delete root;} ~parse() {delete root;}
const error& compile(const lexer&); const error& compile(const lexer&);
static void easter_egg(); static void easter_egg();

21
src/nasal_vm.h
View File

@ -50,6 +50,7 @@ protected:
/* limited mode, will not load unsafe system api if switched on */ /* limited mode, will not load unsafe system api if switched on */
bool flag_limited_mode = false; bool flag_limited_mode = false;
protected:
/* vm initializing function */ /* vm initializing function */
void vm_init_enrty(const std::vector<std::string>&, void vm_init_enrty(const std::vector<std::string>&,
const std::vector<f64>&, const std::vector<f64>&,
@ -60,6 +61,7 @@ protected:
const std::vector<std::string>&); const std::vector<std::string>&);
void context_and_global_init(); void context_and_global_init();
protected:
/* debug functions */ /* debug functions */
bool verbose = false; bool verbose = false;
void value_info(var&); void value_info(var&);
@ -190,13 +192,24 @@ public:
const std::vector<std::string>&); // get command line arguments const std::vector<std::string>&); // get command line arguments
/* set detail report info flag */ /* set detail report info flag */
void set_detail_report_info(bool flag) {verbose = flag;} void set_detail_report_info(bool flag) {
verbose = flag;
}
/* set repl mode flag */ /* set repl mode flag */
void set_repl_mode_flag(bool flag) {is_repl_mode = flag;} void set_repl_mode_flag(bool flag) {
is_repl_mode = flag;
}
/* set repl output flag */ /* set repl output flag */
void set_allow_repl_output_flag(bool flag) {allow_repl_output = flag;} void set_allow_repl_output_flag(bool flag) {
allow_repl_output = flag;
}
/* set limit mode flag */ /* set limit mode flag */
void set_limit_mode_flag(bool flag) {flag_limited_mode = flag;} void set_limit_mode_flag(bool flag) {
flag_limited_mode = flag;
}
}; };
inline bool vm::cond(var& val) { inline bool vm::cond(var& val) {

12
src/repl.cpp
View File

@ -47,12 +47,12 @@ bool repl::check_need_more_input() {
i64 in_brace = 0; i64 in_brace = 0;
for(const auto& t : nasal_lexer->result()) { for(const auto& t : nasal_lexer->result()) {
switch(t.type) { switch(t.type) {
case tok::lcurve: ++in_curve; break; case tok::tk_lcurve: ++in_curve; break;
case tok::rcurve: --in_curve; break; case tok::tk_rcurve: --in_curve; break;
case tok::lbracket: ++in_bracket; break; case tok::tk_lbracket: ++in_bracket; break;
case tok::rbracket: --in_bracket; break; case tok::tk_rbracket: --in_bracket; break;
case tok::lbrace: ++in_brace; break; case tok::tk_lbrace: ++in_brace; break;
case tok::rbrace: --in_brace; break; case tok::tk_rbrace: --in_brace; break;
default: break; default: break;
} }
} }