✨ add nasal_new_lexer

ast/ast_visitor.cpp

@@ -1,21 +1,29 @@
 #include "ast_visitor.h"
 
-void ast_visitor::visit_expr(expr* node) {
+bool ast_visitor::visit_expr(expr* node) {
     node->accept(this);
 }
 
-void ast_visitor::visit_null_expr(null_expr* node) {
+bool ast_visitor::visit_null_expr(null_expr* node) {
-    node->accept(this);
+    return true;
 }
 
-void ast_visitor::visit_nil_expr(nil_expr* node) {
+bool ast_visitor::visit_nil_expr(nil_expr* node) {
-    node->accept(this);
+    return true;
 }
 
-void ast_visitor::visit_number_literal(number_literal* node) {
+bool ast_visitor::visit_number_literal(number_literal* node) {
-    node->accept(this);
+    return true;
 }
 
-void ast_visitor::visit_string_literal(string_literal* node) {
+bool ast_visitor::visit_string_literal(string_literal* node) {
-    node->accept(this);
+    return true;
+}
+
+bool ast_visitor::visit_identifier(identifier* node) {
+    return true;
+}
+
+bool ast_visitor::visit_bool_literal(bool_literal* node) {
+    return true;
 }

ast/ast_visitor.h

@@ -4,9 +4,11 @@
 
 class ast_visitor {
 public:
-    virtual void visit_expr(expr*);
+    virtual bool visit_expr(expr*);
-    virtual void visit_null_expr(null_expr*);
+    virtual bool visit_null_expr(null_expr*);
-    virtual void visit_nil_expr(nil_expr*);
+    virtual bool visit_nil_expr(nil_expr*);
-    virtual void visit_number_literal(number_literal*);
+    virtual bool visit_number_literal(number_literal*);
-    virtual void visit_string_literal(string_literal*);
+    virtual bool visit_string_literal(string_literal*);
+    virtual bool visit_identifier(identifier*);
+    virtual bool visit_bool_literal(bool_literal*);
 };

ast/nasal_new_ast.cpp

@@ -1,27 +1,30 @@
 #include "nasal_new_ast.h"
 #include "ast_visitor.h"
 
-bool expr::accept(ast_visitor* visitor) {
+void expr::accept(ast_visitor* visitor) {
     visitor->visit_expr(this);
-    return true;
 }
 
-bool null_expr::accept(ast_visitor* visitor) {
+void null_expr::accept(ast_visitor* visitor) {
     visitor->visit_null_expr(this);
-    return true;
 }
 
-bool nil_expr::accept(ast_visitor* visitor) {
+void nil_expr::accept(ast_visitor* visitor) {
     visitor->visit_nil_expr(this);
-    return true;
 }
 
-bool number_literal::accept(ast_visitor* visitor) {
+void number_literal::accept(ast_visitor* visitor) {
     visitor->visit_number_literal(this);
-    return true;
 }
 
-bool string_literal::accept(ast_visitor* visitor) {
+void string_literal::accept(ast_visitor* visitor) {
     visitor->visit_string_literal(this);
-    return true;
+}
+
+void identifier::accept(ast_visitor* visitor) {
+    visitor->visit_identifier(this);
+}
+
+void bool_literal::accept(ast_visitor* visitor) {
+    visitor->visit_bool_literal(this);
 }

ast/nasal_new_ast.h

@@ -83,7 +83,7 @@ public:
     expr(const span& location, expr_type node_type):
         nd_loc(location), nd_type(node_type) {}
     ~expr() = default;
-    virtual bool accept(ast_visitor*) = 0;
+    virtual void accept(ast_visitor*) = 0;
 };
 
 class null_expr:public expr {
@@ -91,7 +91,7 @@ public:
     null_expr(const span& location):
         expr(location, expr_type::ast_null) {}
     ~null_expr() = default;
-    virtual bool accept(ast_visitor*);
+    virtual void accept(ast_visitor*) override;
 };
 
 class nil_expr:public expr {
@@ -99,7 +99,7 @@ public:
     nil_expr(const span& location):
         expr(location, expr_type::ast_nil) {}
     ~nil_expr() = default;
-    virtual bool accept(ast_visitor*);
+    virtual void accept(ast_visitor*) override;
 };
 
 class number_literal:public expr {
@@ -110,7 +110,7 @@ public:
     number_literal(const span& location, const f64 num):
         expr(location, expr_type::ast_num), number(num) {}
     ~number_literal() = default;
-    virtual bool accept(ast_visitor*);
+    virtual void accept(ast_visitor*) override;
 };
 
 class string_literal:public expr {
@@ -121,57 +121,147 @@ public:
     string_literal(const span& location, const string& str):
         expr(location, expr_type::ast_str), content(str) {}
     ~string_literal() = default;
-    virtual bool accept(ast_visitor*);
+    virtual void accept(ast_visitor*) override;
 };
 
-class identifier:public expr {};
+class identifier:public expr {
+private:
+    string name;
 
-class bool_literal:public expr {};
+public:
+    identifier(const span& location, const string& str):
+        expr(location, expr_type::ast_id), name(str) {}
+    ~identifier() = default;
+    virtual void accept(ast_visitor*) override;
+};
 
-class vector_expr:public expr {};
+class bool_literal:public expr {
+private:
+    bool flag;
 
-class hash_expr:public expr {};
+public:
+    bool_literal(const span& location, const bool bool_flag):
+        expr(location, expr_type::ast_bool), flag(bool_flag) {}
+    ~bool_literal() = default;
+    virtual void accept(ast_visitor*) override;
+};
 
-class hash_pair:public expr {};
+class vector_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class function:public expr {};
+class hash_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class parameter:public expr {};
+class hash_pair:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class ternary_operator:public expr {};
+class function:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class binary_operator:public expr {};
+class parameter:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class unary_operator:public expr {};
+class ternary_operator:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class call_expr:public expr {};
+class binary_operator:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class call_hash:public expr {};
+class unary_operator:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class call_vector:public expr {};
+class call_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class call_function:public expr {};
+class call_hash:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class slice_vector:public expr {};
+class call_vector:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class definition:public expr {};
+class call_function:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class multi_define:public expr {};
+class slice_vector:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class while_expr:public expr {};
+class definition:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class for_expr:public expr {};
+class multi_define:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class foreach_expr:public expr {};
+class while_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class forindex_expr:public expr {};
+class for_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class condition_expr:public expr {};
+class foreach_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class if_expr:public expr {};
+class forindex_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class continue_expr:public expr {};
+class condition_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class break_expr:public expr {};
+class if_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
 
-class return_expr:public expr {};
+class continue_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
+
+class break_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};
+
+class return_expr:public expr {
+public:
+    virtual void accept(ast_visitor*) override;
+};

ast/nasal_new_lexer.cpp

@@ -0,0 +1,333 @@
+#ifdef _MSC_VER
+#pragma warning (disable:4244)
+#pragma warning (disable:4267)
+#pragma warning (disable:4102)
+#endif
+
+#include "nasal_new_lexer.h"
+
+bool lexer::skip(char c) {
+    return c==' '||c=='\n'||c=='\t'||c=='\r'||c==0;
+}
+
+bool lexer::is_id(char c) {
+    return (c=='_')||('a'<=c && c<='z')||('A'<=c&&c<='Z')||(c<0);   
+}
+
+bool lexer::is_hex(char c) {
+    return ('0'<=c&&c<='9')||('a'<=c&&c<='f')||('A'<=c && c<='F');
+}
+
+bool lexer::is_oct(char c) {
+    return '0'<=c&&c<='7';
+}
+
+bool lexer::is_dec(char c) {
+    return '0'<=c&&c<='9';
+}
+
+bool lexer::is_str(char c) {
+    return c=='\''||c=='\"'||c=='`';
+}
+
+bool lexer::is_single_opr(char c) {
+    return (
+        c=='('||c==')'||c=='['||c==']'||
+        c=='{'||c=='}'||c==','||c==';'||
+        c==':'||c=='?'||c=='`'||c=='@'||
+        c=='%'||c=='$'||c=='\\'
+    );
+}
+
+bool lexer::is_calc_opr(char c) {
+    return (
+        c=='='||c=='+'||c=='-'||c=='*'||
+        c=='!'||c=='/'||c=='<'||c=='>'||
+        c=='~'||c=='|'||c=='&'||c=='^'
+    );
+}
+
+void lexer::skip_note() {
+    // avoid note, after this process ptr will point to a '\n', so next loop line counter+1
+    while(++ptr<res.size() && res[ptr]!='\n') {}
+}
+
+void lexer::err_char() {
+    ++column;
+    char c=res[ptr++];
+    err.err("lexer", {line, column-1, line, column, filename}, "invalid character 0x"+chrhex(c));
+    err.fatal("lexer", "fatal error occurred, stop");
+}
+
+void lexer::open(const string& file) {
+    // check file exsits and it is a regular file
+    struct stat buffer;
+    if (stat(file.c_str(), &buffer)==0 && !S_ISREG(buffer.st_mode)) {
+        err.err("lexer", "<"+file+"> is not a regular file");
+        err.chkerr();
+    }
+
+    // load
+    filename=file;
+    std::ifstream in(file, std::ios::binary);
+    if (in.fail()) {
+        err.err("lexer", "failed to open <"+file+">");
+    } else {
+        err.load(file);
+    }
+    std::stringstream ss;
+    ss<<in.rdbuf();
+    res=ss.str();
+}
+
+tok lexer::get_type(const string& str) {
+    return typetbl.count(str)?typetbl.at(str):tok::null;
+}
+
+string lexer::utf8_gen() {
+    string str="";
+    while(ptr<res.size() && res[ptr]<0) {
+        string tmp="";
+        u32 nbytes=utf8_hdchk(res[ptr]);
+        if (!nbytes) {
+            ++ptr;
+            ++column;
+            continue;
+        }
+
+        tmp+=res[ptr++];
+        for(u32 i=0;i<nbytes;++i,++ptr) {
+            if (ptr<res.size() && (res[ptr]&0xc0)==0x80) {
+                tmp+=res[ptr];
+            }
+        }
+
+        // utf8 character's total length is 1+nbytes
+        if (tmp.length()!=1+nbytes) {
+            ++column;
+            string utf_info="0x"+chrhex(tmp[0]);
+            for(u32 i=1;i<tmp.size();++i) {
+                utf_info+=" 0x"+chrhex(tmp[i]);
+            }
+            err.err("lexer", {line, column-1, line, column, filename}, "invalid utf-8 <"+utf_info+">");
+            err.fatal("lexer", "fatal error occurred, stop");
+        }
+        str+=tmp;
+        column+=2; // may have some problems because not all the unicode takes 2 space
+    }
+    return str;
+}
+
+token lexer::id_gen() {
+    u32 begin_line=line;
+    u32 begin_column=column;
+    string str="";
+    while(ptr<res.size() && (is_id(res[ptr])||is_dec(res[ptr]))) {
+        if (res[ptr]<0) { // utf-8
+            str+=utf8_gen();
+        } else { // ascii
+            str+=res[ptr++];
+            ++column;
+        }
+    }
+    tok type=get_type(str);
+    return {{begin_line, begin_column, line, column, filename}, (type!=tok::null)?type:tok::id, str};
+}
+
+token lexer::num_gen() {
+    u32 begin_line=line;
+    u32 begin_column=column;
+    // generate hex number
+    if (ptr+1<res.size() && res[ptr]=='0' && res[ptr+1]=='x') {
+        string str="0x";
+        ptr+=2;
+        while(ptr<res.size() && is_hex(res[ptr])) {
+            str+=res[ptr++];
+        }
+        column+=str.length();
+        if (str.length()<3) { // "0x"
+            err.err("lexer", {begin_line, begin_column, line, column, filename}, "invalid number `"+str+"`");
+        }
+        return {{begin_line, begin_column, line, column, filename}, tok::num, str};
+    } else if (ptr+1<res.size() && res[ptr]=='0' && res[ptr+1]=='o') { // generate oct number
+        string str="0o";
+        ptr+=2;
+        while(ptr<res.size() && is_oct(res[ptr])) {
+            str+=res[ptr++];
+        }
+        bool erfmt=false;
+        while(ptr<res.size() && (is_dec(res[ptr]) || is_hex(res[ptr]))) {
+            erfmt=true;
+            str+=res[ptr++];
+        }
+        column+=str.length();
+        if (str.length()==2 || erfmt) {
+            err.err("lexer", {begin_line, begin_column, line, column, filename}, "invalid number `"+str+"`");
+        }
+        return {{begin_line, begin_column, line, column, filename}, tok::num, str};
+    }
+    // generate dec number
+    // dec number -> [0~9][0~9]*(.[0~9]*)(e|E(+|-)0|[1~9][0~9]*)
+    string str="";
+    while(ptr<res.size() && is_dec(res[ptr])) {
+        str+=res[ptr++];
+    }
+    if (ptr<res.size() && res[ptr]=='.') {
+        str+=res[ptr++];
+        while(ptr<res.size() && is_dec(res[ptr])) {
+            str+=res[ptr++];
+        }
+        // "xxxx." is not a correct number
+        if (str.back()=='.') {
+            column+=str.length();
+            err.err("lexer",{begin_line, begin_column, line, column, filename}, "invalid number `"+str+"`");
+            return {{begin_line, begin_column, line, column, filename}, tok::num, "0"};
+        }
+    }
+    if (ptr<res.size() && (res[ptr]=='e' || res[ptr]=='E')) {
+        str+=res[ptr++];
+        if (ptr<res.size() && (res[ptr]=='-' || res[ptr]=='+')) {
+            str+=res[ptr++];
+        }
+        while(ptr<res.size() && is_dec(res[ptr])) {
+            str+=res[ptr++];
+        }
+        // "xxxe(-|+)" is not a correct number
+        if (str.back()=='e' || str.back()=='E' || str.back()=='-' || str.back()=='+') {
+            column+=str.length();
+            err.err("lexer",{begin_line, begin_column, line, column, filename}, "invalid number `"+str+"`");
+            return {{begin_line, begin_column, line, column, filename}, tok::num, "0"};
+        }
+    }
+    column+=str.length();
+    return {{begin_line, begin_column, line, column, filename}, tok::num, str};
+}
+
+token lexer::str_gen() {
+    u32 begin_line=line;
+    u32 begin_column=column;
+    string str="";
+    const char begin=res[ptr];
+    ++column;
+    while(++ptr<res.size() && res[ptr]!=begin) {
+        ++column;
+        if (res[ptr]=='\n') {
+            column=0;
+            ++line;
+        }
+        if (res[ptr]=='\\' && ptr+1<res.size()) {
+            ++column;
+            ++ptr;
+            switch(res[ptr]) {
+                case '0': str+='\0';    break;
+                case 'a': str+='\a';    break;
+                case 'b': str+='\b';    break;
+                case 'e': str+='\033';  break;
+                case 't': str+='\t';    break;
+                case 'n': str+='\n';    break;
+                case 'v': str+='\v';    break;
+                case 'f': str+='\f';    break;
+                case 'r': str+='\r';    break;
+                case '?': str+='\?';    break;
+                case '\\':str+='\\';    break;
+                case '\'':str+='\'';    break;
+                case '\"':str+='\"';    break;
+                default:  str+=res[ptr];break;
+            }
+            if (res[ptr]=='\n') {
+                column=0;
+                ++line;
+            }
+            continue;
+        }
+        str+=res[ptr];
+    }
+    // check if this string ends with a " or '
+    if (ptr++>=res.size()) {
+        err.err("lexer", {begin_line, begin_column, line, column, filename}, "get EOF when generating string");
+        return {{begin_line, begin_column, line, column, filename}, tok::str, str};
+    }
+    ++column;
+    if (begin=='`' && str.length()!=1) {
+        err.err("lexer", {begin_line, begin_column, line, column, filename}, "\'`\' is used for string including one character");
+    }
+    return {{begin_line, begin_column, line, column, filename}, tok::str, str};
+}
+
+token lexer::single_opr() {
+    u32 begin_line=line;
+    u32 begin_column=column;
+    string str(1,res[ptr]);
+    ++column;
+    tok type=get_type(str);
+    if (type==tok::null) {
+        err.err("lexer", {begin_line, begin_column, line, column, filename}, "invalid operator `"+str+"`");
+    }
+    ++ptr;
+    return {{begin_line, begin_column, line, column, filename}, type, str};
+}
+
+token lexer::dots() {
+    u32 begin_line=line;
+    u32 begin_column=column;
+    string str=".";
+    if (ptr+2<res.size() && res[ptr+1]=='.' && res[ptr+2]=='.') {
+        str+="..";
+    }
+    ptr+=str.length();
+    column+=str.length();
+    return {{begin_line, begin_column, line, column, filename}, get_type(str), str};
+}
+
+token lexer::calc_opr() {
+    u32 begin_line=line;
+    u32 begin_column=column;
+    // get calculation operator
+    string str(1,res[ptr++]);
+    if (ptr<res.size() && res[ptr]=='=') {
+        str+=res[ptr++];
+    }
+    column+=str.length();
+    return {{begin_line, begin_column, line, column, filename}, get_type(str), str};
+}
+
+const error& lexer::scan(const string& file) {
+    line=1;
+    column=0;
+    ptr=0;
+    open(file);
+
+    while(ptr<res.size()) {
+        while(ptr<res.size() && skip(res[ptr])) {
+            // these characters will be ignored, and '\n' will cause ++line
+            ++column;
+            if (res[ptr++]=='\n') {
+                ++line;
+                column=0;
+            }
+        }
+        if (ptr>=res.size()) {
+            break;
+        }
+        if (is_id(res[ptr])) {
+            toks.push_back(id_gen());
+        } else if (is_dec(res[ptr])) {
+            toks.push_back(num_gen());
+        } else if (is_str(res[ptr])) {
+            toks.push_back(str_gen());
+        } else if (is_single_opr(res[ptr])) {
+            toks.push_back(single_opr());
+        } else if (res[ptr]=='.') {
+            toks.push_back(dots());
+        } else if (is_calc_opr(res[ptr])) {
+            toks.push_back(calc_opr());
+        } else if (res[ptr]=='#') {
+            skip_note();
+        } else {
+            err_char();
+        }
+    }
+    toks.push_back({{line, column, line, column, filename}, tok::eof, "<eof>"});
+    res="";
+    return err;
+}

ast/nasal_new_lexer.h

@@ -0,0 +1,181 @@
+#pragma once
+
+#ifdef _MSC_VER
+#pragma warning (disable:4244)
+#pragma warning (disable:4267)
+#pragma warning (disable:4102)
+#endif
+
+#include <cstring>
+#include <sstream>
+#include <vector>
+#include <unordered_map>
+#include <sys/stat.h>
+
+#include "nasal.h"
+#include "nasal_err.h"
+
+#ifdef _MSC_VER
+#define S_ISREG(m) (((m)&0xF000)==0x8000)
+#endif
+
+enum class tok:u32 {
+    null=0,   // null token (default token type)
+    num,      // number literal
+    str,      // string literal
+    id,       // identifier
+    tktrue,   // keyword true
+    tkfalse,  // keyword false
+    rfor,     // loop keyword for
+    forindex, // loop keyword forindex
+    foreach,  // loop keyword foreach
+    rwhile,   // loop keyword while
+    var,      // keyword for definition
+    func,     // keyword for definition of function
+    brk,      // loop keyword break
+    cont,     // loop keyword continue
+    ret,      // function keyword return
+    rif,      // condition expression keyword if
+    elsif,    // condition expression keyword elsif
+    relse,    // condition expression keyword else
+    tknil,    // nil literal
+    lcurve,   // (
+    rcurve,   // )
+    lbracket, // [
+    rbracket, // ]
+    lbrace,   // {
+    rbrace,   // }
+    semi,     // ;
+    opand,    // operator and
+    opor,     // operator or
+    comma,    // ,
+    dot,      // .
+    ellipsis, // ...
+    quesmark, // ?
+    colon,    // :
+    add,      // operator +
+    sub,      // operator -
+    mult,     // operator *
+    div,      // operator /
+    floater,  // operator ~ and binary operator ~
+    btand,    // bitwise operator &
+    btor,     // bitwise operator |
+    btxor,    // bitwise operator ^
+    opnot,    // operator !
+    eq,       // operator =
+    addeq,    // operator +=
+    subeq,    // operator -=
+    multeq,   // operator *=
+    diveq,    // operator /=
+    lnkeq,    // operator ~=
+    btandeq,  // operator &=
+    btoreq,   // operator |=
+    btxoreq,  // operator ^=
+    cmpeq,    // operator ==
+    neq,      // operator !=
+    less,     // operator <
+    leq,      // operator <=
+    grt,      // operator >
+    geq,      // operator >=
+    eof       // <eof> end of token list
+};
+
+struct token {
+    span loc;    // location
+    tok type;    // token type
+    string str;  // content
+    token() = default;
+    token(const token&) = default;
+};
+
+class lexer {
+private:
+    u32    line;
+    u32    column;
+    usize  ptr;
+    string filename;
+    string res;
+    error& err;
+    std::vector<token> toks;
+    const std::unordered_map<string,tok> typetbl {
+        {"true"    ,tok::tktrue  },
+        {"false"   ,tok::tkfalse },
+        {"for"     ,tok::rfor    },
+        {"forindex",tok::forindex},
+        {"foreach" ,tok::foreach },
+        {"while"   ,tok::rwhile  },
+        {"var"     ,tok::var     },
+        {"func"    ,tok::func    },
+        {"break"   ,tok::brk     },
+        {"continue",tok::cont    },
+        {"return"  ,tok::ret     },
+        {"if"      ,tok::rif     },
+        {"elsif"   ,tok::elsif   },
+        {"else"    ,tok::relse   },
+        {"nil"     ,tok::tknil   },
+        {"("       ,tok::lcurve  },
+        {")"       ,tok::rcurve  },
+        {"["       ,tok::lbracket},
+        {"]"       ,tok::rbracket},
+        {"{"       ,tok::lbrace  },
+        {"}"       ,tok::rbrace  },
+        {";"       ,tok::semi    },
+        {"and"     ,tok::opand   },
+        {"or"      ,tok::opor    },
+        {","       ,tok::comma   },
+        {"."       ,tok::dot     },
+        {"..."     ,tok::ellipsis},
+        {"?"       ,tok::quesmark},
+        {":"       ,tok::colon   },
+        {"+"       ,tok::add     },
+        {"-"       ,tok::sub     },
+        {"*"       ,tok::mult    },
+        {"/"       ,tok::div     },
+        {"~"       ,tok::floater },
+        {"&"       ,tok::btand   },
+        {"|"       ,tok::btor    },
+        {"^"       ,tok::btxor   },
+        {"!"       ,tok::opnot   },
+        {"="       ,tok::eq      },
+        {"+="      ,tok::addeq   },
+        {"-="      ,tok::subeq   },
+        {"*="      ,tok::multeq  },
+        {"/="      ,tok::diveq   },
+        {"~="      ,tok::lnkeq   },
+        {"&="      ,tok::btandeq },
+        {"|="      ,tok::btoreq  },
+        {"^="      ,tok::btxoreq },
+        {"=="      ,tok::cmpeq   },
+        {"!="      ,tok::neq     },
+        {"<"       ,tok::less    },
+        {"<="      ,tok::leq     },
+        {">"       ,tok::grt     },
+        {">="      ,tok::geq     }
+    };
+
+    tok get_type(const string&);
+    bool skip(char);
+    bool is_id(char);
+    bool is_hex(char);
+    bool is_oct(char);
+    bool is_dec(char);
+    bool is_str(char);
+    bool is_single_opr(char);
+    bool is_calc_opr(char);
+
+    void skip_note();
+    void err_char();
+
+    void open(const string&);
+    string utf8_gen();
+    token id_gen();
+    token num_gen();
+    token str_gen();
+    token single_opr();
+    token dots();
+    token calc_opr();
+public:
+    lexer(error& e): line(1), column(0), ptr(0), filename(""), res(""), err(e) {}
+    const error& scan(const string&);
+    const std::vector<token>& result() const {return toks;}
+};

ast/new_main.cpp

@@ -0,0 +1,176 @@
+#include "nasal.h"
+#include "nasal_err.h"
+#include "nasal_lexer.h"
+#include "nasal_ast.h"
+#include "nasal_parse.h"
+#include "nasal_import.h"
+#include "nasal_opt.h"
+#include "nasal_gc.h"
+#include "nasal_builtin.h"
+#include "nasal_codegen.h"
+#include "nasal_vm.h"
+#include "nasal_dbg.h"
+
+#include <unordered_map>
+
+const u32 VM_AST   =0x01;
+const u32 VM_CODE  =0x02;
+const u32 VM_TIME  =0x04;
+const u32 VM_EXEC  =0x08;
+const u32 VM_DETAIL=0x10;
+const u32 VM_DEBUG =0x20;
+
+std::ostream& help(std::ostream& out) {
+    out
+    <<"     ,--#-,\n"
+    <<"<3  / \\____\\  <3\n"
+    <<"    |_|__A_|\n"
+#ifdef _WIN32
+    <<"use command <chcp 65001> to use unicode.\n"
+#endif
+    <<"\nnasal <option>\n"
+    <<"option:\n"
+    <<"   -h,   --help   | get help.\n"
+    <<"\nnasal [option] <file> [argv]\n"
+    <<"option:\n"
+    <<"   -a,   --ast    | view abstract syntax tree.\n"
+    <<"   -c,   --code   | view bytecode.\n"
+    <<"   -e,   --exec   | execute.\n"
+    <<"   -t,   --time   | show execute time.\n"
+    <<"   -d,   --detail | get detail info.\n"
+    <<"   -dbg, --debug  | debug mode.\n"
+    <<"file:\n"
+    <<"   <filename>     | execute file.\n"
+    <<"argv:\n"
+    <<"   <args>         | cmd arguments used in program.\n";
+    return out;
+}
+
+std::ostream& logo(std::ostream& out) {
+    out
+    <<"       __                _\n"
+    <<"    /\\ \\ \\__ _ ___  __ _| |\n"
+    <<"   /  \\/ / _` / __|/ _` | |\n"
+    <<"  / /\\  / (_| \\__ \\ (_| | |\n"
+    <<"  \\_\\ \\/ \\__,_|___/\\__,_|_|\n"
+    <<"ver  : "<<__nasver<<" ("<<__DATE__<<" "<<__TIME__<<")\n"
+    <<"std  : c++ "<<__cplusplus<<"\n"
+    <<"repo : https://github.com/ValKmjolnir/Nasal-Interpreter\n"
+    <<"repo : https://gitee.com/valkmjolnir/Nasal-Interpreter\n"
+    <<"wiki : https://wiki.flightgear.org/Nasal_scripting_language\n"
+    <<"input <nasal -h> to get help .\n";
+    return out;
+}
+
+[[noreturn]]
+void err() {
+    std::cerr
+    <<"invalid argument(s).\n"
+    <<"use <nasal -h> to get help.\n";
+    std::exit(1);
+}
+
+void execute(
+    const string& file,
+    const std::vector<string>& argv,
+    const u32 cmd
+) {
+    using clk=std::chrono::high_resolution_clock;
+    const auto den=clk::duration::period::den;
+
+    error   err;
+    lexer   lex(err);
+    parse   parse(err);
+    linker  ld(err);
+    codegen gen(err);
+    vm      ctx;
+
+    // lexer scans file to get tokens
+    lex.scan(file).chkerr();
+
+    // parser gets lexer's token list to compile
+    parse.compile(lex).chkerr();
+
+    // linker gets parser's ast and load import files to this ast
+    ld.link(parse, file, cmd&VM_DETAIL).chkerr();
+
+    // optimizer does simple optimization on ast
+    optimize(parse.tree());
+    if (cmd&VM_AST) {
+        parse.tree().dump();
+    }
+
+    // code generator gets parser's ast and import file list to generate code
+    gen.compile(parse, ld).chkerr();
+    if (cmd&VM_CODE) {
+        gen.print();
+    }
+
+    // run
+    auto start=clk::now();
+    if (cmd&VM_DEBUG) {
+        dbg(err).run(gen, ld, argv);
+    } else if (cmd&VM_TIME || cmd&VM_EXEC) {
+        ctx.run(gen, ld, argv, cmd&VM_DETAIL);
+    }
+
+    // get running time
+    if (cmd&VM_TIME) {
+        f64 tm=(clk::now()-start).count()*1.0/den;
+        std::clog<<"process exited after "<<tm<<"s.\n\n";
+    }
+}
+
+i32 main(i32 argc, const char* argv[]) {
+    // output version info
+    if (argc<=1) {
+        std::clog<<logo;
+        return 0;
+    }
+
+    // run directly or show help
+    if (argc==2) {
+        string s(argv[1]);
+        if (s=="-h" || s=="--help") {
+            std::clog<<help;
+        } else if (s[0]!='-') {
+            execute(s, {}, VM_EXEC);
+        } else {
+            err();
+        }
+        return 0;
+    }
+
+    // execute with arguments
+    const std::unordered_map<string,u32> cmdlst={
+        {"--ast", VM_AST},
+        {"-a", VM_AST},
+        {"--code", VM_CODE},
+        {"-c", VM_CODE},
+        {"--exec", VM_EXEC},
+        {"-e", VM_EXEC},
+        {"--time", VM_TIME|VM_EXEC},
+        {"-t", VM_TIME|VM_EXEC},
+        {"--detail", VM_DETAIL|VM_EXEC},
+        {"-d", VM_DETAIL|VM_EXEC},
+        {"--debug", VM_DEBUG},
+        {"-dbg", VM_DEBUG}
+    };
+    u32 cmd=0;
+    string filename="";
+    std::vector<string> vm_argv;
+    for(i32 i=1; i<argc; ++i) {
+        if (cmdlst.count(argv[i])) {
+            cmd|=cmdlst.at(argv[i]);
+        } else if (!filename.length()) {
+            filename=argv[i];
+        } else {
+            vm_argv.push_back(argv[i]);
+        }
+    }
+    if (!filename.length()) {
+        err();
+    }
+    execute(filename, vm_argv, cmd?cmd:VM_EXEC);
+    return 0;
+}

makefile

@@ -18,12 +18,15 @@ SRC=\
 
 STD=c++14
 
-nasal:$(SRC) nasal_new_ast.o nasal_new_parse.o ast_visitor.o
+nasal:$(SRC) nasal_new_lexer.o nasal_new_ast.o nasal_new_parse.o ast_visitor.o
 	$(CXX) -std=$(STD) -O3 main.cpp -o nasal -fno-exceptions -ldl -Wshadow -Wall
 
 nasal.exe:$(SRC)
 	$(CXX) -std=$(STD) -O3 main.cpp -o nasal.exe -fno-exceptions -Wshadow -Wall -static
 
+nasal_new_lexer.o: ast/nasal_new_lexer.h ast/nasal_new_lexer.cpp nasal.h
+	$(CXX) -std=$(STD) -c -O3 ast/nasal_new_lexer.cpp -fno-exceptions -fPIC -o nasal_new_lexer.o -I .
+
 nasal_new_ast.o: ast/nasal_new_ast.h ast/nasal_new_ast.cpp nasal.h
 	$(CXX) -std=$(STD) -c -O3 ast/nasal_new_ast.cpp -fno-exceptions -fPIC -o nasal_new_ast.o -I .