⚡ fix out of bound bug & delete fatal in lexer

makefile

@@ -72,7 +72,7 @@ build/nasal_misc.o: src/nasal.h src/nasal_misc.cpp | build
 build/repl.o: src/nasal.h src/repl.h src/repl.cpp | build
 	$(CXX) -std=$(STD) -c -O3 src/repl.cpp -fno-exceptions -fPIC -o build/repl.o -I .
 
-build/nasal_err.o: src/nasal.h src/nasal_err.h src/nasal_err.cpp | build
+build/nasal_err.o: src/nasal.h src/repl.h src/nasal_err.h src/nasal_err.cpp | build
 	$(CXX) -std=$(STD) -c -O3 src/nasal_err.cpp -fno-exceptions -fPIC -o build/nasal_err.o -I .
 
 build/nasal_gc.o: src/nasal.h src/nasal_gc.h src/nasal_gc.cpp | build
@@ -88,6 +88,7 @@ build/nasal_import.o: \
 
 build/nasal_lexer.o: \
 	src/nasal.h\
+	src/repl.h\
 	src/nasal_err.h\
 	src/nasal_lexer.h src/nasal_lexer.cpp | build
 	$(CXX) -std=$(STD) -c -O3 src/nasal_lexer.cpp -fno-exceptions -fPIC -o build/nasal_lexer.o -I .

src/nasal_err.cpp

@@ -1,4 +1,5 @@
 #include "nasal_err.h"
+#include "repl.h"
 
 #ifdef _WIN32
 #include <windows.h> // use SetConsoleTextAttribute
@@ -76,9 +77,9 @@ void flstream::load(const std::string& f) {
         file = f;
     }
 
-    if (repl_file_info::instance()->in_repl_mode &&
-        repl_file_info::instance()->repl_file_name==file) {
-        const auto& source = repl_file_info::instance()->repl_file_source;
+    if (repl::info::instance()->in_repl_mode &&
+        repl::info::instance()->repl_file_name==file) {
+        const auto& source = repl::info::instance()->repl_file_source;
         res = {};
         size_t pos = 0, last = 0;
         while ((pos = source.find("\n", last))!=std::string::npos) {
@@ -87,6 +88,8 @@ void flstream::load(const std::string& f) {
         }
         if (last<source.length()) {
             res.push_back(source.substr(last));
+        } else {
+            res.push_back("");
         }
         return;
     }
@@ -105,11 +108,6 @@ void flstream::load(const std::string& f) {
     }
 }
 
-void error::fatal(const std::string& stage, const std::string& info) {
-    std::cerr << red << stage << ": " << white << info << reset << "\n\n";
-    std::exit(1);
-}
-
 void error::err(const std::string& stage, const std::string& info) {
     ++cnt;
     std::cerr << red << stage << ": " << white << info << reset << "\n\n";

src/nasal_err.h

@@ -8,18 +8,6 @@
 
 #include "nasal.h"
 
-struct repl_file_info {
-    bool in_repl_mode = false;
-    std::string repl_file_name = "<nasal-repl>";
-    std::string repl_file_source = "";
-
-    // singleton
-    static repl_file_info* instance() {
-        static repl_file_info info;
-        return &info;
-    }
-};
-
 struct span {
     u32 begin_line;
     u32 begin_column;
@@ -66,7 +54,6 @@ private:
 
 public:
     error():cnt(0) {}
-    void fatal(const std::string&, const std::string&);
     void err(const std::string&, const std::string&);
     void warn(const std::string&, const std::string&);
     void err(const std::string&, const span&, const std::string&);

src/nasal_lexer.cpp

@@ -5,6 +5,7 @@
 #endif
 
 #include "nasal_lexer.h"
+#include "repl.h"
 
 bool lexer::skip(char c) {
     return c==' ' || c=='\n' || c=='\t' || c=='\r' || c==0;
@@ -58,15 +59,15 @@ void lexer::err_char() {
     err.err("lexer",
         {line, column-1, line, column, filename},
         "invalid character 0x"+chrhex(c));
-    err.fatal("lexer", "fatal error occurred, stop");
+    ++invalid_char;
 }
 
 void lexer::open(const std::string& file) {
-    if (repl_file_info::instance()->in_repl_mode &&
-        repl_file_info::instance()->repl_file_name==file) {
+    if (repl::info::instance()->in_repl_mode &&
+        repl::info::instance()->repl_file_name==file) {
         err.load(file);
         filename = file;
-        res = repl_file_info::instance()->repl_file_source;
+        res = repl::info::instance()->repl_file_source;
         return;
     }
 
@@ -123,7 +124,7 @@ std::string lexer::utf8_gen() {
             err.err("lexer",
                 {line, column-1, line, column, filename},
                 "invalid utf-8 <"+utf_info+">");
-            err.fatal("lexer", "fatal error occurred, stop");
+            ++invalid_char;
         }
         str += tmp;
         column += 2; // may have some problems because not all the unicode takes 2 space
@@ -358,8 +359,16 @@ const error& lexer::scan(const std::string& file) {
         } else {
             err_char();
         }
+        if (invalid_char>10) {
+            err.err("lexer", "too many invalid characters, stop");
+            break;
+        }
+    }
+    if (toks.size()) {
+        toks.push_back({toks.back().loc, tok::eof, "<eof>"});
+    } else {
+        toks.push_back({{line, column, line, column, filename}, tok::eof, "<eof>"});
     }
-    toks.push_back({{line, column, line, column, filename}, tok::eof, "<eof>"});
     res = "";
     return err;
 }

src/nasal_lexer.h

@@ -95,8 +95,11 @@ private:
     usize ptr;
     std::string filename;
     std::string res;
+
     error err;
+    u64 invalid_char;
     std::vector<token> toks;
+
     const std::unordered_map<std::string, tok> typetbl {
         {"true"    ,tok::tktrue  },
         {"false"   ,tok::tkfalse },
@@ -175,7 +178,7 @@ private:
     token dots();
     token calc_opr();
 public:
-    lexer(): line(1), column(0), ptr(0), filename(""), res("") {}
+    lexer(): line(1), column(0), ptr(0), filename(""), res(""), invalid_char(0) {}
     const error& scan(const std::string&);
     const std::vector<token>& result() const {return toks;}
 };

src/repl.cpp

@@ -20,7 +20,7 @@ void repl::update_temp_file() {
     for(const auto& i : source) {
         content += i + "\n";
     }
-    repl_file_info::instance()->repl_file_source = content;
+    info::instance()->repl_file_source = content;
 }
 
 bool repl::check_need_more_input() {
@@ -45,7 +45,7 @@ bool repl::check_need_more_input() {
                 default: break;
             }
         }
-        if (!in_curve && !in_bracket && !in_brace) {
+        if (in_curve<=0 && in_bracket<=0 && in_brace<=0) {
             break;
         }
         auto line = readline("... ");
@@ -94,7 +94,7 @@ bool repl::run() {
     }
 
     auto end = clk::now();
-    std::clog << "[compile time: " << (end-start).count()*1.0/den << "s]\n";
+    std::clog << "[compile time: " << (end-start).count()*1000.0/den << " ms]\n";
     nasal_runtime->run(*nasal_codegen, *nasal_linker, {}, false);
 
     return true;
@@ -102,8 +102,7 @@ bool repl::run() {
 
 void repl::execute() {
     source = {};
-    auto repl_file_handle = repl_file_info::instance();
-    repl_file_handle->in_repl_mode = true;
+    info::instance()->in_repl_mode = true;
 
     std::cout << "Nasal REPL interpreter(experimental).\n";
     help();

src/repl.h

@@ -10,6 +10,18 @@
 
 namespace repl {
 
+struct info {
+    bool in_repl_mode = false;
+    std::string repl_file_name = "<nasal-repl>";
+    std::string repl_file_source = "";
+
+    // singleton
+    static info* instance() {
+        static info info;
+        return &info;
+    }
+};
+
 class repl {
 private:
     std::vector<std::string> source;