WastLexer: log lexing errors directly (#2013)

* Log all lexing errors in WastLexer (rather than via parser)

* Update docs/demo/libwabt.js

include/wabt/wast-lexer.h

@@ -22,6 +22,7 @@
 #include <memory>
 
 #include "wabt/common.h"
+#include "wabt/error.h"
 #include "wabt/lexer-source-line-finder.h"
 #include "wabt/literal.h"
 #include "wabt/make-unique.h"
@@ -32,20 +33,22 @@ namespace wabt {
 
 class ErrorHandler;
 class LexerSource;
-class WastParser;
 
 class WastLexer {
  public:
   WABT_DISALLOW_COPY_AND_ASSIGN(WastLexer);
 
-  WastLexer(std::unique_ptr<LexerSource> source, std::string_view filename);
+  WastLexer(std::unique_ptr<LexerSource> source,
+            std::string_view filename,
+            Errors*);
 
   // Convenience functions.
   static std::unique_ptr<WastLexer> CreateBufferLexer(std::string_view filename,
                                                       const void* data,
-                                                      size_t size);
+                                                      size_t size,
+                                                      Errors*);
 
-  Token GetToken(WastParser* parser);
+  Token GetToken();
 
   // TODO(binji): Move this out of the lexer.
   std::unique_ptr<LexerSourceLineFinder> MakeLineFinder() {
@@ -68,7 +71,7 @@ class WastLexer {
   bool MatchChar(char);
   bool MatchString(std::string_view);
   void Newline();
-  bool ReadBlockComment(WastParser*);  // Returns false if EOF.
+  bool ReadBlockComment();             // Returns false if EOF.
   bool ReadLineComment();              // Returns false if EOF.
   void ReadWhitespace();
 
@@ -87,7 +90,7 @@ class WastLexer {
     return ReadReservedChars() == ReservedChars::None;
   }
   void ReadSign();
-  Token GetStringToken(WastParser*);
+  Token GetStringToken();
   Token GetNumberToken(TokenType);
   Token GetHexNumberToken(TokenType);
   Token GetInfToken();
@@ -105,6 +108,9 @@ class WastLexer {
   const char* line_start_;
   const char* token_start_;
   const char* cursor_;
+
+  Errors* errors_;
+  void WABT_PRINTF_FORMAT(3, 4) Error(Location, const char* format, ...);
 };
 
 }  // namespace wabt

src/emscripten-helpers.cc

@@ -96,9 +96,10 @@ void wabt_destroy_features(wabt::Features* f) {
 
 wabt::WastLexer* wabt_new_wast_buffer_lexer(const char* filename,
                                             const void* data,
-                                            size_t size) {
+                                            size_t size,
+                                            wabt::Errors* errors) {
   std::unique_ptr<wabt::WastLexer> lexer =
-      wabt::WastLexer::CreateBufferLexer(filename, data, size);
+      wabt::WastLexer::CreateBufferLexer(filename, data, size, errors);
   return lexer.release();
 }
 

src/test-wast-parser.cc

@@ -34,8 +34,9 @@ std::string repeat(std::string s, size_t count) {
 }
 
 Errors ParseInvalidModule(std::string text) {
-  auto lexer = WastLexer::CreateBufferLexer("test", text.c_str(), text.size());
   Errors errors;
+  auto lexer =
+      WastLexer::CreateBufferLexer("test", text.c_str(), text.size(), &errors);
   std::unique_ptr<Module> module;
   Features features;
   WastParseOptions options(features);

src/tools/spectest-interp.cc

@@ -1382,9 +1382,9 @@ wabt::Result CommandRunner::ReadInvalidTextModule(
     const std::string& header) {
   std::vector<uint8_t> file_data;
   wabt::Result result = ReadFile(module_filename, &file_data);
-  std::unique_ptr<WastLexer> lexer = WastLexer::CreateBufferLexer(
-      module_filename, file_data.data(), file_data.size());
   Errors errors;
+  std::unique_ptr<WastLexer> lexer = WastLexer::CreateBufferLexer(
+      module_filename, file_data.data(), file_data.size(), &errors);
   if (Succeeded(result)) {
     std::unique_ptr<wabt::Module> module;
     WastParseOptions options(s_features);

src/tools/wast2json.cc

@@ -104,13 +104,13 @@ int ProgramMain(int argc, char** argv) {
 
   std::vector<uint8_t> file_data;
   Result result = ReadFile(s_infile, &file_data);
+  Errors errors;
   std::unique_ptr<WastLexer> lexer = WastLexer::CreateBufferLexer(
-      s_infile, file_data.data(), file_data.size());
+      s_infile, file_data.data(), file_data.size(), &errors);
   if (Failed(result)) {
     WABT_FATAL("unable to read file: %s\n", s_infile);
   }
 
-  Errors errors;
   std::unique_ptr<Script> script;
   WastParseOptions parse_wast_options(s_features);
   result = ParseWastScript(lexer.get(), &script, &errors, &parse_wast_options);

src/tools/wat-desugar.cc

@@ -90,10 +90,10 @@ int ProgramMain(int argc, char** argv) {
     WABT_FATAL("unable to read %s\n", s_infile);
   }
 
+  Errors errors;
   std::unique_ptr<WastLexer> lexer(WastLexer::CreateBufferLexer(
-      s_infile, file_data.data(), file_data.size()));
+      s_infile, file_data.data(), file_data.size(), &errors));
 
-  Errors errors;
   std::unique_ptr<Script> script;
   WastParseOptions parse_wast_options(s_features);
   result = ParseWastScript(lexer.get(), &script, &errors, &parse_wast_options);

src/tools/wat2wasm.cc

@@ -133,13 +133,13 @@ int ProgramMain(int argc, char** argv) {
 
   std::vector<uint8_t> file_data;
   Result result = ReadFile(s_infile, &file_data);
+  Errors errors;
   std::unique_ptr<WastLexer> lexer = WastLexer::CreateBufferLexer(
-      s_infile, file_data.data(), file_data.size());
+      s_infile, file_data.data(), file_data.size(), &errors);
   if (Failed(result)) {
     WABT_FATAL("unable to read file: %s\n", s_infile);
   }
 
-  Errors errors;
   std::unique_ptr<Module> module;
   WastParseOptions parse_wast_options(s_features);
   result = ParseWatModule(lexer.get(), &module, &errors, &parse_wast_options);

src/wabt.post.js

@@ -110,11 +110,12 @@ FEATURES.forEach(function(feature) {
 
 
 /// Lexer
-function Lexer(filename, buffer) {
+function Lexer(filename, buffer, errors) {
   this.filenameObj = allocateCString(filename);
   this.bufferObj = allocateBuffer(buffer);
   this.addr = Module._wabt_new_wast_buffer_lexer(
-      this.filenameObj.addr, this.bufferObj.addr, this.bufferObj.size);
+      this.filenameObj.addr, this.bufferObj.addr, this.bufferObj.size,
+      errors.addr);
 }
 Lexer.prototype = Object.create(Object.prototype);
 
@@ -159,10 +160,9 @@ OutputBuffer.prototype.destroy = function() {
 
 
 /// Errors
-function Errors(kind, lexer) {
+function Errors(kind) {
   this.kind = kind;
   this.addr = Module._wabt_new_errors();
-  this.lexer = lexer;
 }
 Errors.prototype = Object.create(Object.prototype);
 
@@ -194,8 +194,9 @@ Errors.prototype.destroy = function() {
 
 /// parseWat
 function parseWat(filename, buffer, options) {
-  var lexer = new Lexer(filename, buffer);
-  var errors = new Errors('text', lexer);
+  var errors = new Errors('text');
+  var lexer = new Lexer(filename, buffer, errors);
+  errors.lexer = lexer;
   var features = new Features(options || {});
 
   try {

src/wast-lexer.cc

@@ -22,11 +22,8 @@
 #include "wabt/config.h"
 
 #include "wabt/lexer-source.h"
-#include "wabt/wast-parser.h"
 
-#define ERROR(...) \
-  if (parser)      \
-  parser->Error(GetLocation(), __VA_ARGS__)
+#define ERROR(...) Error(GetLocation(), __VA_ARGS__)
 
 namespace wabt {
 
@@ -37,25 +34,29 @@ namespace {
 }  // namespace
 
 WastLexer::WastLexer(std::unique_ptr<LexerSource> source,
-                     std::string_view filename)
+                     std::string_view filename,
+                     Errors* errors)
     : source_(std::move(source)),
       filename_(filename),
       line_(1),
       buffer_(static_cast<const char*>(source_->data())),
       buffer_end_(buffer_ + source_->size()),
       line_start_(buffer_),
       token_start_(buffer_),
-      cursor_(buffer_) {}
+      cursor_(buffer_),
+      errors_(errors) {}
 
 // static
 std::unique_ptr<WastLexer> WastLexer::CreateBufferLexer(
     std::string_view filename,
     const void* data,
-    size_t size) {
-  return MakeUnique<WastLexer>(MakeUnique<LexerSource>(data, size), filename);
+    size_t size,
+    Errors* errors) {
+  return MakeUnique<WastLexer>(MakeUnique<LexerSource>(data, size), filename,
+                               errors);
 }
 
-Token WastLexer::GetToken(WastParser* parser) {
+Token WastLexer::GetToken() {
   while (true) {
     token_start_ = cursor_;
     switch (PeekChar()) {
@@ -64,7 +65,7 @@ Token WastLexer::GetToken(WastParser* parser) {
 
       case '(':
         if (MatchString("(;")) {
-          if (ReadBlockComment(parser)) {
+          if (ReadBlockComment()) {
             continue;
           }
           return BareToken(TokenType::Eof);
@@ -103,7 +104,7 @@ Token WastLexer::GetToken(WastParser* parser) {
         continue;
 
       case '"':
-        return GetStringToken(parser);
+        return GetStringToken();
 
       case '+':
       case '-':
@@ -234,7 +235,7 @@ void WastLexer::Newline() {
   line_start_ = cursor_;
 }
 
-bool WastLexer::ReadBlockComment(WastParser* parser) {
+bool WastLexer::ReadBlockComment() {
   int nesting = 1;
   while (true) {
     switch (ReadChar()) {
@@ -294,7 +295,7 @@ void WastLexer::ReadWhitespace() {
   }
 }
 
-Token WastLexer::GetStringToken(WastParser* parser) {
+Token WastLexer::GetStringToken() {
   const char* saved_token_start = token_start_;
   bool has_error = false;
   bool in_string = true;
@@ -472,7 +473,7 @@ WastLexer::ReservedChars WastLexer::ReadReservedChars() {
         ret = ReservedChars::Id;
       }
     } else if (peek == '"') {
-      GetStringToken(nullptr);
+      GetStringToken();
       ret = ReservedChars::Some;
     } else {
       break;
@@ -608,4 +609,9 @@ Token WastLexer::GetReservedToken() {
   return TextToken(TokenType::Reserved);
 }
 
+void WastLexer::Error(Location loc, const char* format, ...) {
+  WABT_SNPRINTF_ALLOCA(buffer, length, format);
+  errors_->emplace_back(ErrorLevel::Error, loc, buffer);
+}
+
 }  // namespace wabt

src/wast-parser.cc

@@ -584,7 +584,7 @@ void WastParser::Error(Location loc, const char* format, ...) {
 
 Token WastParser::GetToken() {
   if (tokens_.empty()) {
-    tokens_.push_back(lexer_->GetToken(this));
+    tokens_.push_back(lexer_->GetToken());
   }
   return tokens_.front();
 }
@@ -595,7 +595,7 @@ Location WastParser::GetLocation() {
 
 TokenType WastParser::Peek(size_t n) {
   while (tokens_.size() <= n) {
-    Token cur = lexer_->GetToken(this);
+    Token cur = lexer_->GetToken();
     if (cur.token_type() != TokenType::LparAnn) {
       tokens_.push_back(cur);
     } else {
@@ -613,7 +613,7 @@ TokenType WastParser::Peek(size_t n) {
       }
       int indent = 1;
       while (indent > 0) {
-        cur = lexer_->GetToken(this);
+        cur = lexer_->GetToken();
         switch (cur.token_type()) {
           case TokenType::Lpar:
           case TokenType::LparAnn:

test/spec/tokens.txt

@@ -33,6 +33,9 @@ out/test/spec/tokens.wast:144: assert_malformed passed:
   (data $l"a ")
         ^^^^^^
 out/test/spec/tokens.wast:154: assert_malformed passed:
+  out/test/spec/tokens/tokens.33.wat:1:7: error: invalid string token
+  (data $l"a ""b")
+        ^^^^^^
   out/test/spec/tokens/tokens.33.wat:1:7: error: unexpected token $l"a ""b", expected ).
   (data $l"a ""b")
         ^^^^^^^^^