From f1b4ee84c5a8cc16bcbb914ec9cc5f6d2d4f53f0 Mon Sep 17 00:00:00 2001 From: Keith Winstein Date: Mon, 3 Oct 2022 11:37:13 -0700 Subject: [PATCH 1/3] Log all lexing errors in WastLexer (rather than via parser) --- include/wabt/wast-lexer.h | 18 ++++++++++++------ src/test-wast-parser.cc | 3 ++- src/tools/spectest-interp.cc | 4 ++-- src/tools/wast2json.cc | 4 ++-- src/tools/wat-desugar.cc | 4 ++-- src/tools/wat2wasm.cc | 4 ++-- src/wast-lexer.cc | 34 ++++++++++++++++++++-------------- src/wast-parser.cc | 6 +++--- test/spec/tokens.txt | 3 +++ 9 files changed, 48 insertions(+), 32 deletions(-) diff --git a/include/wabt/wast-lexer.h b/include/wabt/wast-lexer.h index 3c64fea919..23ee5ea91e 100644 --- a/include/wabt/wast-lexer.h +++ b/include/wabt/wast-lexer.h @@ -22,6 +22,7 @@ #include #include "wabt/common.h" +#include "wabt/error.h" #include "wabt/lexer-source-line-finder.h" #include "wabt/literal.h" #include "wabt/make-unique.h" @@ -32,20 +33,22 @@ namespace wabt { class ErrorHandler; class LexerSource; -class WastParser; class WastLexer { public: WABT_DISALLOW_COPY_AND_ASSIGN(WastLexer); - WastLexer(std::unique_ptr source, std::string_view filename); + WastLexer(std::unique_ptr source, + std::string_view filename, + Errors*); // Convenience functions. static std::unique_ptr CreateBufferLexer(std::string_view filename, const void* data, - size_t size); + size_t size, + Errors*); - Token GetToken(WastParser* parser); + Token GetToken(); // TODO(binji): Move this out of the lexer. std::unique_ptr MakeLineFinder() { @@ -68,7 +71,7 @@ class WastLexer { bool MatchChar(char); bool MatchString(std::string_view); void Newline(); - bool ReadBlockComment(WastParser*); // Returns false if EOF. + bool ReadBlockComment(); // Returns false if EOF. bool ReadLineComment(); // Returns false if EOF. void ReadWhitespace(); @@ -87,7 +90,7 @@ class WastLexer { return ReadReservedChars() == ReservedChars::None; } void ReadSign(); - Token GetStringToken(WastParser*); + Token GetStringToken(); Token GetNumberToken(TokenType); Token GetHexNumberToken(TokenType); Token GetInfToken(); @@ -105,6 +108,9 @@ class WastLexer { const char* line_start_; const char* token_start_; const char* cursor_; + + Errors* errors_; + void WABT_PRINTF_FORMAT(3, 4) Error(Location, const char* format, ...); }; } // namespace wabt diff --git a/src/test-wast-parser.cc b/src/test-wast-parser.cc index 69b5cf67fd..60fc601dbd 100644 --- a/src/test-wast-parser.cc +++ b/src/test-wast-parser.cc @@ -34,8 +34,9 @@ std::string repeat(std::string s, size_t count) { } Errors ParseInvalidModule(std::string text) { - auto lexer = WastLexer::CreateBufferLexer("test", text.c_str(), text.size()); Errors errors; + auto lexer = + WastLexer::CreateBufferLexer("test", text.c_str(), text.size(), &errors); std::unique_ptr module; Features features; WastParseOptions options(features); diff --git a/src/tools/spectest-interp.cc b/src/tools/spectest-interp.cc index af50bfb29a..aae1134714 100644 --- a/src/tools/spectest-interp.cc +++ b/src/tools/spectest-interp.cc @@ -1382,9 +1382,9 @@ wabt::Result CommandRunner::ReadInvalidTextModule( const std::string& header) { std::vector file_data; wabt::Result result = ReadFile(module_filename, &file_data); - std::unique_ptr lexer = WastLexer::CreateBufferLexer( - module_filename, file_data.data(), file_data.size()); Errors errors; + std::unique_ptr lexer = WastLexer::CreateBufferLexer( + module_filename, file_data.data(), file_data.size(), &errors); if (Succeeded(result)) { std::unique_ptr module; WastParseOptions options(s_features); diff --git a/src/tools/wast2json.cc b/src/tools/wast2json.cc index bdc4a6740e..4b92e2793b 100644 --- a/src/tools/wast2json.cc +++ b/src/tools/wast2json.cc @@ -104,13 +104,13 @@ int ProgramMain(int argc, char** argv) { std::vector file_data; Result result = ReadFile(s_infile, &file_data); + Errors errors; std::unique_ptr lexer = WastLexer::CreateBufferLexer( - s_infile, file_data.data(), file_data.size()); + s_infile, file_data.data(), file_data.size(), &errors); if (Failed(result)) { WABT_FATAL("unable to read file: %s\n", s_infile); } - Errors errors; std::unique_ptr