Merge pull request #171 from sparkprime/fix_keywords

Fix #170

Author: sparkprime

core/formatter.cpp

@@ -16,6 +16,7 @@ limitations under the License.
 
 #include <typeinfo>
 
+#include "lexer.h"
 #include "formatter.h"
 #include "string_utils.h"
 #include "unicode.h"
@@ -1281,6 +1282,9 @@ class PrettyFieldNames : public Pass {
                 continue;
             return false;
         }
+        // Filter out keywords.
+        if (lex_get_keyword_kind(encode_utf8(str)) != Token::IDENTIFIER)
+            return false;
         return true;
     }
 
@@ -1964,12 +1968,12 @@ std::string jsonnet_fmt(AST *ast, Fodder &final_fodder, const FmtOpts &opts)
         StripAllButComments(alloc, opts).file(ast, final_fodder);
     else if (opts.stripEverything)
         StripEverything(alloc, opts).file(ast, final_fodder);
+    if (opts.prettyFieldNames)
+        PrettyFieldNames(alloc, opts).file(ast, final_fodder);
     if (opts.stringStyle != 'l')
         EnforceStringStyle(alloc, opts).file(ast, final_fodder);
     if (opts.commentStyle != 'l')
         EnforceCommentStyle(alloc, opts).file(ast, final_fodder);
-    if (opts.prettyFieldNames)
-        PrettyFieldNames(alloc, opts).file(ast, final_fodder);
     if (opts.indent > 0)
         FixIndentation(alloc, opts).file(ast, final_fodder);
 

core/lexer.cpp

@@ -16,6 +16,7 @@ limitations under the License.
 
 #include <cassert>
 
+#include <map>
 #include <string>
 #include <sstream>
 
@@ -153,6 +154,33 @@ static bool is_symbol(char c)
     return false;
 }
 
+static const std::map<std::string, Token::Kind> keywords = {
+    {"assert", Token::ASSERT},
+    {"else", Token::ELSE},
+    {"error", Token::ERROR},
+    {"false", Token::FALSE},
+    {"for", Token::FOR},
+    {"function", Token::FUNCTION},
+    {"if", Token::IF},
+    {"import", Token::IMPORT},
+    {"importstr", Token::IMPORTSTR},
+    {"in", Token::IN},
+    {"local", Token::LOCAL},
+    {"null", Token::NULL_LIT},
+    {"self", Token::SELF},
+    {"super", Token::SUPER},
+    {"tailstrict", Token::TAILSTRICT},
+    {"then", Token::THEN},
+    {"true", Token::TRUE},
+};
+
+Token::Kind lex_get_keyword_kind(const std::string &identifier)
+{
+    auto it = keywords.find(identifier);
+    if (it == keywords.end()) return Token::IDENTIFIER;
+    return it->second;
+}
+
 std::string lex_number(const char *&c, const std::string &filename, const Location &begin)
 {
     // This function should be understood with reference to the linked image:
@@ -485,44 +513,7 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
                 std::string id;
                 for (; is_identifier(*c); ++c)
                     id += *c;
-                if (id == "assert") {
-                    kind = Token::ASSERT;
-                } else if (id == "else") {
-                    kind = Token::ELSE;
-                } else if (id == "error") {
-                    kind = Token::ERROR;
-                } else if (id == "false") {
-                    kind = Token::FALSE;
-                } else if (id == "for") {
-                    kind = Token::FOR;
-                } else if (id == "function") {
-                    kind = Token::FUNCTION;
-                } else if (id == "if") {
-                    kind = Token::IF;
-                } else if (id == "import") {
-                    kind = Token::IMPORT;
-                } else if (id == "importstr") {
-                    kind = Token::IMPORTSTR;
-                } else if (id == "in") {
-                    kind = Token::IN;
-                } else if (id == "local") {
-                    kind = Token::LOCAL;
-                } else if (id == "null") {
-                    kind = Token::NULL_LIT;
-                } else if (id == "self") {
-                    kind = Token::SELF;
-                } else if (id == "super") {
-                    kind = Token::SUPER;
-                } else if (id == "tailstrict") {
-                    kind = Token::TAILSTRICT;
-                } else if (id == "then") {
-                    kind = Token::THEN;
-                } else if (id == "true") {
-                    kind = Token::TRUE;
-                } else {
-                    // Not a keyword, must be an identifier.
-                    kind = Token::IDENTIFIER;
-                }
+                kind = lex_get_keyword_kind(id);
                 data = id;
 
             } else if (is_symbol(*c) || *c == '#') {

core/lexer.h

@@ -236,6 +236,9 @@ static inline std::ostream &operator<<(std::ostream &o, const Token &v)
     return o;
 }
 
+/** IF the given identifier is a keyword, return its kind, otherwise return IDENTIFIER. */
+Token::Kind lex_get_keyword_kind(const std::string &identifier);
+
 Tokens jsonnet_lex(const std::string &filename, const char *input);
 
 std::string jsonnet_unlex(const Tokens &tokens);

test_suite/formatter.jsonnet

@@ -269,4 +269,9 @@ limitations under the License.
     test_field19:: [1, 2, 3][/*foo*/ 1/*bar*/:],
     test_field20:: [1, 2, 3][/*foo*/ 1/*bar*/:/*baz*/:],
 
+    prettyFields: {
+        'identifier': true,
+        "not identifier": true,
+        "function": true,  // Test keyword.
+    }
 }

test_suite/formatter.jsonnet.fmt.golden

@@ -269,4 +269,9 @@ limitations under the License.
     test_field19:: [1, 2, 3][/*foo*/ 1/*bar*/:],
     test_field20:: [1, 2, 3][/*foo*/ 1/*bar*/:/*baz*/],
 
+    prettyFields: {
+        identifier: true,
+        "not identifier": true,
+        "function": true,  // Test keyword.
+    },
 }

test_suite/formatter.jsonnet.golden

@@ -1,5 +1,10 @@
 {
    "func": null,
+   "prettyFields": {
+      "function": true,
+      "identifier": true,
+      "not identifier": true
+   },
    "test_field0A": {
       "g": 1
    },

test_suite/unparse.jsonnet.fmt.golden

@@ -24,9 +24,9 @@ limitations under the License.
     string2: '"foo\n bar\n\n\'bar\u0005\"\'\t \u0050\b\f\r\\',
     lit_field1: 1,
     lit_field2: 1,
-    false: false,
-    true: true,
-    null: null,
+    "false": false,
+    "true": true,
+    "null": null,
     hidden_field:: null,
     hidden_field2:: null,
 }