Implement verbatim string literals (still need tests)

Fixes google#246

Author: benley

core/ast.h

@@ -473,7 +473,7 @@ struct LiteralNumber : public AST {
 /** Represents JSON strings. */
 struct LiteralString : public AST {
     String value;
-    enum TokenKind { SINGLE, DOUBLE, BLOCK };
+    enum TokenKind { SINGLE, DOUBLE, BLOCK, VERBATIM_SINGLE, VERBATIM_DOUBLE };
     TokenKind tokenKind;
     std::string blockIndent;  // Only contains ' ' and '\t'.
     std::string blockTermIndent;  // Only contains ' ' and '\t'.

core/desugarer.cpp

@@ -609,7 +609,9 @@ class Desugarer {
             // Nothing to do.
 
         } else if (auto *ast = dynamic_cast<LiteralString*>(ast_)) {
-            if (ast->tokenKind != LiteralString::BLOCK) {
+            if ((ast->tokenKind != LiteralString::BLOCK) &&
+                (ast->tokenKind != LiteralString::VERBATIM_DOUBLE) &&
+                (ast->tokenKind != LiteralString::VERBATIM_SINGLE)) {
                 ast->value = jsonnet_string_unescape(ast->location, ast->value);
             }
             ast->tokenKind = LiteralString::DOUBLE;

core/formatter.cpp

@@ -463,6 +463,14 @@ class Unparser {
                     }
                 }
                 o << ast->blockTermIndent << "|||";
+            } else if (ast->tokenKind == LiteralString::VERBATIM_DOUBLE) {
+                o << "@\"";
+                o << encode_utf8(ast->value);
+                o << "\"";
+            } else if (ast->tokenKind == LiteralString::VERBATIM_SINGLE) {
+                o << "@'";
+                o << encode_utf8(ast->value);
+                o << "'";
             }
 
         } else if (dynamic_cast<const LiteralNull*>(ast_)) {

core/lexer.cpp

@@ -507,6 +507,39 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
             }
             break;
 
+            // Verbatim string literals.
+            case '@': {
+                c++;
+                if (*c != '"' && *c != '\'') {
+                    std::stringstream ss;
+                    ss << "Couldn't lex verbatim string, junk after '@': " << *c;
+                    throw StaticError(filename, begin, ss.str());
+                }
+                const char quot = *c;
+                c++;  // Advance beyond the opening quote.
+                for (; ; ++c) {
+                    if (*c == '\0')  {
+                        throw StaticError(filename, begin, "Unterminated verbatim string");
+                    }
+                    if (*c == quot) {
+                        if (*(c+1) == quot) {
+                           data += *c;
+                           ++c;
+                       } else {
+                           break;
+                       }
+                    }
+                    data += *c;
+                }
+                c++;  // Advance beyond the closing quote.
+                if (quot == '"') {
+                    kind = Token::VERBATIM_STRING_DOUBLE;
+                } else {
+                    kind = Token::VERBATIM_STRING_SINGLE;
+                }
+            }
+            break;
+
             // Keywords
             default:
             if (is_identifier_first(*c)) {

core/lexer.h

@@ -102,6 +102,8 @@ struct Token {
         STRING_DOUBLE,
         STRING_SINGLE,
         STRING_BLOCK,
+        VERBATIM_STRING_SINGLE,
+        VERBATIM_STRING_DOUBLE,
 
         // Keywords
         ASSERT,

core/parser.cpp

@@ -634,6 +634,13 @@ class Parser {
             return alloc->make<LiteralString>(
                 span(tok), tok.fodder, tok.data32(), LiteralString::BLOCK,
                 tok.stringBlockIndent, tok.stringBlockTermIndent);
+            case Token::VERBATIM_STRING_SINGLE:
+            return alloc->make<LiteralString>(
+                span(tok), tok.fodder, tok.data32(), LiteralString::VERBATIM_SINGLE, "", "");
+            case Token::VERBATIM_STRING_DOUBLE:
+            return alloc->make<LiteralString>(
+                span(tok), tok.fodder, tok.data32(), LiteralString::VERBATIM_DOUBLE, "", "");
+
 
             case Token::FALSE:
             return alloc->make<LiteralBoolean>(span(tok), tok.fodder, false);