Recognize \UHHHHHH as an escape sequence (#201)

Author: stasm

spec/fluent.ebnf

@@ -132,7 +132,8 @@ indented_char       ::= text_char - "[" - "*" - "."
 special_quoted_char ::= "\""
                       | "\\"
 special_escape      ::= "\\" special_quoted_char
-unicode_escape      ::= "\\u" /[0-9a-fA-F]{4}/
+unicode_escape      ::= ("\\u" /[0-9a-fA-F]{4}/)
+                      | ("\\U" /[0-9a-fA-F]{6}/)
 quoted_char         ::= (any_char - special_quoted_char - line_end)
                       | special_escape
                       | unicode_escape

syntax/abstract.mjs

@@ -242,20 +242,22 @@ function remove_blank_lines(element) {
     return typeof(element) !== "string";
 }
 
-const KNOWN_ESCAPES = /(?:\\\\|\\\"|\\u([0-9a-fA-F]{4}))/g;
+// Backslash backslash, backslash double quote, uHHHH, UHHHHHH.
+const KNOWN_ESCAPES =
+    /(?:\\\\|\\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g;
 
 function unescape(raw) {
     return raw.replace(KNOWN_ESCAPES, from_escape_sequence);
 }
 
-function from_escape_sequence(match, group1) {
+function from_escape_sequence(match, codepoint4, codepoint6) {
     switch (match) {
         case "\\\\":
             return "\\";
         case "\\\"":
             return "\"";
         default:
-            let codepoint = parseInt(group1, 16);
+            let codepoint = parseInt(codepoint4 || codepoint6, 16);
             if (codepoint <= 0xD7FF || 0xE000 <= codepoint) {
                 // It's a Unicode scalar value.
                 return String.fromCodePoint(codepoint);

syntax/grammar.mjs

@@ -457,9 +457,13 @@ let special_escape =
     .map(join);
 
 let unicode_escape =
-    sequence(
-        string("\\u"),
-        regex(/[0-9a-fA-F]{4}/))
+    either(
+        sequence(
+            string("\\u"),
+            regex(/[0-9a-fA-F]{4}/)),
+        sequence(
+            string("\\U"),
+            regex(/[0-9a-fA-F]{6}/)))
     .map(join);
 
 let quoted_char = defer(() =>

test/fixtures/escaped_characters.ftl

@@ -14,8 +14,20 @@ mismatched-quote = {"\\""}
 unknown-escape = {"\x"}
 
 ## Unicode escapes
-string-unicode-sequence = {"\u0041"}
-string-escaped-unicode = {"\\u0041"}
+string-unicode-4digits = {"\u0041"}
+escape-unicode-4digits = {"\\u0041"}
+string-unicode-6digits = {"\U01F602"}
+escape-unicode-6digits = {"\\U01F602"}
+
+# OK The trailing "00" is part of the literal value.
+string-too-many-4digits = {"\u004100"}
+# OK The trailing "00" is part of the literal value.
+string-too-many-6digits = {"\U01F60200"}
+
+# ERROR Too few hex digits after \u.
+string-too-few-4digits = {"\u41"}
+# ERROR Too few hex digits after \U.
+string-too-few-6digits = {"\U1F602"}
 
 ## Literal braces
 brace-open = An opening {"{"} brace.

test/fixtures/escaped_characters.json

@@ -179,7 +179,7 @@
             "type": "Message",
             "id": {
                 "type": "Identifier",
-                "name": "string-unicode-sequence"
+                "name": "string-unicode-4digits"
             },
             "value": {
                 "type": "Pattern",
@@ -201,7 +201,7 @@
             "type": "Message",
             "id": {
                 "type": "Identifier",
-                "name": "string-escaped-unicode"
+                "name": "escape-unicode-4digits"
             },
             "value": {
                 "type": "Pattern",
@@ -219,6 +219,118 @@
             "attributes": [],
             "comment": null
         },
+        {
+            "type": "Message",
+            "id": {
+                "type": "Identifier",
+                "name": "string-unicode-6digits"
+            },
+            "value": {
+                "type": "Pattern",
+                "elements": [
+                    {
+                        "type": "Placeable",
+                        "expression": {
+                            "type": "StringLiteral",
+                            "raw": "\\U01F602",
+                            "value": "😂"
+                        }
+                    }
+                ]
+            },
+            "attributes": [],
+            "comment": null
+        },
+        {
+            "type": "Message",
+            "id": {
+                "type": "Identifier",
+                "name": "escape-unicode-6digits"
+            },
+            "value": {
+                "type": "Pattern",
+                "elements": [
+                    {
+                        "type": "Placeable",
+                        "expression": {
+                            "type": "StringLiteral",
+                            "raw": "\\\\U01F602",
+                            "value": "\\U01F602"
+                        }
+                    }
+                ]
+            },
+            "attributes": [],
+            "comment": null
+        },
+        {
+            "type": "Message",
+            "id": {
+                "type": "Identifier",
+                "name": "string-too-many-4digits"
+            },
+            "value": {
+                "type": "Pattern",
+                "elements": [
+                    {
+                        "type": "Placeable",
+                        "expression": {
+                            "type": "StringLiteral",
+                            "raw": "\\u004100",
+                            "value": "A00"
+                        }
+                    }
+                ]
+            },
+            "attributes": [],
+            "comment": {
+                "type": "Comment",
+                "content": "OK The trailing \"00\" is part of the literal value."
+            }
+        },
+        {
+            "type": "Message",
+            "id": {
+                "type": "Identifier",
+                "name": "string-too-many-6digits"
+            },
+            "value": {
+                "type": "Pattern",
+                "elements": [
+                    {
+                        "type": "Placeable",
+                        "expression": {
+                            "type": "StringLiteral",
+                            "raw": "\\U01F60200",
+                            "value": "😂00"
+                        }
+                    }
+                ]
+            },
+            "attributes": [],
+            "comment": {
+                "type": "Comment",
+                "content": "OK The trailing \"00\" is part of the literal value."
+            }
+        },
+        {
+            "type": "Comment",
+            "content": "ERROR Too few hex digits after \\u."
+        },
+        {
+            "type": "Junk",
+            "annotations": [],
+            "content": "string-too-few-4digits = {\"\\u41\"}\n"
+        },
+        {
+            "type": "Comment",
+            "content": "ERROR Too few hex digits after \\U."
+        },
+        {
+            "type": "Junk",
+            "annotations": [],
+            "content": "string-too-few-6digits = {\"\\U1F602\"}\n\n"
+        },
         {
             "type": "GroupComment",
             "content": "Literal braces"