diff --git a/spec/fluent.ebnf b/spec/fluent.ebnf index 13a16f2..8f04a84 100644 --- a/spec/fluent.ebnf +++ b/spec/fluent.ebnf @@ -119,7 +119,7 @@ indented_char ::= text_char - "[" - "*" - "." special_quoted_char ::= "\"" | "\\" special_escape ::= "\\" special_quoted_char -unicode_escape ::= "\\u" /[0-9a-fA-F]{4}/ +unicode_escape ::= "\\u{" /[0-9a-fA-F]{1,6}/ "}" quoted_char ::= (any_char - special_quoted_char) | special_escape | unicode_escape diff --git a/syntax/grammar.mjs b/syntax/grammar.mjs index e52f2f9..08ae138 100644 --- a/syntax/grammar.mjs +++ b/syntax/grammar.mjs @@ -438,8 +438,9 @@ let special_escape = let unicode_escape = sequence( - string("\\u"), - regex(/[0-9a-fA-F]{4}/)) + string("\\u{"), + regex(/[0-9a-fA-F]{1,6}/), + string("}")) .map(join); let quoted_char = diff --git a/test/fixtures/astral.ftl b/test/fixtures/astral.ftl index b77e32e..7063c57 100644 --- a/test/fixtures/astral.ftl +++ b/test/fixtures/astral.ftl @@ -2,8 +2,8 @@ face-with-tears-of-joy = 😂 tetragram-for-centre = 𝌆 surrogates-in-text = \uD83D\uDE02 -surrogates-in-string = {"\uD83D\uDE02"} -surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"} +surrogates-in-string = {"\u{D83D}\u{DE02}"} +surrogates-in-adjacent-strings = {"\u{D83D}"}{"\u{DE02}"} emoji-in-text = A face 😂 with tears of joy. emoji-in-string = {"A face 😂 with tears of joy."} diff --git a/test/fixtures/astral.json b/test/fixtures/astral.json index 6056fb7..422f5b8 100644 --- a/test/fixtures/astral.json +++ b/test/fixtures/astral.json @@ -68,7 +68,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D\\uDE02" + "value": "\\u{D83D}\\u{DE02}" } } ] @@ -89,14 +89,14 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uD83D" + "value": "\\u{D83D}" } }, { "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\uDE02" + "value": "\\u{DE02}" } } ] diff --git a/test/fixtures/escaped_characters.ftl b/test/fixtures/escaped_characters.ftl index 5242a4b..589e5c6 100644 --- a/test/fixtures/escaped_characters.ftl +++ b/test/fixtures/escaped_characters.ftl @@ -2,8 +2,8 @@ text-backslash-one = Value with \ a backslash text-backslash-two = Value with \\ two backslashes text-backslash-brace = Value with \{placeable} -text-backslash-u = \u0041 -text-backslash-backslash-u = \\u0041 +text-backslash-u = \u{41} +text-backslash-backslash-u = \\u{41} ## String literals quote-in-string = {"\""} @@ -14,8 +14,25 @@ mismatched-quote = {"\\""} unknown-escape = {"\x"} ## Unicode escapes -string-unicode-sequence = {"\u0041"} -string-escaped-unicode = {"\\u0041"} +string-unicode-1digit = {"\u{9}"} +string-unicode-2digits = {"\u{09}"} +string-unicode-3digits = {"\u{009}"} +string-unicode-4digits = {"\u{0009}"} +string-unicode-5digits = {"\u{00009}"} +string-unicode-6digits = {"\u{000009}"} + +escape-unicode-4digits = {"\\u{41}"} +escape-unicode-6digits = {"\\u{01F602}"} + +# ERROR Too few hex digits. +string-unicode-0digits = {"\u{}"} +# ERROR Too many hex digits. +string-unicode-7digits = {"\U{001F602}"} + +# ERROR Missing opening brace. +string-unicode-missing-open = {"\u9}"} +# ERROR Missing closing brace. +string-unicode-missing-close = {"\u{9"} ## Literal braces brace-open = An opening {"{"} brace. diff --git a/test/fixtures/escaped_characters.json b/test/fixtures/escaped_characters.json index 26b1974..2dab38d 100644 --- a/test/fixtures/escaped_characters.json +++ b/test/fixtures/escaped_characters.json @@ -80,7 +80,14 @@ "elements": [ { "type": "TextElement", - "value": "\\u0041" + "value": "\\u" + }, + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "41" + } } ] }, @@ -98,7 +105,14 @@ "elements": [ { "type": "TextElement", - "value": "\\\\u0041" + "value": "\\\\u" + }, + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "41" + } } ] }, @@ -177,7 +191,133 @@ "type": "Message", "id": { "type": "Identifier", - "name": "string-unicode-sequence" + "name": "string-unicode-1digit" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u{9}" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-2digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u{09}" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-3digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u{009}" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u{0009}" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-5digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u{00009}" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "\\u{000009}" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-4digits" }, "value": { "type": "Pattern", @@ -186,7 +326,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\u0041" + "value": "\\\\u{41}" } } ] @@ -198,7 +338,7 @@ "type": "Message", "id": { "type": "Identifier", - "name": "string-escaped-unicode" + "name": "escape-unicode-6digits" }, "value": { "type": "Pattern", @@ -207,7 +347,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\u0041" + "value": "\\\\u{01F602}" } } ] @@ -215,6 +355,42 @@ "attributes": [], "comment": null }, + { + "type": "Comment", + "content": "ERROR Too few hex digits." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-unicode-0digits = {\"\\u{}\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Too many hex digits." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-unicode-7digits = {\"\\U{001F602}\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Missing opening brace." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-unicode-missing-open = {\"\\u9}\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Missing closing brace." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-unicode-missing-close = {\"\\u{9\"}\n" + }, { "type": "GroupComment", "content": "Literal braces"