From 6cc26cac72be1452b7574e8801c51d5926a07fa7 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Fri, 9 Aug 2019 12:46:45 +0200 Subject: [PATCH 1/2] Allow whitespace before CallArguments (fixes #124) This is picking up the tests for projectfluent/fluent#281, and adjusts the parser to pass those tests. --- fluent.syntax/fluent/syntax/parser.py | 8 ++++++-- .../tests/syntax/fixtures_reference/call_expressions.ftl | 5 +++-- .../tests/syntax/fixtures_reference/term_parameters.ftl | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/fluent.syntax/fluent/syntax/parser.py b/fluent.syntax/fluent/syntax/parser.py index 5b8e20c3..eded08d8 100644 --- a/fluent.syntax/fluent/syntax/parser.py +++ b/fluent.syntax/fluent/syntax/parser.py @@ -564,17 +564,21 @@ def get_inline_expression(self, ps): ps.next() attribute = self.get_identifier(ps) arguments = None - if ps.current_char == '(': + ps.peek_blank() + if ps.current_peek == '(': + ps.skip_to_peek() arguments = self.get_call_arguments(ps) return ast.TermReference(id, attribute, arguments) if ps.is_identifier_start(): id = self.get_identifier(ps) + ps.peek_blank() - if ps.current_char == '(': + if ps.current_peek == '(': # It's a Function. Ensure it's all upper-case. if not re.match('^[A-Z][A-Z0-9_-]*$', id.name): raise ParseError('E0008') + ps.skip_to_peek() args = self.get_call_arguments(ps) return ast.FunctionReference(id, args) diff --git a/fluent.syntax/tests/syntax/fixtures_reference/call_expressions.ftl b/fluent.syntax/tests/syntax/fixtures_reference/call_expressions.ftl index 98c908d9..77c2188a 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/call_expressions.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/call_expressions.ftl @@ -29,14 +29,15 @@ duplicate-named-args = {FUN(x: 1, x: "X")} ## Whitespace around arguments -sparse-inline-call = {FUN( "a" , msg, x: 1 )} +sparse-inline-call = {FUN ( "a" , msg, x: 1 )} empty-inline-call = {FUN( )} multiline-call = {FUN( "a", msg, x: 1 )} -sparse-multiline-call = {FUN( +sparse-multiline-call = {FUN + ( "a" , msg diff --git a/fluent.syntax/tests/syntax/fixtures_reference/term_parameters.ftl b/fluent.syntax/tests/syntax/fixtures_reference/term_parameters.ftl index 61442361..600c12c9 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/term_parameters.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/term_parameters.ftl @@ -3,6 +3,6 @@ } key01 = { -term } -key02 = { -term() } +key02 = { -term () } key03 = { -term(arg: 1) } key04 = { -term("positional", narg1: 1, narg2: 2) } From b43b4280ecd77b832c19edef14ab24a0ba7dfe18 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Thu, 5 Sep 2019 14:49:09 +0200 Subject: [PATCH 2/2] Uplift more complete reference test fixtures. This loops back to projectfluent/fluent#286. --- .../syntax/fixtures_reference/comments.ftl | 7 +- .../syntax/fixtures_reference/comments.json | 19 +++++ .../fixtures_reference/escaped_characters.ftl | 3 + .../escaped_characters.json | 11 ++- .../syntax/fixtures_reference/messages.ftl | 6 ++ .../syntax/fixtures_reference/messages.json | 70 ++++++++++++++++++ .../fixtures_reference/select_expressions.ftl | 9 ++- .../select_expressions.json | 58 ++++++++++++++- .../fixtures_reference/select_indent.ftl | 1 + .../fixtures_reference/select_indent.json | 17 +++++ .../tests/syntax/fixtures_reference/terms.ftl | 6 ++ .../syntax/fixtures_reference/terms.json | 72 ++++++++++++++++++- .../syntax/fixtures_reference/zero_length.ftl | 0 .../fixtures_reference/zero_length.json | 4 ++ 14 files changed, 276 insertions(+), 7 deletions(-) create mode 100644 fluent.syntax/tests/syntax/fixtures_reference/zero_length.ftl create mode 100644 fluent.syntax/tests/syntax/fixtures_reference/zero_length.json diff --git a/fluent.syntax/tests/syntax/fixtures_reference/comments.ftl b/fluent.syntax/tests/syntax/fixtures_reference/comments.ftl index cc3246ea..d3566930 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/comments.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/comments.ftl @@ -9,7 +9,12 @@ foo = Foo -term = Term # Another standalone -# +# # with indent ## Group Comment ### Resource Comment + +# Errors +#error +##error +###error diff --git a/fluent.syntax/tests/syntax/fixtures_reference/comments.json b/fluent.syntax/tests/syntax/fixtures_reference/comments.json index c28115ac..073886a5 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/comments.json +++ b/fluent.syntax/tests/syntax/fixtures_reference/comments.json @@ -58,6 +58,25 @@ { "type": "ResourceComment", "content": "Resource Comment" + }, + { + "type": "Comment", + "content": "Errors" + }, + { + "type": "Junk", + "annotations": [], + "content": "#error\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "##error\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "###error\n" } ] } diff --git a/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.ftl b/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.ftl index ec862320..fccf2572 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.ftl @@ -12,6 +12,9 @@ backslash-in-string = {"\\"} mismatched-quote = {"\\""} # ERROR Unknown escape unknown-escape = {"\x"} +# ERROR Multiline literal +invalid-multiline-literal = {" + "} ## Unicode escapes string-unicode-4digits = {"\u0041"} diff --git a/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.json b/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.json index 3676d8d6..2c18ffce 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.json +++ b/fluent.syntax/tests/syntax/fixtures_reference/escaped_characters.json @@ -168,7 +168,16 @@ { "type": "Junk", "annotations": [], - "content": "unknown-escape = {\"\\x\"}\n\n" + "content": "unknown-escape = {\"\\x\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Multiline literal" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-multiline-literal = {\"\n \"}\n\n" }, { "type": "GroupComment", diff --git a/fluent.syntax/tests/syntax/fixtures_reference/messages.ftl b/fluent.syntax/tests/syntax/fixtures_reference/messages.ftl index a8cd0896..dbc616ee 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/messages.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/messages.ftl @@ -18,6 +18,12 @@ key04 = key05 = .attr1 = Attribute 1 +no-whitespace=Value + .attr1=Attribute 1 + +extra-whitespace = Value + .attr1 = Attribute 1 + key06 = {""} # JUNK Missing value diff --git a/fluent.syntax/tests/syntax/fixtures_reference/messages.json b/fluent.syntax/tests/syntax/fixtures_reference/messages.json index f78a316a..5f3ead4d 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/messages.json +++ b/fluent.syntax/tests/syntax/fixtures_reference/messages.json @@ -205,6 +205,76 @@ "content": " < whitespace >" } }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "no-whitespace" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "extra-whitespace" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + } + ], + "comment": null + }, { "type": "Message", "id": { diff --git a/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.ftl b/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.ftl index c7742d52..603a1226 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.ftl @@ -22,13 +22,13 @@ invalid-selector-term-variant = } # ERROR Nested expressions are not valid selectors -invalid-selector-select-expression = +invalid-selector-nested-expression = { { 3 } -> *[key] default } # ERROR Select expressions are not valid selectors -invalid-selector-nested-expression = +invalid-selector-select-expression = { { $sel -> *[key] value } -> @@ -40,6 +40,11 @@ empty-variant = *[key] {""} } +reduced-whitespace = + {FOO()-> + *[key] {""} + } + nested-select = { $sel -> *[one] { $sel -> diff --git a/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.json b/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.json index 737da34f..4715b83e 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.json +++ b/fluent.syntax/tests/syntax/fixtures_reference/select_expressions.json @@ -152,7 +152,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-select-expression =\n { { 3 } ->\n *[key] default\n }\n\n" + "content": "invalid-selector-nested-expression =\n { { 3 } ->\n *[key] default\n }\n\n" }, { "type": "Comment", @@ -161,7 +161,7 @@ { "type": "Junk", "annotations": [], - "content": "invalid-selector-nested-expression =\n { { $sel ->\n *[key] value\n } ->\n *[key] default\n }\n\n" + "content": "invalid-selector-select-expression =\n { { $sel ->\n *[key] value\n } ->\n *[key] default\n }\n\n" }, { "type": "Message", @@ -212,6 +212,60 @@ "attributes": [], "comment": null }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "reduced-whitespace" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FOO" + }, + "arguments": { + "type": "CallArguments", + "positional": [], + "named": [] + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "value": "", + "type": "StringLiteral" + } + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, { "type": "Message", "id": { diff --git a/fluent.syntax/tests/syntax/fixtures_reference/select_indent.ftl b/fluent.syntax/tests/syntax/fixtures_reference/select_indent.ftl index 6c13076b..7455e935 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/select_indent.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/select_indent.ftl @@ -15,6 +15,7 @@ select-1tbs-indent = { select-allman-inline = { $selector -> *[key] Value + [other] Other } select-allman-newline = diff --git a/fluent.syntax/tests/syntax/fixtures_reference/select_indent.json b/fluent.syntax/tests/syntax/fixtures_reference/select_indent.json index d8c0fa88..75f3f16f 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/select_indent.json +++ b/fluent.syntax/tests/syntax/fixtures_reference/select_indent.json @@ -176,6 +176,23 @@ ] }, "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false } ] } diff --git a/fluent.syntax/tests/syntax/fixtures_reference/terms.ftl b/fluent.syntax/tests/syntax/fixtures_reference/terms.ftl index b8791fcf..893188d4 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/terms.ftl +++ b/fluent.syntax/tests/syntax/fixtures_reference/terms.ftl @@ -21,3 +21,9 @@ # JUNK Missing = -term07 + +-term08=Value + .attr=Attribute + +-term09 = Value + .attr = Attribute diff --git a/fluent.syntax/tests/syntax/fixtures_reference/terms.json b/fluent.syntax/tests/syntax/fixtures_reference/terms.json index 0c812198..330eb138 100644 --- a/fluent.syntax/tests/syntax/fixtures_reference/terms.json +++ b/fluent.syntax/tests/syntax/fixtures_reference/terms.json @@ -100,7 +100,77 @@ { "type": "Junk", "annotations": [], - "content": "-term07\n" + "content": "-term07\n\n" + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term08" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null } ] } diff --git a/fluent.syntax/tests/syntax/fixtures_reference/zero_length.ftl b/fluent.syntax/tests/syntax/fixtures_reference/zero_length.ftl new file mode 100644 index 00000000..e69de29b diff --git a/fluent.syntax/tests/syntax/fixtures_reference/zero_length.json b/fluent.syntax/tests/syntax/fixtures_reference/zero_length.json new file mode 100644 index 00000000..b1992785 --- /dev/null +++ b/fluent.syntax/tests/syntax/fixtures_reference/zero_length.json @@ -0,0 +1,4 @@ +{ + "type": "Resource", + "body": [] +}