From 787db45674520bf78032878428d3e43c2e27a0e8 Mon Sep 17 00:00:00 2001 From: unclenachoduh Date: Thu, 9 Aug 2018 16:07:33 -0700 Subject: [PATCH 1/2] Replace VariantName with Identifier --- fluent/syntax/ast.py | 6 +++--- fluent/syntax/ftlstream.py | 20 ++++++++++---------- fluent/syntax/parser.py | 26 +++++++++++++------------- fluent/syntax/serializer.py | 8 ++++---- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/fluent/syntax/ast.py b/fluent/syntax/ast.py index f2e94806..6f609103 100644 --- a/fluent/syntax/ast.py +++ b/fluent/syntax/ast.py @@ -303,9 +303,9 @@ def __init__(self, name, **kwargs): self.name = name -class VariantName(Identifier): - def __init__(self, name, **kwargs): - super(VariantName, self).__init__(name, **kwargs) +# class VariantName(Identifier): +# def __init__(self, name, **kwargs): +# super(VariantName, self).__init__(name, **kwargs) class BaseComment(Entry): diff --git a/fluent/syntax/ftlstream.py b/fluent/syntax/ftlstream.py index a6c73357..1274da5d 100644 --- a/fluent/syntax/ftlstream.py +++ b/fluent/syntax/ftlstream.py @@ -268,16 +268,16 @@ def closure(ch): cc == 95 or cc == 45) return self.take_char(closure) - def take_variant_name_char(self): - def closure(ch): - if ch is None: - return False - cc = ord(ch) - return (cc >= 97 and cc <= 122) or \ - (cc >= 65 and cc <= 90) or \ - (cc >= 48 and cc <= 57) or \ - cc == 95 or cc == 45 or cc == 32 - return self.take_char(closure) + # def take_variant_name_char(self): + # def closure(ch): + # if ch is None: + # return False + # cc = ord(ch) + # return (cc >= 97 and cc <= 122) or \ + # (cc >= 65 and cc <= 90) or \ + # (cc >= 48 and cc <= 57) or \ + # cc == 95 or cc == 45 or cc == 32 + # return self.take_char(closure) def take_digit(self): def closure(ch): diff --git a/fluent/syntax/parser.py b/fluent/syntax/parser.py index a658b098..2484cac5 100644 --- a/fluent/syntax/parser.py +++ b/fluent/syntax/parser.py @@ -222,7 +222,7 @@ def get_group_comment_from_section(self, ps): ps.skip_inline_ws() - self.get_variant_name(ps) + self.get_identifier(ps) ps.skip_inline_ws() @@ -334,7 +334,7 @@ def get_variant_key(self, ps): if ((cc >= 48 and cc <= 57) or cc == 45): # 0-9, - return self.get_number(ps) - return self.get_variant_name(ps) + return self.get_identifier(ps) @with_span def get_variant(self, ps, has_default): @@ -380,17 +380,17 @@ def get_variants(self, ps): return variants - @with_span - def get_variant_name(self, ps): - name = ps.take_id_start() - while True: - ch = ps.take_variant_name_char() - if ch: - name += ch - else: - break - - return ast.VariantName(name.rstrip(' \t\n\r')) + # @with_span + # def get_variant_name(self, ps): + # name = ps.take_id_start() + # while True: + # ch = ps.take_variant_name_char() + # if ch: + # name += ch + # else: + # break + + # return ast.VariantName(name.rstrip(' \t\n\r')) def get_digits(self, ps): num = '' diff --git a/fluent/syntax/serializer.py b/fluent/syntax/serializer.py index 55618891..04e7bfb0 100644 --- a/fluent/syntax/serializer.py +++ b/fluent/syntax/serializer.py @@ -271,13 +271,13 @@ def serialize_identifier(identifier): return identifier.name -def serialize_variant_name(symbol): - return symbol.name +# def serialize_variant_name(symbol): +# return symbol.name def serialize_variant_key(key): - if isinstance(key, ast.VariantName): - return serialize_variant_name(key) + if isinstance(key, ast.Identifier): + return serialize_identifier(key) if isinstance(key, ast.NumberLiteral): return serialize_number_literal(key) raise Exception('Unknown variant key type: {}'.format(type(key))) From 6ddfbf14a26b585411ba57df51733ab986ccd827 Mon Sep 17 00:00:00 2001 From: unclenachoduh Date: Thu, 9 Aug 2018 22:16:04 -0700 Subject: [PATCH 2/2] Fix tests for variant key --- fluent/syntax/ast.py | 4 ++-- fluent/syntax/parser.py | 2 +- .../section_with_nl_in_the_middle.ftl | 2 +- .../section_with_no_nl_after_it.ftl | 5 ++++- .../section_with_one_bracket_at_the_end.ftl | 2 +- .../variant_with_symbol_with_space.ftl | 3 ++- .../fixtures_structure/leading_dots.json | 4 ++-- tests/syntax/fixtures_structure/section.json | 20 +++++++++---------- .../fixtures_structure/sparse-messages.json | 8 ++++---- tests/syntax/fixtures_structure/term.json | 12 +++++------ .../variant_with_empty_pattern.json | 2 +- tests/syntax/test_serializer.py | 16 +++++++-------- 12 files changed, 41 insertions(+), 39 deletions(-) diff --git a/fluent/syntax/ast.py b/fluent/syntax/ast.py index 6f609103..a50d0ef9 100644 --- a/fluent/syntax/ast.py +++ b/fluent/syntax/ast.py @@ -303,9 +303,9 @@ def __init__(self, name, **kwargs): self.name = name -# class VariantName(Identifier): +# class Identifier(Identifier): # def __init__(self, name, **kwargs): -# super(VariantName, self).__init__(name, **kwargs) +# super(Identifier, self).__init__(name, **kwargs) class BaseComment(Entry): diff --git a/fluent/syntax/parser.py b/fluent/syntax/parser.py index 2484cac5..11ebc40d 100644 --- a/fluent/syntax/parser.py +++ b/fluent/syntax/parser.py @@ -390,7 +390,7 @@ def get_variants(self, ps): # else: # break - # return ast.VariantName(name.rstrip(' \t\n\r')) + # return ast.Identifier(name.rstrip(' \t\n\r')) def get_digits(self, ps): num = '' diff --git a/tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl b/tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl index 899ef512..eaccec21 100644 --- a/tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl +++ b/tests/syntax/fixtures_behavior/section_with_nl_in_the_middle.ftl @@ -1,4 +1,4 @@ [[ This is a broken section]] -# ~ERROR E0003, pos 10, args "]" +# ~ERROR E0003, pos 8, args "]" # ~ERROR E0005, pos 13, args "a" diff --git a/tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl b/tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl index f11f4c06..2b801472 100644 --- a/tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl +++ b/tests/syntax/fixtures_behavior/section_with_no_nl_after_it.ftl @@ -1 +1,4 @@ -[[ This is a correct section ]] +# ~ERROR E0003, pos 8, args "]" +# ~ERROR E0005, pos 13, args "a" +[[ This is +a broken section]] \ No newline at end of file diff --git a/tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl b/tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl index 22ccf6fe..f6557bbc 100644 --- a/tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl +++ b/tests/syntax/fixtures_behavior/section_with_one_bracket_at_the_end.ftl @@ -1,2 +1,2 @@ [[ This is a broken section ] -# ~ERROR E0003, pos 29, args "]" +# ~ERROR E0003, pos 8, args "]" diff --git a/tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl b/tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl index a3a630c1..53e649ec 100644 --- a/tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl +++ b/tests/syntax/fixtures_behavior/variant_with_symbol_with_space.ftl @@ -1,3 +1,4 @@ +# ~ERROR E0003, pos 23, args "]" -term = { *[New York] Nowy Jork - } + } \ No newline at end of file diff --git a/tests/syntax/fixtures_structure/leading_dots.json b/tests/syntax/fixtures_structure/leading_dots.json index 34a19f38..837427b4 100644 --- a/tests/syntax/fixtures_structure/leading_dots.json +++ b/tests/syntax/fixtures_structure/leading_dots.json @@ -745,7 +745,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "one", "span": { "type": "Span", @@ -782,7 +782,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "other", "span": { "type": "Span", diff --git a/tests/syntax/fixtures_structure/section.json b/tests/syntax/fixtures_structure/section.json index 989f27f3..61fad1a5 100644 --- a/tests/syntax/fixtures_structure/section.json +++ b/tests/syntax/fixtures_structure/section.json @@ -1,16 +1,14 @@ { + "body": [{ + "annotations": [{"args": ["]"], + "code": "E0003", + "message": "Expected token: \"]\"", + "span": {"end": 9, "start": 9, "type": "Span"}, + "type": "Annotation"}], + "content": "[[ This is a section ]]\n", + "span": {"end": 25, "start": 1, "type": "Span"}, + "type": "Junk"}], "type": "Resource", - "body": [ - { - "type": "GroupComment", - "content": "", - "span": { - "type": "Span", - "start": 1, - "end": 24 - } - } - ], "span": { "type": "Span", "start": 0, diff --git a/tests/syntax/fixtures_structure/sparse-messages.json b/tests/syntax/fixtures_structure/sparse-messages.json index b58ccaa3..90f8fdb3 100644 --- a/tests/syntax/fixtures_structure/sparse-messages.json +++ b/tests/syntax/fixtures_structure/sparse-messages.json @@ -248,7 +248,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "one", "span": { "type": "Span", @@ -285,7 +285,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "two", "span": { "type": "Span", @@ -364,7 +364,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "one", "span": { "type": "Span", @@ -430,7 +430,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "one", "span": { "type": "Span", diff --git a/tests/syntax/fixtures_structure/term.json b/tests/syntax/fixtures_structure/term.json index 4250e4c6..f9cf94d4 100644 --- a/tests/syntax/fixtures_structure/term.json +++ b/tests/syntax/fixtures_structure/term.json @@ -18,7 +18,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "nominative", "span": { "type": "Span", @@ -55,7 +55,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "accusative", "span": { "type": "Span", @@ -186,7 +186,7 @@ } }, "key": { - "type": "VariantName", + "type": "Identifier", "name": "accusative", "span": { "type": "Span", @@ -286,7 +286,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "masculine", "span": { "type": "Span", @@ -348,7 +348,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "feminine", "span": { "type": "Span", @@ -410,7 +410,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "other", "span": { "type": "Span", diff --git a/tests/syntax/fixtures_structure/variant_with_empty_pattern.json b/tests/syntax/fixtures_structure/variant_with_empty_pattern.json index 007f49f8..19fe137d 100644 --- a/tests/syntax/fixtures_structure/variant_with_empty_pattern.json +++ b/tests/syntax/fixtures_structure/variant_with_empty_pattern.json @@ -32,7 +32,7 @@ { "type": "Variant", "key": { - "type": "VariantName", + "type": "Identifier", "name": "one", "span": { "type": "Span", diff --git a/tests/syntax/test_serializer.py b/tests/syntax/test_serializer.py index 66a7fd58..68829f16 100644 --- a/tests/syntax/test_serializer.py +++ b/tests/syntax/test_serializer.py @@ -269,14 +269,14 @@ def test_variant_multiline_first_inline(self): """ self.assertEqual(self.pretty_ftl(input), dedent_ftl(output)) - def test_variant_key_words(self): - input = """\ - foo = - { $sel -> - *[a b c] A B C - } - """ - self.assertEqual(self.pretty_ftl(input), dedent_ftl(input)) + # def test_variant_key_words(self): + # input = """\ + # foo = + # { $sel -> + # *[a b c] A B C + # } + # """ + # self.assertEqual(self.pretty_ftl(input), dedent_ftl(input)) def test_variant_key_number(self): input = """\