From 9ea274ff8d3dcaae6610d7bd2fd0e1f550ce74ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 21 Nov 2018 18:25:20 +0100 Subject: [PATCH 1/6] Dedent fluent-syntax/src --- eslint_src.json | 1 - fluent-syntax/src/parser.js | 223 +++++++++++++++++++++++------------- fluent-syntax/src/stream.js | 110 ++++++------------ 3 files changed, 181 insertions(+), 153 deletions(-) diff --git a/eslint_src.json b/eslint_src.json index 328febcee..dc1c72151 100644 --- a/eslint_src.json +++ b/eslint_src.json @@ -48,7 +48,6 @@ "no-extend-native": 2, "no-global-assign": 2, "no-extra-bind": 2, - "no-redeclare": 2, "array-bracket-spacing": 2, "brace-style": [ 1, diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 1f8651a66..5176772d4 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -68,7 +68,9 @@ export default class FluentParser { // they should parse as standalone when they're followed by Junk. // Consequently, we only attach Comments once we know that the Message // or the Term parsed successfully. - if (entry.type === "Comment" && blankLines === 0 && ps.currentChar) { + if (entry.type === "Comment" + && blankLines.length === 0 + && ps.currentChar) { // Stash the comment and decide what to do with it in the next pass. lastComment = entry; continue; @@ -200,7 +202,7 @@ export default class FluentParser { } } - if (ps.isNextLineComment(level, {skip: false})) { + if (ps.isNextLineComment(level)) { content += ps.currentChar; ps.next(); } else { @@ -229,19 +231,14 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - var pattern = this.getPattern(ps); - } - - if (ps.isNextLineAttributeStart({skip: true})) { - var attrs = this.getAttributes(ps); - } + const value = this.getInlineOrBlock(ps, this.getPattern); + const attrs = this.getAttributes(ps); - if (pattern === undefined && attrs === undefined) { + if (value === null && attrs.length === 0) { throw new ParseError("E0005", id.name); } - return new AST.Message(id, pattern, attrs); + return new AST.Message(id, value, attrs); } getTerm(ps) { @@ -251,17 +248,29 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - var value = this.getValue(ps); - } else { + const value = this.getInlineOrBlock(ps, this.getValue); + if (value === null) { throw new ParseError("E0006", id.name); } - if (ps.isNextLineAttributeStart({skip: true})) { - var attrs = this.getAttributes(ps); + const attrs = this.getAttributes(ps); + return new AST.Term(id, value, attrs); + } + + getInlineOrBlock(ps, method) { + ps.peekBlankInline(); + if (ps.isValueStart()) { + ps.skipToPeek(); + return method.call(this, ps, {isBlock: false}); } - return new AST.Term(id, value, attrs); + ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + return method.call(this, ps, {isBlock: true}); + } + + return null; } getAttribute(ps) { @@ -272,24 +281,22 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - if (ps.isValueStart({skip: true})) { - const value = this.getPattern(ps); - return new AST.Attribute(key, value); + const value = this.getInlineOrBlock(ps, this.getPattern); + if (value === null) { + throw new ParseError("E0012"); } - throw new ParseError("E0012"); + return new AST.Attribute(key, value); } getAttributes(ps) { const attrs = []; - - while (true) { + ps.peekBlank(); + while (ps.isAttributeStart()) { + ps.skipToPeek(); const attr = this.getAttribute(ps); attrs.push(attr); - - if (!ps.isNextLineAttributeStart({skip: true})) { - break; - } + ps.peekBlank(); } return attrs; } @@ -340,22 +347,22 @@ export default class FluentParser { const key = this.getVariantKey(ps); ps.skipBlank(); - ps.expectChar("]"); - if (ps.isValueStart({skip: true})) { - const value = this.getValue(ps); - return new AST.Variant(key, value, defaultIndex); + const value = this.getInlineOrBlock(ps, this.getValue); + if (value === null) { + throw new ParseError("E0012"); } - throw new ParseError("E0012"); + return new AST.Variant(key, value, defaultIndex); } getVariants(ps) { const variants = []; let hasDefault = false; - while (true) { + ps.skipBlank(); + while (ps.isVariantStart()) { const variant = this.getVariant(ps, hasDefault); if (variant.default) { @@ -363,13 +370,14 @@ export default class FluentParser { } variants.push(variant); - - if (!ps.isNextLineVariantStart({skip: false})) { - break; - } + ps.expectLineEnd(); ps.skipBlank(); } + if (variants.length === 0) { + throw new ParseError("E0011"); + } + if (!hasDefault) { throw new ParseError("E0010"); } @@ -411,65 +419,135 @@ export default class FluentParser { return new AST.NumberLiteral(num); } - getValue(ps) { - if (ps.currentChar === "{") { + getValue(ps, {isBlock}) { + ps.peekBlankInline(); + const peekOffset = ps.peekOffset; + if (ps.currentPeek === "{") { ps.peek(); - ps.peekBlankInline(); - if (ps.isNextLineVariantStart({skip: false})) { + ps.peekBlank(); + if (ps.isVariantStart()) { + ps.resetPeek(peekOffset); + ps.skipToPeek(); return this.getVariantList(ps); } - - ps.resetPeek(); } - return this.getPattern(ps); + ps.resetPeek(); + return this.getPattern(ps, {isBlock}); } getVariantList(ps) { ps.expectChar("{"); ps.skipBlankInline(); ps.expectLineEnd(); - ps.skipBlank(); const variants = this.getVariants(ps); - ps.expectLineEnd(); ps.skipBlank(); ps.expectChar("}"); return new AST.VariantList(variants); } - getPattern(ps) { + getPattern(ps, {isBlock}) { const elements = []; + if (isBlock) { + const blankStart = ps.index; + const firstIndent = ps.skipBlankInline(); + elements.push(this.getIndent(ps, firstIndent, blankStart)); + var commonIndentLength = firstIndent.length; + } else { + var commonIndentLength = Infinity; + } let ch; - while ((ch = ps.currentChar)) { + elements: while ((ch = ps.currentChar)) { + switch (ch) { + case EOL: { + const blankStart = ps.index; + const blankLines = ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + const indent = ps.skipBlankInline(); + commonIndentLength = Math.min(commonIndentLength, indent.length); + elements.push(this.getIndent(ps, blankLines + indent, blankStart)); + continue elements; + } - // The end condition for getPattern's while loop is a newline - // which is not followed by a valid pattern continuation. - if (ch === EOL && !ps.isNextLineValue({skip: false})) { - break; + // The end condition for getPattern's while loop is a newline + // which is not followed by a valid pattern continuation. + ps.resetPeek(); + break elements; + } + case "{": + elements.push(this.getPlaceable(ps)); + continue elements; + case "}": + throw new ParseError("E0027"); + default: + const element = this.getTextElement(ps); + elements.push(element); } + } - if (ch === "{") { - const element = this.getPlaceable(ps); - elements.push(element); - } else if (ch === "}") { - throw new ParseError("E0027"); - } else { - const element = this.getTextElement(ps); - elements.push(element); + return new AST.Pattern(this.dedent(elements, commonIndentLength)); + } + + getIndent(ps, value, start) { + return { + type: "Indent", + span: {start, end: ps.index}, + value, + }; + } + + dedent(elements, commonIndent) { + const trimmed = []; + + for (let element of elements) { + if (element.type === "Placeable") { + trimmed.push(element); + continue; + } + + if (element.type === "Indent") { + // Strip common indent. + element.value = element.value.slice( + 0, element.value.length - commonIndent); + if (element.value.length === 0) { + continue; + } + } + + let prev = trimmed[trimmed.length - 1]; + if (prev && prev.type === "TextElement") { + // Join adjacent TextElements by replacing them with their sum. + const sum = new AST.TextElement(prev.value + element.value); + if (this.withSpans) { + sum.addSpan(prev.span.start, element.span.end); + } + trimmed[trimmed.length - 1] = sum; + continue; + } + + if (element.type === "Indent") { + const textElement = new AST.TextElement(element.value); + if (this.withSpans) { + textElement.addSpan(element.span.start, element.span.end); + } + element = textElement; } + + trimmed.push(element); } // Trim trailing whitespace. - const lastElement = elements[elements.length - 1]; + const lastElement = trimmed[trimmed.length - 1]; if (lastElement.type === "TextElement") { lastElement.value = lastElement.value.replace(trailingWSRe, ""); - if (lastElement.value === "") { - elements.pop(); + if (lastElement.value.length === 0) { + trimmed.pop(); } } - return new AST.Pattern(elements); + return trimmed; } getTextElement(ps) { @@ -482,15 +560,7 @@ export default class FluentParser { } if (ch === EOL) { - if (!ps.isNextLineValue({skip: false})) { - return new AST.TextElement(buffer); - } - - ps.next(); - ps.skipBlankInline(); - - buffer += EOL; - continue; + return new AST.TextElement(buffer); } buffer += ch; @@ -567,14 +637,8 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectLineEnd(); - ps.skipBlank(); const variants = this.getVariants(ps); - ps.skipBlank(); - - if (variants.length === 0) { - throw new ParseError("E0011"); - } // VariantLists are only allowed in other VariantLists. if (variants.some(v => v.value.type === "VariantList")) { @@ -596,6 +660,7 @@ export default class FluentParser { if (ps.currentChar === "{") { return this.getPlaceable(ps); } + const literal = this.getLiteral(ps); if (literal.type !== "MessageReference" diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js index c95190aa5..352b2031b 100644 --- a/fluent-syntax/src/stream.js +++ b/fluent-syntax/src/stream.js @@ -68,9 +68,12 @@ const SPECIAL_LINE_START_CHARS = ["}", ".", "[", "*"]; export class FluentParserStream extends ParserStream { skipBlankInline() { + let blank = ""; while (this.currentChar === " ") { + blank += " "; this.next(); } + return blank; } peekBlankInline() { @@ -80,40 +83,40 @@ export class FluentParserStream extends ParserStream { } skipBlankBlock() { - let lineCount = 0; + let blank = ""; while (true) { this.peekBlankInline(); if (this.currentPeek === EOL) { this.skipToPeek(); this.next(); - lineCount++; + blank += EOL; continue; } if (this.currentPeek === EOF) { // Consume any inline blanks before the EOF. this.skipToPeek(); - return lineCount; + return blank; } // Any other char; reset to column 1 on this line. this.resetPeek(); - return lineCount; + return blank; } } peekBlankBlock() { + let blank = ""; while (true) { const lineStart = this.peekOffset; - this.peekBlankInline(); - if (this.currentPeek === EOL) { this.peek(); + blank += EOL; } else { this.resetPeek(lineStart); - break; + return blank; } } } @@ -204,29 +207,39 @@ export class FluentParserStream extends ParserStream { return !includes(SPECIAL_LINE_START_CHARS, ch); } - isValueStart({skip = true}) { - if (skip === false) throw new Error("Unimplemented"); + isValueStart() { + // Inline Patterns may start with any char. + const ch = this.currentPeek; + return ch !== EOL && ch !== EOF; + } + isValueContinuation() { + const column1 = this.peekOffset; this.peekBlankInline(); - const ch = this.currentPeek; - // Inline Patterns may start with any char. - if (ch !== EOF && ch !== EOL) { - this.skipToPeek(); + if (this.currentPeek === "{") { + this.resetPeek(column1); + return true; + } + + if (this.peekOffset - column1 === 0) { + return false; + } + + if (this.isCharPatternContinuation(this.currentPeek)) { + this.resetPeek(column1); return true; } - return this.isNextLineValue({skip}); + return false; } // -1 - any // 0 - comment // 1 - group comment // 2 - resource comment - isNextLineComment(level = -1, {skip = false}) { - if (skip === true) throw new Error("Unimplemented"); - - if (this.currentPeek !== EOL) { + isNextLineComment(level = -1) { + if (this.currentChar !== EOL) { return false; } @@ -254,70 +267,21 @@ export class FluentParserStream extends ParserStream { return false; } - isNextLineVariantStart({skip = false}) { - if (skip === true) throw new Error("Unimplemented"); - - if (this.currentPeek !== EOL) { - return false; - } - - this.peekBlank(); - + isVariantStart() { + const currentPeekOffset = this.peekOffset; if (this.currentPeek === "*") { this.peek(); } - if (this.currentPeek === "[") { - this.resetPeek(); + this.resetPeek(currentPeekOffset); return true; } - this.resetPeek(); + this.resetPeek(currentPeekOffset); return false; } - isNextLineAttributeStart({skip = true}) { - if (skip === false) throw new Error("Unimplemented"); - - this.peekBlank(); - - if (this.currentPeek === ".") { - this.skipToPeek(); - return true; - } - - this.resetPeek(); - return false; - } - - isNextLineValue({skip = true}) { - if (this.currentPeek !== EOL) { - return false; - } - - this.peekBlankBlock(); - - const ptr = this.peekOffset; - - this.peekBlankInline(); - - if (this.currentPeek !== "{") { - if (this.peekOffset - ptr === 0) { - this.resetPeek(); - return false; - } - - if (!this.isCharPatternContinuation(this.currentPeek)) { - this.resetPeek(); - return false; - } - } - - if (skip) { - this.skipToPeek(); - } else { - this.resetPeek(); - } - return true; + isAttributeStart() { + return this.currentPeek === "."; } skipToNextEntryStart(junkStart) { From 6ec13ab4ad6776a6942f211f93ad8bfb10af05ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 21 Nov 2018 18:25:35 +0100 Subject: [PATCH 2/6] Dedent fluent-syntax/test --- .../select_expression_without_variants.ftl | 2 +- .../selector_expression_ends_abruptly.ftl | 2 +- .../fixtures_reference/multiline_values.ftl | 25 +++++ .../fixtures_reference/multiline_values.json | 106 +++++++++++++++++- .../test/fixtures_structure/blank_lines.json | 6 +- .../test/fixtures_structure/crlf.json | 2 +- .../test/fixtures_structure/leading_dots.json | 12 +- .../fixtures_structure/multiline_pattern.json | 6 +- .../fixtures_structure/placeable_at_eol.json | 4 +- .../fixtures_structure/sparse-messages.json | 4 +- .../test/fixtures_structure/term.json | 4 +- .../variant_with_empty_pattern.json | 2 +- 12 files changed, 152 insertions(+), 23 deletions(-) diff --git a/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl index 7e43355d1..8eea7b423 100644 --- a/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl +++ b/fluent-syntax/test/fixtures_behavior/select_expression_without_variants.ftl @@ -3,4 +3,4 @@ key = { $foo -> } key = { $foo -> } -# ~ERROR E0003, pos 39, args "[" +# ~ERROR E0011, pos 39 diff --git a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl index a11a18144..8cb339853 100644 --- a/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl +++ b/fluent-syntax/test/fixtures_behavior/selector_expression_ends_abruptly.ftl @@ -1,2 +1,2 @@ key = { $foo -> -# ~ERROR E0003, pos 16, args "[" +# ~ERROR E0011, pos 16 diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.ftl b/fluent-syntax/test/fixtures_reference/multiline_values.ftl index 4dd811548..e3739bb5c 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.ftl +++ b/fluent-syntax/test/fixtures_reference/multiline_values.ftl @@ -33,3 +33,28 @@ key07 = {"A multiline value"} starting and ending {"with a placeable"} key08 = Leading and trailing whitespace. + +key09 = zero + three + two + one + zero + +key10 = + two + zero + four + +key11 = + + + two + zero + +key12 = +{"."} + four + +key13 = + four +{"."} diff --git a/fluent-syntax/test/fixtures_reference/multiline_values.json b/fluent-syntax/test/fixtures_reference/multiline_values.json index 645a09b38..17fb6f8e4 100644 --- a/fluent-syntax/test/fixtures_reference/multiline_values.json +++ b/fluent-syntax/test/fixtures_reference/multiline_values.json @@ -102,7 +102,7 @@ "elements": [ { "type": "TextElement", - "value": "A multiline value with non-standard\n\nindentation." + "value": "A multiline value with non-standard\n\n indentation." } ] }, @@ -212,6 +212,110 @@ }, "attributes": [], "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "zero\n three\n two\n one\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + }, + { + "type": "TextElement", + "value": "\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " four\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null } ] } diff --git a/fluent-syntax/test/fixtures_structure/blank_lines.json b/fluent-syntax/test/fixtures_structure/blank_lines.json index 14fab74f6..998e7da64 100644 --- a/fluent-syntax/test/fixtures_structure/blank_lines.json +++ b/fluent-syntax/test/fixtures_structure/blank_lines.json @@ -112,7 +112,7 @@ ], "span": { "type": "Span", - "start": 130, + "start": 126, "end": 153 } }, @@ -150,7 +150,7 @@ ], "span": { "type": "Span", - "start": 240, + "start": 236, "end": 267 } }, @@ -196,7 +196,7 @@ ], "span": { "type": "Span", - "start": 339, + "start": 335, "end": 347 } }, diff --git a/fluent-syntax/test/fixtures_structure/crlf.json b/fluent-syntax/test/fixtures_structure/crlf.json index b72afa33f..9461bfaa2 100644 --- a/fluent-syntax/test/fixtures_structure/crlf.json +++ b/fluent-syntax/test/fixtures_structure/crlf.json @@ -65,7 +65,7 @@ ], "span": { "type": "Span", - "start": 35, + "start": 31, "end": 58 } }, diff --git a/fluent-syntax/test/fixtures_structure/leading_dots.json b/fluent-syntax/test/fixtures_structure/leading_dots.json index 837427b45..afb84b8d0 100644 --- a/fluent-syntax/test/fixtures_structure/leading_dots.json +++ b/fluent-syntax/test/fixtures_structure/leading_dots.json @@ -175,7 +175,7 @@ ], "span": { "type": "Span", - "start": 61, + "start": 57, "end": 71 } }, @@ -514,7 +514,7 @@ ], "span": { "type": "Span", - "start": 428, + "start": 424, "end": 484 } }, @@ -565,7 +565,7 @@ ], "span": { "type": "Span", - "start": 515, + "start": 511, "end": 516 } }, @@ -696,7 +696,7 @@ ], "span": { "type": "Span", - "start": 589, + "start": 580, "end": 599 } }, @@ -822,7 +822,7 @@ ], "span": { "type": "Span", - "start": 669, + "start": 657, "end": 679 } }, @@ -849,7 +849,7 @@ ], "span": { "type": "Span", - "start": 613, + "start": 609, "end": 685 } }, diff --git a/fluent-syntax/test/fixtures_structure/multiline_pattern.json b/fluent-syntax/test/fixtures_structure/multiline_pattern.json index 1793692b6..8f5c7fadd 100644 --- a/fluent-syntax/test/fixtures_structure/multiline_pattern.json +++ b/fluent-syntax/test/fixtures_structure/multiline_pattern.json @@ -65,7 +65,7 @@ ], "span": { "type": "Span", - "start": 47, + "start": 43, "end": 72 } }, @@ -103,7 +103,7 @@ ], "span": { "type": "Span", - "start": 171, + "start": 167, "end": 176 } }, @@ -173,7 +173,7 @@ ], "span": { "type": "Span", - "start": 311, + "start": 307, "end": 316 } }, diff --git a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json index f79d7454b..7018303eb 100644 --- a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json +++ b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json @@ -61,7 +61,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 131 } }, @@ -124,7 +124,7 @@ ], "span": { "type": "Span", - "start": 144, + "start": 140, "end": 184 } }, diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.json b/fluent-syntax/test/fixtures_structure/sparse-messages.json index bdeb917b5..d6a51ff07 100644 --- a/fluent-syntax/test/fixtures_structure/sparse-messages.json +++ b/fluent-syntax/test/fixtures_structure/sparse-messages.json @@ -27,7 +27,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 8, "end": 17 } }, @@ -122,7 +122,7 @@ ], "span": { "type": "Span", - "start": 63, + "start": 59, "end": 104 } }, diff --git a/fluent-syntax/test/fixtures_structure/term.json b/fluent-syntax/test/fixtures_structure/term.json index 6da90f3bb..aadfd9424 100644 --- a/fluent-syntax/test/fixtures_structure/term.json +++ b/fluent-syntax/test/fixtures_structure/term.json @@ -218,7 +218,7 @@ ], "span": { "type": "Span", - "start": 131, + "start": 127, "end": 171 } }, @@ -494,7 +494,7 @@ ], "span": { "type": "Span", - "start": 197, + "start": 193, "end": 437 } }, diff --git a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json index 19fe137d2..c75825177 100644 --- a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json @@ -90,7 +90,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 42 } }, From fd0f59e39bb0669822ce672ff07d8d73db482138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 21 Nov 2018 18:25:46 +0100 Subject: [PATCH 3/6] Dedent fluent/src --- fluent/src/resolver.js | 2 + fluent/src/resource.js | 88 +++++++++++++++++++++++++----------------- 2 files changed, 55 insertions(+), 35 deletions(-) diff --git a/fluent/src/resolver.js b/fluent/src/resolver.js index f2a216d25..b1a29a0d1 100644 --- a/fluent/src/resolver.js +++ b/fluent/src/resolver.js @@ -305,6 +305,8 @@ function Type(env, expr) { switch (expr.type) { + case "str": + return expr.value; case "num": return new FluentNumber(expr.value); case "var": diff --git a/fluent/src/resource.js b/fluent/src/resource.js index b457a38ec..d1b6421c4 100644 --- a/fluent/src/resource.js +++ b/fluent/src/resource.js @@ -27,11 +27,13 @@ const RE_STRING_RUN = /([^\\"\n\r]*)/y; const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})/y; const RE_STRING_ESCAPE = /\\([\\"])/y; -// Used for trimming TextElements and indents. With the /m flag, the $ matches -// the end of every line. -const RE_TRAILING_SPACES = / +$/mg; -// CRLFs are normalized to LF. -const RE_CRLF = /\r\n/g; +// Used for trimming TextElements and indents. +const RE_LEADING_NEWLINES = /^\n+/; +const RE_TRAILING_SPACES = / +$/; +// Used in makeIndent to strip spaces from blank lines and normalize CRLF to LF. +const RE_BLANK_LINES = / *\r?\n/g; +// Used in makeIndent to measure the indentation. +const RE_INDENT = /( *)$/; // Common tokens. const TOKEN_BRACE_OPEN = /{\s*/y; @@ -180,42 +182,41 @@ export default class FluentResource extends Map { // If there's a placeable on the first line, parse a complex pattern. if (source[cursor] === "{" || source[cursor] === "}") { - return first - // Re-use the text parsed above, if possible. - ? parsePatternElements(first) - : parsePatternElements(); + // Re-use the text parsed above, if possible. + return parsePatternElements(first ? [first] : [], Infinity); } // RE_TEXT_VALUE stops at newlines. Only continue parsing the pattern if // what comes after the newline is indented. let indent = parseIndent(); if (indent) { - return first + if (first) { // If there's text on the first line, the blank block is part of the - // translation content. - ? parsePatternElements(first, trim(indent)) - // Otherwise, we're dealing with a block pattern. The blank block is - // the leading whitespace; discard it. - : parsePatternElements(); + // translation content in its entirety. + return parsePatternElements([first, indent], indent.length); + } + // Otherwise, we're dealing with a block pattern, i.e. a pattern which + // starts on a new line. Discrad the leading newlines but keep the + // inline indent; it will be used by the dedentation logic. + indent.value = trim(indent.value, RE_LEADING_NEWLINES); + return parsePatternElements([indent], indent.length); } if (first) { // It was just a simple inline text after all. - return trim(first); + return trim(first, RE_TRAILING_SPACES); } return null; } // Parse a complex pattern as an array of elements. - function parsePatternElements(...elements) { + function parsePatternElements(elements = [], commonIndent) { let placeableCount = 0; - let needsTrimming = false; while (true) { if (test(RE_TEXT_RUN)) { elements.push(match(RE_TEXT_RUN)); - needsTrimming = true; continue; } @@ -224,7 +225,6 @@ export default class FluentResource extends Map { throw new FluentError("Too many placeables"); } elements.push(parsePlaceable()); - needsTrimming = false; continue; } @@ -234,23 +234,34 @@ export default class FluentResource extends Map { let indent = parseIndent(); if (indent) { - elements.push(trim(indent)); - needsTrimming = false; + elements.push(indent); + commonIndent = Math.min(commonIndent, indent.length); continue; } break; } - if (needsTrimming) { - // Trim the trailing whitespace of the last element if it's a - // TextElement. Use a flag rather than a typeof check to tell - // TextElements and StringLiterals apart (both are strings). - let lastIndex = elements.length - 1; - elements[lastIndex] = trim(elements[lastIndex]); + let lastIndex = elements.length - 1; + // Trim the trailing spaces in the last element if it's a TextElement. + if (typeof elements[lastIndex] === "string") { + elements[lastIndex] = trim(elements[lastIndex], RE_TRAILING_SPACES); } - return elements; + let baked = []; + for (let element of elements) { + if (element.type === "indent") { + // Dedent indented lines by the maximum common indent. + element = element.value.slice(0, element.value.length - commonIndent); + } else if (element.type === "str") { + // Optimize StringLiterals into their value. + element = element.value; + } + if (element) { + baked.push(element); + } + } + return baked; } function parsePlaceable() { @@ -401,7 +412,7 @@ export default class FluentResource extends Map { } if (consumeChar("\"")) { - return value; + return {type: "str", value}; } // We've reached an EOL of EOF. @@ -447,7 +458,7 @@ export default class FluentResource extends Map { case "{": // Placeables don't require indentation (in EBNF: block-placeable). // Continue the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // If the first character on the line is not one of the special characters @@ -456,7 +467,7 @@ export default class FluentResource extends Map { if (source[cursor - 1] === " ") { // It's an indented text character (in EBNF: indented-char). Continue // the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // A not-indented text character is likely the identifier of the next @@ -464,9 +475,16 @@ export default class FluentResource extends Map { return false; } - // Trim spaces trailing on every line of text. - function trim(text) { - return text.replace(RE_TRAILING_SPACES, ""); + // Trim blanks in text according to the given regex. + function trim(text, re) { + return text.replace(re, ""); + } + + // Normalize a blank block and extract the indent details. + function makeIndent(blank) { + let value = blank.replace(RE_BLANK_LINES, "\n"); + let length = RE_INDENT.exec(blank)[1].length; + return {type: "indent", value, length}; } } } From 45b2b75a3c6dd1ee2da0a91919095d78681c5e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 21 Nov 2018 18:25:55 +0100 Subject: [PATCH 4/6] Dedent fluent/test --- .../fixtures_reference/call_expressions.json | 60 +++++++++++++++---- fluent/test/fixtures_reference/cr.json | 1 + fluent/test/fixtures_reference/messages.json | 4 +- .../fixtures_reference/multiline_values.json | 40 ++++++++++++- .../select_expressions.json | 5 +- .../fixtures_reference/select_indent.json | 3 +- fluent/test/fixtures_reference/terms.json | 4 +- .../variant_with_empty_pattern.json | 4 +- .../whitespace_leading.json | 1 - 9 files changed, 92 insertions(+), 30 deletions(-) diff --git a/fluent/test/fixtures_reference/call_expressions.json b/fluent/test/fixtures_reference/call_expressions.json index 3e0fe30e5..431a25b5a 100644 --- a/fluent/test/fixtures_reference/call_expressions.json +++ b/fluent/test/fixtures_reference/call_expressions.json @@ -11,7 +11,10 @@ "type": "num", "value": "1" }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -38,7 +41,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -62,7 +68,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -79,7 +88,10 @@ "type": "num", "value": "1" }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -95,7 +107,10 @@ { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } } ] } @@ -120,11 +135,17 @@ "value": "1" } }, - "a", + { + "type": "str", + "value": "a" + }, { "type": "narg", "name": "y", - "value": "Y" + "value": { + "type": "str", + "value": "Y" + } }, { "type": "ref", @@ -152,7 +173,10 @@ { "type": "narg", "name": "x", - "value": "X" + "value": { + "type": "str", + "value": "X" + } } ] } @@ -165,7 +189,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -199,7 +226,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -223,7 +253,10 @@ "name": "FUN" }, "args": [ - "a", + { + "type": "str", + "value": "a" + }, { "type": "ref", "name": "msg" @@ -272,7 +305,10 @@ "name": "FUN" }, "args": [ - "a" + { + "type": "str", + "value": "a" + } ] } ], diff --git a/fluent/test/fixtures_reference/cr.json b/fluent/test/fixtures_reference/cr.json index 6c6f39082..88e772c02 100644 --- a/fluent/test/fixtures_reference/cr.json +++ b/fluent/test/fixtures_reference/cr.json @@ -3,6 +3,7 @@ "err02": "Value 02", "err03": { "value": [ + "\r\r", "Value 03", "\r", "Continued" diff --git a/fluent/test/fixtures_reference/messages.json b/fluent/test/fixtures_reference/messages.json index 49d97ae73..03f07b796 100644 --- a/fluent/test/fixtures_reference/messages.json +++ b/fluent/test/fixtures_reference/messages.json @@ -26,7 +26,5 @@ "attr1": "Attribute 1" } }, - "key06": [ - "" - ] + "key06": [] } diff --git a/fluent/test/fixtures_reference/multiline_values.json b/fluent/test/fixtures_reference/multiline_values.json index a4cb0a170..c6d153fcb 100644 --- a/fluent/test/fixtures_reference/multiline_values.json +++ b/fluent/test/fixtures_reference/multiline_values.json @@ -35,7 +35,7 @@ }, "key05": [ "A multiline value with non-standard", - "\n\n", + "\n\n ", "indentation." ], "key06": [ @@ -53,5 +53,41 @@ " starting and ending ", "with a placeable" ], - "key08": "Leading and trailing whitespace." + "key08": "Leading and trailing whitespace.", + "key09": [ + "zero", + "\n ", + "three", + "\n ", + "two", + "\n ", + "one", + "\n", + "zero" + ], + "key10": [ + " ", + "two", + "\n", + "zero", + "\n ", + "four" + ], + "key11": [ + " ", + "two", + "\n", + "zero" + ], + "key12": [ + ".", + "\n ", + "four" + ], + "key13": [ + " ", + "four", + "\n", + "." + ] } diff --git a/fluent/test/fixtures_reference/select_expressions.json b/fluent/test/fixtures_reference/select_expressions.json index 6881a51a7..65ed7c426 100644 --- a/fluent/test/fixtures_reference/select_expressions.json +++ b/fluent/test/fixtures_reference/select_expressions.json @@ -21,7 +21,6 @@ { "key": "other", "value": [ - "", "Other" ] } @@ -79,9 +78,7 @@ "variants": [ { "key": "one", - "value": [ - "" - ] + "value": [] } ], "star": 0 diff --git a/fluent/test/fixtures_reference/select_indent.json b/fluent/test/fixtures_reference/select_indent.json index 52b4c28bd..c8e21fea3 100644 --- a/fluent/test/fixtures_reference/select_indent.json +++ b/fluent/test/fixtures_reference/select_indent.json @@ -229,7 +229,6 @@ } ], "star": 0 - }, - "" + } ] } diff --git a/fluent/test/fixtures_reference/terms.json b/fluent/test/fixtures_reference/terms.json index 81e77bc5c..0195fe337 100644 --- a/fluent/test/fixtures_reference/terms.json +++ b/fluent/test/fixtures_reference/terms.json @@ -5,9 +5,7 @@ "attr": "Attribute" } }, - "-term02": [ - "" - ], + "-term02": [], "-term03": { "value": null, "attrs": { diff --git a/fluent/test/fixtures_structure/variant_with_empty_pattern.json b/fluent/test/fixtures_structure/variant_with_empty_pattern.json index 3ffa51e1b..23d0fb0ca 100644 --- a/fluent/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent/test/fixtures_structure/variant_with_empty_pattern.json @@ -9,9 +9,7 @@ "variants": [ { "key": "one", - "value": [ - "" - ] + "value": [] } ], "star": 0 diff --git a/fluent/test/fixtures_structure/whitespace_leading.json b/fluent/test/fixtures_structure/whitespace_leading.json index 7a69cc773..96fc498e8 100644 --- a/fluent/test/fixtures_structure/whitespace_leading.json +++ b/fluent/test/fixtures_structure/whitespace_leading.json @@ -2,7 +2,6 @@ "key1": "Value", "key2": "  Value", "key3": [ - "", " Value" ], "key4": [ From 7356eb40406eb566277511effb558fbc1d28d901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Wed, 21 Nov 2018 19:11:52 +0100 Subject: [PATCH 5/6] Refactor skipBlank* methods to use peekBlank* --- fluent-syntax/src/stream.js | 59 ++++++++++++++----------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js index 352b2031b..2f474be99 100644 --- a/fluent-syntax/src/stream.js +++ b/fluent-syntax/src/stream.js @@ -67,64 +67,44 @@ export const EOF = undefined; const SPECIAL_LINE_START_CHARS = ["}", ".", "[", "*"]; export class FluentParserStream extends ParserStream { - skipBlankInline() { - let blank = ""; - while (this.currentChar === " ") { - blank += " "; - this.next(); - } - return blank; - } - peekBlankInline() { + const start = this.index + this.peekOffset; while (this.currentPeek === " ") { this.peek(); } + return this.string.slice(start, this.index + this.peekOffset); } - skipBlankBlock() { + skipBlankInline() { + const blank = this.peekBlankInline(); + this.skipToPeek(); + return blank; + } + + peekBlankBlock() { let blank = ""; while (true) { + const lineStart = this.peekOffset; this.peekBlankInline(); - if (this.currentPeek === EOL) { - this.skipToPeek(); - this.next(); blank += EOL; + this.peek(); continue; } - if (this.currentPeek === EOF) { - // Consume any inline blanks before the EOF. - this.skipToPeek(); + // Treat the blank line at EOF as a blank block. return blank; } - // Any other char; reset to column 1 on this line. - this.resetPeek(); + this.resetPeek(lineStart); return blank; } } - peekBlankBlock() { - let blank = ""; - while (true) { - const lineStart = this.peekOffset; - this.peekBlankInline(); - if (this.currentPeek === EOL) { - this.peek(); - blank += EOL; - } else { - this.resetPeek(lineStart); - return blank; - } - } - } - - skipBlank() { - while (this.currentChar === " " || this.currentChar === EOL) { - this.next(); - } + skipBlankBlock() { + const blank = this.peekBlankBlock(); + this.skipToPeek(); + return blank; } peekBlank() { @@ -133,6 +113,11 @@ export class FluentParserStream extends ParserStream { } } + skipBlank() { + this.peekBlank(); + this.skipToPeek(); + } + expectChar(ch) { if (this.currentChar === ch) { this.next(); From 9a530e769281e4c5d7a2d2bae4d7f89b4c533d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 22 Nov 2018 11:43:09 +0100 Subject: [PATCH 6/6] Refactor maybeGetValue to take allowVariantList --- fluent-syntax/src/errors.js | 2 - fluent-syntax/src/parser.js | 118 ++++++++++-------- .../test/fixtures_behavior/variant_lists.ftl | 4 +- .../test/fixtures_structure/leading_dots.json | 2 +- .../fixtures_structure/sparse-messages.json | 2 +- .../test/fixtures_structure/term.json | 2 +- .../variant_with_empty_pattern.json | 2 +- 7 files changed, 73 insertions(+), 59 deletions(-) diff --git a/fluent-syntax/src/errors.js b/fluent-syntax/src/errors.js index f721d25a4..b13c301e3 100644 --- a/fluent-syntax/src/errors.js +++ b/fluent-syntax/src/errors.js @@ -62,8 +62,6 @@ function getErrorMessage(code, args) { return "Positional arguments must not follow named arguments"; case "E0022": return "Named arguments must be unique"; - case "E0023": - return "VariantLists are only allowed inside of other VariantLists."; case "E0024": return "Cannot access variants of a message."; case "E0025": { diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 5176772d4..965a45aa7 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -43,7 +43,7 @@ export default class FluentParser { // Poor man's decorators. const methodNames = [ "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", - "getVariant", "getNumber", "getValue", "getPattern", "getVariantList", + "getVariant", "getNumber", "getPattern", "getVariantList", "getTextElement", "getPlaceable", "getExpression", "getSelectorExpression", "getCallArg", "getString", "getLiteral" ]; @@ -231,7 +231,7 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - const value = this.getInlineOrBlock(ps, this.getPattern); + const value = this.maybeGetValue(ps, {allowVariantList: false}); const attrs = this.getAttributes(ps); if (value === null && attrs.length === 0) { @@ -248,7 +248,11 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - const value = this.getInlineOrBlock(ps, this.getValue); + // XXX Once https://github.com/projectfluent/fluent/pull/220 lands, + // getTerm will be the only place where VariantLists are still legal. Move + // the code from getPatternOrVariantList up to here then, and remove the + // allowVariantList switch. + const value = this.maybeGetValue(ps, {allowVariantList: true}); if (value === null) { throw new ParseError("E0006", id.name); } @@ -257,22 +261,6 @@ export default class FluentParser { return new AST.Term(id, value, attrs); } - getInlineOrBlock(ps, method) { - ps.peekBlankInline(); - if (ps.isValueStart()) { - ps.skipToPeek(); - return method.call(this, ps, {isBlock: false}); - } - - ps.peekBlankBlock(); - if (ps.isValueContinuation()) { - ps.skipToPeek(); - return method.call(this, ps, {isBlock: true}); - } - - return null; - } - getAttribute(ps) { ps.expectChar("."); @@ -281,7 +269,7 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectChar("="); - const value = this.getInlineOrBlock(ps, this.getPattern); + const value = this.maybeGetValue(ps, {allowVariantList: false}); if (value === null) { throw new ParseError("E0012"); } @@ -328,7 +316,7 @@ export default class FluentParser { return this.getIdentifier(ps); } - getVariant(ps, hasDefault) { + getVariant(ps, {hasDefault, allowVariantList}) { let defaultIndex = false; if (ps.currentChar === "*") { @@ -349,7 +337,9 @@ export default class FluentParser { ps.skipBlank(); ps.expectChar("]"); - const value = this.getInlineOrBlock(ps, this.getValue); + // XXX We need to pass allowVariantList all the way down to here because + // nested VariantLists in Terms are legal for now. + const value = this.maybeGetValue(ps, {allowVariantList}); if (value === null) { throw new ParseError("E0012"); } @@ -357,13 +347,13 @@ export default class FluentParser { return new AST.Variant(key, value, defaultIndex); } - getVariants(ps) { + getVariants(ps, {allowVariantList}) { const variants = []; let hasDefault = false; ps.skipBlank(); while (ps.isVariantStart()) { - const variant = this.getVariant(ps, hasDefault); + const variant = this.getVariant(ps, {allowVariantList, hasDefault}); if (variant.default) { hasDefault = true; @@ -419,29 +409,55 @@ export default class FluentParser { return new AST.NumberLiteral(num); } - getValue(ps, {isBlock}) { + // maybeGetValue distinguishes between patterns which start on the same line + // as the identifier (a.k.a. inline signleline patterns and inline multiline + // patterns) and patterns which start on a new line (a.k.a. block multiline + // patterns). The distinction is important for the dedentation logic: the + // indent of the first line of a block pattern must be taken into account when + // calculating the maximum common indent. + maybeGetValue(ps, {allowVariantList}) { ps.peekBlankInline(); - const peekOffset = ps.peekOffset; - if (ps.currentPeek === "{") { + if (ps.isValueStart()) { + ps.skipToPeek(); + return this.getPatternOrVariantList( + ps, {isBlock: false, allowVariantList}); + } + + ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + return this.getPatternOrVariantList( + ps, {isBlock: true, allowVariantList}); + } + + return null; + } + + // Parse a VariantList (if allowed) or a Pattern. + getPatternOrVariantList(ps, {isBlock, allowVariantList}) { + ps.peekBlankInline(); + if (allowVariantList && ps.currentPeek === "{") { + const start = ps.peekOffset; ps.peek(); - ps.peekBlank(); - if (ps.isVariantStart()) { - ps.resetPeek(peekOffset); - ps.skipToPeek(); - return this.getVariantList(ps); + ps.peekBlankInline(); + if (ps.currentPeek === EOL) { + ps.peekBlank(); + if (ps.isVariantStart()) { + ps.resetPeek(start); + ps.skipToPeek(); + return this.getVariantList(ps, {allowVariantList}); + } } } ps.resetPeek(); - return this.getPattern(ps, {isBlock}); + const pattern = this.getPattern(ps, {isBlock}); + return pattern; } getVariantList(ps) { ps.expectChar("{"); - ps.skipBlankInline(); - ps.expectLineEnd(); - const variants = this.getVariants(ps); - ps.skipBlank(); + var variants = this.getVariants(ps, {allowVariantList: true}); ps.expectChar("}"); return new AST.VariantList(variants); } @@ -449,6 +465,8 @@ export default class FluentParser { getPattern(ps, {isBlock}) { const elements = []; if (isBlock) { + // A block pattern is a pattern which starts on a new line. Store and + // measure the indent of this first line for the dedentation logic. const blankStart = ps.index; const firstIndent = ps.skipBlankInline(); elements.push(this.getIndent(ps, firstIndent, blankStart)); @@ -487,9 +505,13 @@ export default class FluentParser { } } - return new AST.Pattern(this.dedent(elements, commonIndentLength)); + const dedented = this.dedent(elements, commonIndentLength); + return new AST.Pattern(dedented); } + // Create a token representing an indent. It's not part of the AST and it will + // be trimmed and merged into adjacent TextElements, or turned into a new + // TextElement, if it's surrounded by two Placeables. getIndent(ps, value, start) { return { type: "Indent", @@ -498,6 +520,8 @@ export default class FluentParser { }; } + // Dedent a list of elements by removing the maximum common indent from the + // beginning of text lines. The common indent is calculated in getPattern. dedent(elements, commonIndent) { const trimmed = []; @@ -528,6 +552,8 @@ export default class FluentParser { } if (element.type === "Indent") { + // If the indent hasn't been merged into a preceding TextElement, + // convert it into a new TextElement. const textElement = new AST.TextElement(element.value); if (this.withSpans) { textElement.addSpan(element.span.start, element.span.end); @@ -538,7 +564,7 @@ export default class FluentParser { trimmed.push(element); } - // Trim trailing whitespace. + // Trim trailing whitespace from the Pattern. const lastElement = trimmed[trimmed.length - 1]; if (lastElement.type === "TextElement") { lastElement.value = lastElement.value.replace(trailingWSRe, ""); @@ -600,16 +626,14 @@ export default class FluentParser { getPlaceable(ps) { ps.expectChar("{"); + ps.skipBlank(); const expression = this.getExpression(ps); ps.expectChar("}"); return new AST.Placeable(expression); } getExpression(ps) { - ps.skipBlank(); - const selector = this.getSelectorExpression(ps); - ps.skipBlank(); if (ps.currentChar === "-") { @@ -638,21 +662,13 @@ export default class FluentParser { ps.skipBlankInline(); ps.expectLineEnd(); - const variants = this.getVariants(ps); - - // VariantLists are only allowed in other VariantLists. - if (variants.some(v => v.value.type === "VariantList")) { - throw new ParseError("E0023"); - } - + const variants = this.getVariants(ps, {allowVariantList: false}); return new AST.SelectExpression(selector, variants); } else if (selector.type === "AttributeExpression" && selector.ref.type === "TermReference") { throw new ParseError("E0019"); } - ps.skipBlank(); - return selector; } diff --git a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl index 10f5cab3d..2f3cc5b8d 100644 --- a/fluent-syntax/test/fixtures_behavior/variant_lists.ftl +++ b/fluent-syntax/test/fixtures_behavior/variant_lists.ftl @@ -4,7 +4,7 @@ message1 = *[one] One } -# ~ERROR E0023, pos 123 +# ~ERROR E0014, pos 97 message2 = { $sel -> *[one] { @@ -24,7 +24,7 @@ message2 = } } -# ~ERROR E0023, pos 318 +# ~ERROR E0014, pos 292 -term3 = { $sel -> *[one] { diff --git a/fluent-syntax/test/fixtures_structure/leading_dots.json b/fluent-syntax/test/fixtures_structure/leading_dots.json index afb84b8d0..d2e234c38 100644 --- a/fluent-syntax/test/fixtures_structure/leading_dots.json +++ b/fluent-syntax/test/fixtures_structure/leading_dots.json @@ -836,7 +836,7 @@ ], "span": { "type": "Span", - "start": 614, + "start": 615, "end": 684 } }, diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.json b/fluent-syntax/test/fixtures_structure/sparse-messages.json index d6a51ff07..32780f68c 100644 --- a/fluent-syntax/test/fixtures_structure/sparse-messages.json +++ b/fluent-syntax/test/fixtures_structure/sparse-messages.json @@ -322,7 +322,7 @@ ], "span": { "type": "Span", - "start": 154, + "start": 155, "end": 208 } }, diff --git a/fluent-syntax/test/fixtures_structure/term.json b/fluent-syntax/test/fixtures_structure/term.json index aadfd9424..c36ec4509 100644 --- a/fluent-syntax/test/fixtures_structure/term.json +++ b/fluent-syntax/test/fixtures_structure/term.json @@ -481,7 +481,7 @@ ], "span": { "type": "Span", - "start": 198, + "start": 199, "end": 436 } }, diff --git a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json index c75825177..2b61ee990 100644 --- a/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/variant_with_empty_pattern.json @@ -77,7 +77,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 13, "end": 41 } },