diff --git a/fluent-syntax/src/ast.js b/fluent-syntax/src/ast.js index 6fb91af54..65bf65899 100644 --- a/fluent-syntax/src/ast.js +++ b/fluent-syntax/src/ast.js @@ -67,6 +67,14 @@ export class TextElement extends SyntaxNode { } } +export class Placeable extends SyntaxNode { + constructor(expression) { + super(); + this.type = 'Placeable'; + this.expression = expression; + } +} + export class Expression extends SyntaxNode { constructor() { super(); diff --git a/fluent-syntax/src/ftlstream.js b/fluent-syntax/src/ftlstream.js index 9fc92701b..d76515231 100644 --- a/fluent-syntax/src/ftlstream.js +++ b/fluent-syntax/src/ftlstream.js @@ -2,23 +2,26 @@ import { ParserStream } from './stream'; import { ParseError } from './errors'; +import { includes } from './util'; + +const INLINE_WS = [' ', '\t']; export class FTLParserStream extends ParserStream { - peekLineWS() { + peekInlineWS() { let ch = this.currentPeek(); while (ch) { - if (ch !== ' ' && ch !== '\t') { + if (!includes(INLINE_WS, ch)) { break; } ch = this.peek(); } } - skipWSLines() { + skipBlankLines() { while (true) { - this.peekLineWS(); + this.peekInlineWS(); - if (this.currentPeek() === '\n') { + if (this.currentPeekIs('\n')) { this.skipToPeek(); this.next(); } else { @@ -28,9 +31,9 @@ export class FTLParserStream extends ParserStream { } } - skipLineWS() { + skipInlineWS() { while (this.ch) { - if (this.ch !== ' ' && this.ch !== '\t') { + if (!includes(INLINE_WS, this.ch)) { break; } this.next(); @@ -84,22 +87,6 @@ export class FTLParserStream extends ParserStream { return ((cc >= 48 && cc <= 57) || cc === 45); // 0-9 } - isPeekNextLineIndented() { - if (!this.currentPeekIs('\n')) { - return false; - } - - this.peek(); - - if (this.currentPeekIs(' ')) { - this.resetPeek(); - return true; - } - - this.resetPeek(); - return false; - } - isPeekNextLineVariantStart() { if (!this.currentPeekIs('\n')) { return false; @@ -109,7 +96,7 @@ export class FTLParserStream extends ParserStream { const ptr = this.getPeekIndex(); - this.peekLineWS(); + this.peekInlineWS(); if (this.getPeekIndex() - ptr === 0) { this.resetPeek(); @@ -137,7 +124,7 @@ export class FTLParserStream extends ParserStream { const ptr = this.getPeekIndex(); - this.peekLineWS(); + this.peekInlineWS(); if (this.getPeekIndex() - ptr === 0) { this.resetPeek(); @@ -162,7 +149,7 @@ export class FTLParserStream extends ParserStream { const ptr = this.getPeekIndex(); - this.peekLineWS(); + this.peekInlineWS(); if (this.getPeekIndex() - ptr === 0) { this.resetPeek(); @@ -191,7 +178,7 @@ export class FTLParserStream extends ParserStream { const ptr = this.getPeekIndex(); - this.peekLineWS(); + this.peekInlineWS(); if (this.getPeekIndex() - ptr === 0) { this.resetPeek(); @@ -208,7 +195,7 @@ export class FTLParserStream extends ParserStream { } skipToNextEntryStart() { - while (this.next()) { + while (this.ch) { if (this.currentIs('\n') && !this.peekCharIs('\n')) { this.next(); if (this.ch === undefined || this.isIDStart() || @@ -217,6 +204,7 @@ export class FTLParserStream extends ParserStream { break; } } + this.next(); } } diff --git a/fluent-syntax/src/index.js b/fluent-syntax/src/index.js index 0b57c0d8e..20851ab0f 100644 --- a/fluent-syntax/src/index.js +++ b/fluent-syntax/src/index.js @@ -15,14 +15,22 @@ export function serialize(resource, opts) { } export function lineOffset(source, pos) { - // Substract 1 to get the offset. + // Subtract 1 to get the offset. return source.substring(0, pos).split('\n').length - 1; } export function columnOffset(source, pos) { - const lastLineBreak = source.lastIndexOf('\n', pos); - return lastLineBreak === -1 - ? pos - // Substracting two offsets gives length; substract 1 to get the offset. - : pos - lastLineBreak - 1; + // Find the last line break starting backwards from the index just before + // pos. This allows us to correctly handle ths case where the character at + // pos is a line break as well. + const fromIndex = pos - 1; + const prevLineBreak = source.lastIndexOf('\n', fromIndex); + + // pos is a position in the first line of source. + if (prevLineBreak === -1) { + return pos; + } + + // Subtracting two offsets gives length; subtract 1 to get the offset. + return pos - prevLineBreak - 1; } diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index df2bddc72..00780bef2 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -44,8 +44,8 @@ export default class FluentParser { [ 'getComment', 'getSection', 'getMessage', 'getAttribute', 'getTag', 'getIdentifier', 'getVariant', 'getSymbol', 'getNumber', 'getPattern', - 'getExpression', 'getSelectorExpression', 'getCallArg', 'getString', - 'getLiteral', + 'getTextElement', 'getPlaceable', 'getExpression', + 'getSelectorExpression', 'getCallArg', 'getString', 'getLiteral', ].forEach( name => this[name] = withSpan(this[name]) ); @@ -55,7 +55,7 @@ export default class FluentParser { let comment = null; const ps = new FTLParserStream(source); - ps.skipWSLines(); + ps.skipBlankLines(); const entries = []; @@ -68,7 +68,7 @@ export default class FluentParser { entries.push(entry); } - ps.skipWSLines(); + ps.skipBlankLines(); } const res = new AST.Resource(entries, comment); @@ -82,7 +82,7 @@ export default class FluentParser { parseEntry(source) { const ps = new FTLParserStream(source); - ps.skipWSLines(); + ps.skipBlankLines(); return this.getEntryOrJunk(ps); } @@ -169,16 +169,16 @@ export default class FluentParser { ps.expectChar('['); ps.expectChar('['); - ps.skipLineWS(); + ps.skipInlineWS(); const symb = this.getSymbol(ps); - ps.skipLineWS(); + ps.skipInlineWS(); ps.expectChar(']'); ps.expectChar(']'); - ps.skipLineWS(); + ps.skipInlineWS(); ps.expectChar('\n'); @@ -188,7 +188,7 @@ export default class FluentParser { getMessage(ps, comment) { const id = this.getIdentifier(ps); - ps.skipLineWS(); + ps.skipInlineWS(); let pattern; let attrs; @@ -196,7 +196,7 @@ export default class FluentParser { if (ps.currentIs('=')) { ps.next(); - ps.skipLineWS(); + ps.skipInlineWS(); pattern = this.getPattern(ps); } @@ -224,9 +224,9 @@ export default class FluentParser { const key = this.getIdentifier(ps); - ps.skipLineWS(); + ps.skipInlineWS(); ps.expectChar('='); - ps.skipLineWS(); + ps.skipInlineWS(); const value = this.getPattern(ps); @@ -242,7 +242,7 @@ export default class FluentParser { while (true) { ps.expectChar('\n'); - ps.skipLineWS(); + ps.skipInlineWS(); const attr = this.getAttribute(ps); attrs.push(attr); @@ -265,7 +265,7 @@ export default class FluentParser { while (true) { ps.expectChar('\n'); - ps.skipLineWS(); + ps.skipInlineWS(); const tag = this.getTag(ps); tags.push(tag); @@ -324,7 +324,7 @@ export default class FluentParser { ps.expectChar(']'); - ps.skipLineWS(); + ps.skipInlineWS(); const value = this.getPattern(ps); @@ -341,7 +341,7 @@ export default class FluentParser { while (true) { ps.expectChar('\n'); - ps.skipLineWS(); + ps.skipInlineWS(); const variant = this.getVariant(ps, hasDefault); @@ -415,80 +415,83 @@ export default class FluentParser { } getPattern(ps) { - let buffer = ''; const elements = []; - let firstLine = true; + ps.skipInlineWS(); - if (this.withSpans) { - var spanStart = ps.getIndex(); + // Special-case: trim leading whitespace and newlines. + if (ps.isPeekNextLinePattern()) { + ps.skipBlankLines(); + ps.skipInlineWS(); } let ch; while ((ch = ps.current())) { - if (ch === '\n') { - if (firstLine && buffer.length !== 0) { - break; - } + // The end condition for getPattern's while loop is a newline + // which is not followed by a valid pattern continuation. + if (ch === '\n' && !ps.isPeekNextLinePattern()) { + break; + } + + if (ch === '{') { + const element = this.getPlaceable(ps); + elements.push(element); + } else { + const element = this.getTextElement(ps); + elements.push(element); + } + } + + return new AST.Pattern(elements); + } + + getTextElement(ps) { + let buffer = ''; + + let ch; + while ((ch = ps.current())) { + + if (ch === '{') { + return new AST.TextElement(buffer); + } + + if (ch === '\n') { if (!ps.isPeekNextLinePattern()) { - break; + return new AST.TextElement(buffer); } ps.next(); - ps.skipLineWS(); + ps.skipInlineWS(); - if (!firstLine) { - buffer += ch; - } - firstLine = false; + // Add the new line to the buffer + buffer += ch; continue; - } else if (ch === '\\') { - const ch2 = ps.peek(); + } + + if (ch === '\\') { + const ch2 = ps.next(); + if (ch2 === '{' || ch2 === '"') { buffer += ch2; } else { buffer += ch + ch2; } - ps.next(); - } else if (ch === '{') { - ps.next(); - - ps.skipLineWS(); - - if (buffer.length !== 0) { - const text = new AST.TextElement(buffer); - if (this.withSpans) { - text.addSpan(spanStart, ps.getIndex()); - } - elements.push(text); - } - - buffer = ''; - - elements.push(this.getExpression(ps)); - - ps.expectChar('}'); - - if (this.withSpans) { - spanStart = ps.getIndex(); - } - continue; } else { buffer += ps.ch; } + ps.next(); } - if (buffer.length !== 0) { - const text = new AST.TextElement(buffer); - if (this.withSpans) { - text.addSpan(spanStart, ps.getIndex()); - } - elements.push(text); - } + return new AST.TextElement(buffer); + } - return new AST.Pattern(elements); + getPlaceable(ps) { + ps.expectChar('{'); + const expression = this.getExpression(ps); + ps.expectChar('}'); + return new AST.Placeable(expression); } getExpression(ps) { @@ -497,14 +500,16 @@ export default class FluentParser { ps.expectChar('\n'); ps.expectChar(' '); - ps.skipLineWS(); + ps.skipInlineWS(); return new AST.SelectExpression(null, variants); } + ps.skipInlineWS(); + const selector = this.getSelectorExpression(ps); - ps.skipLineWS(); + ps.skipInlineWS(); if (ps.currentIs('-')) { ps.peek(); @@ -514,7 +519,7 @@ export default class FluentParser { ps.next(); ps.next(); - ps.skipLineWS(); + ps.skipInlineWS(); const variants = this.getVariants(ps); @@ -524,7 +529,7 @@ export default class FluentParser { ps.expectChar('\n'); ps.expectChar(' '); - ps.skipLineWS(); + ps.skipInlineWS(); return new AST.SelectExpression(selector, variants); } @@ -582,7 +587,7 @@ export default class FluentParser { getCallArg(ps) { const exp = this.getSelectorExpression(ps); - ps.skipLineWS(); + ps.skipInlineWS(); if (ps.current() !== ':') { return exp; @@ -593,7 +598,7 @@ export default class FluentParser { } ps.next(); - ps.skipLineWS(); + ps.skipInlineWS(); const val = this.getArgVal(ps); @@ -603,7 +608,7 @@ export default class FluentParser { getCallArgs(ps) { const args = []; - ps.skipLineWS(); + ps.skipInlineWS(); while (true) { if (ps.current() === ')') { @@ -613,11 +618,11 @@ export default class FluentParser { const arg = this.getCallArg(ps); args.push(arg); - ps.skipLineWS(); + ps.skipInlineWS(); if (ps.current() === ',') { ps.next(); - ps.skipLineWS(); + ps.skipInlineWS(); continue; } else { break; diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index 2ceb20ae7..9a617c362 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -1,10 +1,12 @@ +import { includes } from './util'; + function indent(content) { return content.split('\n').join('\n '); } function containNewLine(elems) { const withNewLine = elems.filter( - elem => (elem.type === 'TextElement' && elem.value.includes('\n')) + elem => (elem.type === 'TextElement' && includes(elem.value, '\n')) ); return !!withNewLine.length; } @@ -137,10 +139,10 @@ function serializeElement(element) { switch (element.type) { case 'TextElement': return serializeTextElement(element); - case 'SelectExpression': - return `{${serializeSelectExpression(element)}}`; + case 'Placeable': + return serializePlaceable(element); default: - return `{ ${serializeExpression(element)} }`; + throw new Error(`Unknown element type: ${element.type}`); } } @@ -150,6 +152,20 @@ function serializeTextElement(text) { } +function serializePlaceable(placeable) { + const expr = placeable.expression; + + switch (expr.type) { + case 'Placeable': + return `{${serializePlaceable(expr)}}`; + case 'SelectExpression': + return `{${serializeSelectExpression(expr)}}`; + default: + return `{ ${serializeExpression(expr)} }`; + } +} + + function serializeExpression(expr) { switch (expr.type) { case 'StringExpression': diff --git a/fluent-syntax/src/util.js b/fluent-syntax/src/util.js new file mode 100644 index 000000000..c733bd069 --- /dev/null +++ b/fluent-syntax/src/util.js @@ -0,0 +1,3 @@ +export function includes(arr, elem) { + return arr.indexOf(elem) > -1; +} diff --git a/fluent-syntax/test/behavior_test.js b/fluent-syntax/test/behavior_test.js index dd9d0acff..3838ded53 100644 --- a/fluent-syntax/test/behavior_test.js +++ b/fluent-syntax/test/behavior_test.js @@ -77,8 +77,7 @@ readdir(fixtures, function(err, filenames) { const ast = parse(source); const actual = ast.body.reduce(toDirectives, []).join('\n') + '\n'; assert.deepEqual( - actual, expected, - 'Actual Annotations don\'t match the expected ones' + actual, expected, 'Annotations mismatch' ); }); }); diff --git a/fluent-syntax/test/fixtures_behavior/indent.ftl b/fluent-syntax/test/fixtures_behavior/indent.ftl new file mode 100644 index 000000000..6bcdd5698 --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/indent.ftl @@ -0,0 +1,15 @@ + key1 = A +//~ ERROR E0002, pos 0 + +key2 = { +a } +//~ ERROR E0004, pos 20, args "a-zA-Z_" +//~ ERROR E0005, pos 23, args "a" + +key3 = { a +} +//~ ERROR E0003, pos 36, args "}" + +key4 = { +{ a }} +//~ ERROR E0004, pos 48, args "a-zA-Z_" diff --git a/fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl b/fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl index 70417cc57..e39b65da2 100644 --- a/fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl +++ b/fluent-syntax/test/fixtures_behavior/multiline_with_non_empty_first_line.ftl @@ -1,3 +1,2 @@ key = Value Value 2 -//~ ERROR E0002, pos 12 diff --git a/fluent-syntax/test/fixtures_behavior/multiline_with_placeables.ftl b/fluent-syntax/test/fixtures_behavior/multiline_with_placeables.ftl new file mode 100644 index 000000000..2606ff12c --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/multiline_with_placeables.ftl @@ -0,0 +1,3 @@ +key = + Foo { bar } + Baz diff --git a/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl new file mode 100644 index 000000000..46e03df6b --- /dev/null +++ b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl @@ -0,0 +1,14 @@ +// key1 = {{ foo }} + +// key2 = { { foo } } + +// key3 = +// { +// { foo } +// } + +key4 = { { foo } +//~ ERROR E0004, pos 96, args "a-zA-Z_" + + +// key5 = { foo } } diff --git a/fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl b/fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl index d4731b4a9..242d116d0 100644 --- a/fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl +++ b/fluent-syntax/test/fixtures_behavior/section_with_nl_in_the_middle.ftl @@ -1,3 +1,4 @@ [[ This is a broken section]] //~ ERROR E0003, pos 10, args "]" +//~ ERROR E0005, pos 13, args "a" diff --git a/fluent-syntax/test/fixtures_structure/placeable_at_eol.ftl b/fluent-syntax/test/fixtures_structure/placeable_at_eol.ftl index 37665cc24..0046013fd 100644 --- a/fluent-syntax/test/fixtures_structure/placeable_at_eol.ftl +++ b/fluent-syntax/test/fixtures_structure/placeable_at_eol.ftl @@ -1,4 +1,9 @@ -key = +key1 = A multiline message with a { placeable } at the end of line. The message should consist of three lines of text. + +key2 = + A multiline message with a { placeable } + +key3 = A singleline message with a { placeable } diff --git a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json index 75f559d0e..8310b559e 100644 --- a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json +++ b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json @@ -6,11 +6,11 @@ "annotations": [], "id": { "type": "Identifier", - "name": "key", + "name": "key1", "span": { "type": "Span", "start": 0, - "end": 3 + "end": 4 } }, "value": { @@ -21,25 +21,33 @@ "value": "A multiline message with a ", "span": { "type": "Span", - "start": 5, - "end": 39 + "start": 11, + "end": 38 } }, { - "type": "MessageReference", - "id": { - "type": "Identifier", - "name": "placeable", + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable", + "span": { + "type": "Span", + "start": 40, + "end": 49 + } + }, "span": { "type": "Span", - "start": 39, - "end": 48 + "start": 40, + "end": 49 } }, "span": { "type": "Span", - "start": 39, - "end": 48 + "start": 38, + "end": 51 } }, { @@ -47,15 +55,15 @@ "value": "\nat the end of line. The message should\nconsist of three lines of text.", "span": { "type": "Span", - "start": 50, - "end": 130 + "start": 51, + "end": 131 } } ], "span": { "type": "Span", - "start": 5, - "end": 130 + "start": 6, + "end": 131 } }, "attributes": [], @@ -64,7 +72,137 @@ "span": { "type": "Span", "start": 0, - "end": 130 + "end": 131 + } + }, + { + "type": "Message", + "annotations": [], + "id": { + "type": "Identifier", + "name": "key2", + "span": { + "type": "Span", + "start": 133, + "end": 137 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline message with a ", + "span": { + "type": "Span", + "start": 144, + "end": 171 + } + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable", + "span": { + "type": "Span", + "start": 173, + "end": 182 + } + }, + "span": { + "type": "Span", + "start": 173, + "end": 182 + } + }, + "span": { + "type": "Span", + "start": 171, + "end": 184 + } + } + ], + "span": { + "type": "Span", + "start": 139, + "end": 184 + } + }, + "attributes": [], + "tags": [], + "comment": null, + "span": { + "type": "Span", + "start": 133, + "end": 184 + } + }, + { + "type": "Message", + "annotations": [], + "id": { + "type": "Identifier", + "name": "key3", + "span": { + "type": "Span", + "start": 186, + "end": 190 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A singleline message with a ", + "span": { + "type": "Span", + "start": 193, + "end": 221 + } + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable", + "span": { + "type": "Span", + "start": 223, + "end": 232 + } + }, + "span": { + "type": "Span", + "start": 223, + "end": 232 + } + }, + "span": { + "type": "Span", + "start": 221, + "end": 234 + } + } + ], + "span": { + "type": "Span", + "start": 193, + "end": 234 + } + }, + "attributes": [], + "tags": [], + "comment": null, + "span": { + "type": "Span", + "start": 186, + "end": 234 } } ], @@ -72,6 +210,6 @@ "span": { "type": "Span", "start": 0, - "end": 131 + "end": 235 } } diff --git a/fluent-syntax/test/structure_test.js b/fluent-syntax/test/structure_test.js index 961aaeeb6..0b17bceff 100644 --- a/fluent-syntax/test/structure_test.js +++ b/fluent-syntax/test/structure_test.js @@ -27,7 +27,7 @@ readdir(fixtures, function(err, filenames) { const ast = parse(ftl); assert.deepEqual( ast, JSON.parse(expected), - 'Actual Annotations don\'t match the expected ones' + 'Parsed AST doesn\'t match the expected one' ); }); }); diff --git a/fluent/test/fixtures_structure/placeable_at_eol.json b/fluent/test/fixtures_structure/placeable_at_eol.json index dd58b6eb5..a9ba1ab14 100644 --- a/fluent/test/fixtures_structure/placeable_at_eol.json +++ b/fluent/test/fixtures_structure/placeable_at_eol.json @@ -1,5 +1,5 @@ { - "key": { + "key1": { "val": [ "A multiline message with a ", { @@ -8,5 +8,23 @@ }, "\nat the end of line. The message should\nconsist of three lines of text." ] + }, + "key2": { + "val": [ + "A multiline message with a ", + { + "type": "ref", + "name": "placeable" + } + ] + }, + "key3": { + "val": [ + "A singleline message with a ", + { + "type": "ref", + "name": "placeable" + } + ] } } diff --git a/fluent/test/fixtures_structure/placeable_in_placeable.json b/fluent/test/fixtures_structure/placeable_in_placeable.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/fluent/test/fixtures_structure/placeable_in_placeable.json @@ -0,0 +1 @@ +{} diff --git a/tools/parse.js b/tools/parse.js index edcb9fdb7..4307b5021 100755 --- a/tools/parse.js +++ b/tools/parse.js @@ -64,7 +64,7 @@ function printAnnotations(source, entry) { } function printAnnotation(source, span, annot) { - const { name, message, span: { start } } = annot; + const { code, message, span: { start } } = annot; const slice = source.substring(span.start, span.end).trimRight(); const lineNumber = FluentSyntax.lineOffset(source, start) + 1; const columnOffset = FluentSyntax.columnOffset(source, start); @@ -74,7 +74,7 @@ function printAnnotation(source, span, annot) { const tail = lines.slice(showLines); console.log(); - console.log(`! ${name} on line ${lineNumber}:`); + console.log(`! ${code} on line ${lineNumber}:`); console.log(head.map(line => ` | ${line}`).join('\n')); console.log(` … ${indent(columnOffset)}^----- ${message}`); console.log(tail.map(line => ` | ${line}`).join('\n'));