diff --git a/fluent-syntax/src/ftlstream.js b/fluent-syntax/src/ftlstream.js index b16f57302..39b014ba3 100644 --- a/fluent-syntax/src/ftlstream.js +++ b/fluent-syntax/src/ftlstream.js @@ -10,8 +10,8 @@ const SPECIAL_LINE_START_CHARS = ["}", ".", "[", "*"]; export class FTLParserStream extends ParserStream { skipBlankInline() { - while (this.ch) { - if (this.ch !== INLINE_WS) { + while (this.currentChar) { + if (this.currentChar !== INLINE_WS) { break; } this.next(); @@ -19,7 +19,7 @@ export class FTLParserStream extends ParserStream { } peekBlankInline() { - let ch = this.currentPeek(); + let ch = this.currentPeek; while (ch) { if (ch !== INLINE_WS) { break; @@ -33,7 +33,7 @@ export class FTLParserStream extends ParserStream { while (true) { this.peekBlankInline(); - if (this.currentPeekIs("\n")) { + if (this.currentPeek === "\n") { this.skipToPeek(); this.next(); lineCount++; @@ -46,11 +46,11 @@ export class FTLParserStream extends ParserStream { peekBlankBlock() { while (true) { - const lineStart = this.getPeekIndex(); + const lineStart = this.peekOffset; this.peekBlankInline(); - if (this.currentPeekIs("\n")) { + if (this.currentPeek === "\n") { this.peek(); } else { this.resetPeek(lineStart); @@ -60,19 +60,19 @@ export class FTLParserStream extends ParserStream { } skipBlank() { - while (includes(ANY_WS, this.ch)) { + while (includes(ANY_WS, this.currentChar)) { this.next(); } } peekBlank() { - while (includes(ANY_WS, this.currentPeek())) { + while (includes(ANY_WS, this.currentPeek)) { this.peek(); } } expectChar(ch) { - if (this.ch === ch) { + if (this.currentChar === ch) { this.next(); return true; } @@ -86,7 +86,7 @@ export class FTLParserStream extends ParserStream { } expectLineEnd() { - if (this.ch === undefined) { + if (this.currentChar === undefined) { // EOF is a valid line end in Fluent. return true; } @@ -95,7 +95,7 @@ export class FTLParserStream extends ParserStream { } takeChar(f) { - const ch = this.ch; + const ch = this.currentChar; if (ch !== undefined && f(ch)) { this.next(); return ch; @@ -114,16 +114,15 @@ export class FTLParserStream extends ParserStream { } isIdentifierStart() { - const ch = this.currentPeek(); - const isID = this.isCharIDStart(ch); + const isID = this.isCharIDStart(this.currentPeek); this.resetPeek(); return isID; } isNumberStart() { - const ch = this.currentIs("-") + const ch = this.currentChar === "-" ? this.peek() - : this.current(); + : this.currentChar; if (ch === undefined) { this.resetPeek(); @@ -148,7 +147,7 @@ export class FTLParserStream extends ParserStream { if (skip === false) throw new Error("Unimplemented"); this.peekBlankInline(); - const ch = this.currentPeek(); + const ch = this.currentPeek; // Inline Patterns may start with any char. if (ch !== undefined && ch !== "\n") { @@ -166,7 +165,7 @@ export class FTLParserStream extends ParserStream { isNextLineComment(level = -1, {skip = false}) { if (skip === true) throw new Error("Unimplemented"); - if (!this.currentPeekIs("\n")) { + if (this.currentPeek !== "\n") { return false; } @@ -174,7 +173,7 @@ export class FTLParserStream extends ParserStream { while (i <= level || (level === -1 && i < 3)) { this.peek(); - if (!this.currentPeekIs("#")) { + if (this.currentPeek !== "#") { if (i <= level && level !== -1) { this.resetPeek(); return false; @@ -185,7 +184,7 @@ export class FTLParserStream extends ParserStream { } this.peek(); - if ([" ", "\n"].includes(this.currentPeek())) { + if ([" ", "\n"].includes(this.currentPeek)) { this.resetPeek(); return true; } @@ -197,17 +196,17 @@ export class FTLParserStream extends ParserStream { isNextLineVariantStart({skip = false}) { if (skip === true) throw new Error("Unimplemented"); - if (!this.currentPeekIs("\n")) { + if (this.currentPeek !== "\n") { return false; } this.peekBlank(); - if (this.currentPeekIs("*")) { + if (this.currentPeek === "*") { this.peek(); } - if (this.currentPeekIs("[")) { + if (this.currentPeek === "[") { this.resetPeek(); return true; } @@ -220,7 +219,7 @@ export class FTLParserStream extends ParserStream { this.peekBlank(); - if (this.currentPeekIs(".")) { + if (this.currentPeek === ".") { this.skipToPeek(); return true; } @@ -230,23 +229,23 @@ export class FTLParserStream extends ParserStream { } isNextLineValue({skip = true}) { - if (!this.currentPeekIs("\n")) { + if (this.currentPeek !== "\n") { return false; } this.peekBlankBlock(); - const ptr = this.getPeekIndex(); + const ptr = this.peekOffset; this.peekBlankInline(); - if (!this.currentPeekIs("{")) { - if (this.getPeekIndex() - ptr === 0) { + if (this.currentPeek !== "{") { + if (this.peekOffset - ptr === 0) { this.resetPeek(); return false; } - if (!this.isCharPatternContinuation(this.currentPeek())) { + if (!this.isCharPatternContinuation(this.currentPeek)) { this.resetPeek(); return false; } @@ -260,14 +259,19 @@ export class FTLParserStream extends ParserStream { return true; } - skipToNextEntryStart() { - while (this.ch) { - if (this.currentIs("\n") && !this.peekCharIs("\n")) { + skipToNextEntryStart(junkStart) { + let lastNewline = this.string.lastIndexOf("\n", this.index); + if (junkStart < lastNewline) { + // We're beyond the start of the junk. Rewind to the last seen newline. + this.index = lastNewline; + } + while (this.currentChar) { + if (this.currentChar === "\n" && this.peek() !== "\n") { this.next(); - if (this.ch === undefined || + if (this.currentChar === undefined || this.isIdentifierStart() || - this.currentIs("-") || - this.currentIs("#")) { + this.currentChar === "-" || + this.currentChar === "#") { break; } } @@ -276,8 +280,8 @@ export class FTLParserStream extends ParserStream { } takeIDStart() { - if (this.isCharIDStart(this.ch)) { - const ret = this.ch; + if (this.isCharIDStart(this.currentChar)) { + const ret = this.currentChar; this.next(); return ret; } diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 598da4d89..d7460bb7b 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -14,7 +14,7 @@ function withSpan(fn) { return fn.call(this, ps, ...args); } - const start = ps.getIndex(); + const start = ps.index; const node = fn.call(this, ps, ...args); // Don't re-add the span if the node already has it. This may happen when @@ -23,7 +23,7 @@ function withSpan(fn) { return node; } - const end = ps.getIndex(); + const end = ps.index; node.addSpan(start, end); return node; }; @@ -56,7 +56,7 @@ export default class FluentParser { const entries = []; let lastComment = null; - while (ps.current()) { + while (ps.currentChar) { const entry = this.getEntryOrJunk(ps); const blankLines = ps.skipBlankBlock(); @@ -65,7 +65,7 @@ export default class FluentParser { // they should parse as standalone when they're followed by Junk. // Consequently, we only attach Comments once we know that the Message // or the Term parsed successfully. - if (entry.type === "Comment" && blankLines === 0 && ps.current()) { + if (entry.type === "Comment" && blankLines === 0 && ps.currentChar) { // Stash the comment and decide what to do with it in the next pass. lastComment = entry; continue; @@ -91,7 +91,7 @@ export default class FluentParser { const res = new AST.Resource(entries); if (this.withSpans) { - res.addSpan(0, ps.getIndex()); + res.addSpan(0, ps.index); } return res; @@ -110,7 +110,7 @@ export default class FluentParser { const ps = new FTLParserStream(source.replace(/\r\n/g, "\n")); ps.skipBlankBlock(); - while (ps.currentIs("#")) { + while (ps.currentChar === "#") { const skipped = this.getEntryOrJunk(ps); if (skipped.type === "Junk") { // Don't skip Junk comments. @@ -123,7 +123,7 @@ export default class FluentParser { } getEntryOrJunk(ps) { - const entryStartPos = ps.getIndex(); + const entryStartPos = ps.index; try { const entry = this.getEntry(ps); @@ -134,12 +134,16 @@ export default class FluentParser { throw err; } - const errorIndex = ps.getIndex(); - ps.skipToNextEntryStart(); - const nextEntryStart = ps.getIndex(); + let errorIndex = ps.index; + ps.skipToNextEntryStart(entryStartPos); + const nextEntryStart = ps.index; + if (nextEntryStart < errorIndex) { + // The position of the error must be inside of the Junk's span. + errorIndex = nextEntryStart; + } // Create a Junk instance - const slice = ps.getSlice(entryStartPos, nextEntryStart); + const slice = ps.string.substring(entryStartPos, nextEntryStart); const junk = new AST.Junk(slice); if (this.withSpans) { junk.addSpan(entryStartPos, nextEntryStart); @@ -152,11 +156,11 @@ export default class FluentParser { } getEntry(ps) { - if (ps.currentIs("#")) { + if (ps.currentChar === "#") { return this.getComment(ps); } - if (ps.currentIs("-")) { + if (ps.currentChar === "-") { return this.getTerm(ps); } @@ -176,7 +180,7 @@ export default class FluentParser { while (true) { let i = -1; - while (ps.currentIs("#") && (i < (level === -1 ? 2 : level))) { + while (ps.currentChar === "#" && (i < (level === -1 ? 2 : level))) { ps.next(); i++; } @@ -185,7 +189,7 @@ export default class FluentParser { level = i; } - if (!ps.currentIs("\n")) { + if (ps.currentChar !== "\n") { ps.expectChar(" "); let ch; while ((ch = ps.takeChar(x => x !== "\n"))) { @@ -194,7 +198,7 @@ export default class FluentParser { } if (ps.isNextLineComment(level, {skip: false})) { - content += ps.current(); + content += ps.currentChar; ps.next(); } else { break; @@ -305,7 +309,7 @@ export default class FluentParser { } getVariantKey(ps) { - const ch = ps.current(); + const ch = ps.currentChar; if (!ch) { throw new ParseError("E0013"); @@ -323,7 +327,7 @@ export default class FluentParser { getVariant(ps, hasDefault) { let defaultIndex = false; - if (ps.currentIs("*")) { + if (ps.currentChar === "*") { if (hasDefault) { throw new ParseError("E0015"); } @@ -394,14 +398,14 @@ export default class FluentParser { getNumber(ps) { let num = ""; - if (ps.currentIs("-")) { + if (ps.currentChar === "-") { num += "-"; ps.next(); } num = `${num}${this.getDigits(ps)}`; - if (ps.currentIs(".")) { + if (ps.currentChar === ".") { num += "."; ps.next(); num = `${num}${this.getDigits(ps)}`; @@ -411,7 +415,7 @@ export default class FluentParser { } getValue(ps) { - if (ps.currentIs("{")) { + if (ps.currentChar === "{") { ps.peek(); ps.peekBlankInline(); if (ps.isNextLineVariantStart({skip: false})) { @@ -441,7 +445,7 @@ export default class FluentParser { ps.skipBlankInline(); let ch; - while ((ch = ps.current())) { + while ((ch = ps.currentChar)) { // The end condition for getPattern's while loop is a newline // which is not followed by a valid pattern continuation. @@ -474,7 +478,7 @@ export default class FluentParser { let buffer = ""; let ch; - while ((ch = ps.current())) { + while ((ch = ps.currentChar)) { if (ch === "{") { return new AST.TextElement(buffer); } @@ -505,7 +509,7 @@ export default class FluentParser { } getEscapeSequence(ps, specials = ["{", "\\"]) { - const next = ps.current(); + const next = ps.currentChar; if (specials.includes(next)) { ps.next(); @@ -520,7 +524,7 @@ export default class FluentParser { const ch = ps.takeHexDigit(); if (ch === undefined) { - throw new ParseError("E0026", sequence + ps.current()); + throw new ParseError("E0026", sequence + ps.currentChar); } sequence += ch; @@ -546,10 +550,10 @@ export default class FluentParser { ps.skipBlank(); - if (ps.currentIs("-")) { + if (ps.currentChar === "-") { ps.peek(); - if (!ps.currentPeekIs(">")) { + if (ps.currentPeek !== ">") { ps.resetPeek(); return selector; } @@ -598,7 +602,7 @@ export default class FluentParser { } getSelectorExpression(ps) { - if (ps.currentIs("{")) { + if (ps.currentChar === "{") { return this.getPlaceable(ps); } const literal = this.getLiteral(ps); @@ -608,7 +612,7 @@ export default class FluentParser { return literal; } - const ch = ps.current(); + const ch = ps.currentChar; if (ch === ".") { ps.next(); @@ -662,7 +666,7 @@ export default class FluentParser { ps.skipBlank(); - if (ps.current() !== ":") { + if (ps.currentChar !== ":") { return exp; } @@ -686,7 +690,7 @@ export default class FluentParser { ps.skipBlank(); while (true) { - if (ps.current() === ")") { + if (ps.currentChar === ")") { break; } @@ -705,7 +709,7 @@ export default class FluentParser { ps.skipBlank(); - if (ps.current() === ",") { + if (ps.currentChar === ",") { ps.next(); ps.skipBlank(); continue; @@ -722,7 +726,7 @@ export default class FluentParser { getArgVal(ps) { if (ps.isNumberStart()) { return this.getNumber(ps); - } else if (ps.currentIs('"')) { + } else if (ps.currentChar === '"') { return this.getString(ps); } throw new ParseError("E0012"); @@ -742,7 +746,7 @@ export default class FluentParser { } } - if (ps.currentIs("\n")) { + if (ps.currentChar === "\n") { throw new ParseError("E0020"); } @@ -753,7 +757,7 @@ export default class FluentParser { } getLiteral(ps) { - const ch = ps.current(); + const ch = ps.currentChar; if (!ch) { throw new ParseError("E0014"); diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js index 0eb281c5f..8ecb23fa0 100644 --- a/fluent-syntax/src/stream.js +++ b/fluent-syntax/src/stream.js @@ -1,130 +1,35 @@ export class ParserStream { constructor(string) { this.string = string; - this.iter = string[Symbol.iterator](); - this.buf = []; - this.peekIndex = 0; this.index = 0; - - this.iterEnd = false; - this.peekEnd = false; - - this.ch = this.iter.next().value; + this.peekOffset = 0; } - next() { - if (this.iterEnd) { - return undefined; - } - - if (this.buf.length === 0) { - this.ch = this.iter.next().value; - } else { - this.ch = this.buf.shift(); - } - - this.index++; - - if (this.ch === undefined) { - this.iterEnd = true; - this.peekEnd = true; - } - - this.peekIndex = this.index; - - return this.ch; + get currentChar() { + return this.string[this.index]; } - current() { - return this.ch; + get currentPeek() { + return this.string[this.index + this.peekOffset]; } - currentIs(ch) { - return this.ch === ch; - } - - currentPeek() { - if (this.peekEnd) { - return undefined; - } - - const diff = this.peekIndex - this.index; - - if (diff === 0) { - return this.ch; - } - return this.buf[diff - 1]; - } - - currentPeekIs(ch) { - return this.currentPeek() === ch; + next() { + this.index++; + this.peekOffset = 0; + return this.string[this.index]; } peek() { - if (this.peekEnd) { - return undefined; - } - - this.peekIndex += 1; - - const diff = this.peekIndex - this.index; - - if (diff > this.buf.length) { - const ch = this.iter.next().value; - if (ch !== undefined) { - this.buf.push(ch); - } else { - this.peekEnd = true; - return undefined; - } - } - - return this.buf[diff - 1]; + this.peekOffset++; + return this.string[this.index + this.peekOffset]; } - getIndex() { - return this.index; - } - - getPeekIndex() { - return this.peekIndex; - } - - peekCharIs(ch) { - if (this.peekEnd) { - return false; - } - - const ret = this.peek(); - - this.peekIndex -= 1; - - return ret === ch; - } - - resetPeek(pos) { - if (pos) { - if (pos < this.peekIndex) { - this.peekEnd = false; - } - this.peekIndex = pos; - } else { - this.peekIndex = this.index; - this.peekEnd = this.iterEnd; - } + resetPeek(offset = 0) { + this.peekOffset = offset; } skipToPeek() { - const diff = this.peekIndex - this.index; - - for (let i = 0; i < diff; i++) { - this.ch = this.buf.shift(); - } - - this.index = this.peekIndex; - } - - getSlice(start, end) { - return this.string.substring(start, end); + this.index += this.peekOffset; + this.peekOffset = 0; } } diff --git a/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl index aacdc70fe..7ece456be 100644 --- a/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl +++ b/fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl @@ -11,4 +11,4 @@ key4 = { { foo } # ~ERROR E0003, pos 96, args "}" -# key5 = { foo } } +key5 = { foo } } diff --git a/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl b/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl index 350b6b0d1..5f812a949 100644 --- a/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl +++ b/fluent-syntax/test/fixtures_behavior/unclosed_empty_placeable_error.ftl @@ -1,2 +1,5 @@ -bar = Bar { -# ~ERROR E0014, pos 12 +# ~ERROR E0003, pos 8, args "}" +foo = { +bar = Bar +# ~ERROR E0014, pos 26 +baz = { diff --git a/fluent-syntax/test/fixtures_structure/multiline_pattern.ftl b/fluent-syntax/test/fixtures_structure/multiline_pattern.ftl new file mode 100644 index 000000000..41611a04b --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/multiline_pattern.ftl @@ -0,0 +1,20 @@ +key01 = Value + Continued here. + +key02 = + Value + Continued here. + +# ERROR "Continued" looks like a new message. +# key03 parses fine with just "Value". +key03 = + Value +Continued here + and here. + +# ERROR "Continued" and "and" look like new messages +# key04 parses fine with just "Value". +key04 = + Value +Continued here +and even here. diff --git a/fluent-syntax/test/fixtures_structure/multiline_pattern.json b/fluent-syntax/test/fixtures_structure/multiline_pattern.json new file mode 100644 index 000000000..1793692b6 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/multiline_pattern.json @@ -0,0 +1,250 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01", + "span": { + "type": "Span", + "start": 0, + "end": 5 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued here.", + "span": { + "type": "Span", + "start": 8, + "end": 33 + } + } + ], + "span": { + "type": "Span", + "start": 8, + "end": 33 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 0, + "end": 33 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02", + "span": { + "type": "Span", + "start": 35, + "end": 40 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued here.", + "span": { + "type": "Span", + "start": 47, + "end": 72 + } + } + ], + "span": { + "type": "Span", + "start": 47, + "end": 72 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 35, + "end": 72 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03", + "span": { + "type": "Span", + "start": 159, + "end": 164 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value", + "span": { + "type": "Span", + "start": 171, + "end": 176 + } + } + ], + "span": { + "type": "Span", + "start": 171, + "end": 176 + } + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR \"Continued\" looks like a new message.\nkey03 parses fine with just \"Value\".", + "span": { + "type": "Span", + "start": 74, + "end": 158 + } + }, + "span": { + "type": "Span", + "start": 74, + "end": 176 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "=" + ], + "message": "Expected token: \"=\"", + "span": { + "type": "Span", + "start": 187, + "end": 187 + } + } + ], + "content": "Continued here\n and here.\n\n", + "span": { + "type": "Span", + "start": 177, + "end": 207 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04", + "span": { + "type": "Span", + "start": 299, + "end": 304 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value", + "span": { + "type": "Span", + "start": 311, + "end": 316 + } + } + ], + "span": { + "type": "Span", + "start": 311, + "end": 316 + } + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR \"Continued\" and \"and\" look like new messages\nkey04 parses fine with just \"Value\".", + "span": { + "type": "Span", + "start": 207, + "end": 298 + } + }, + "span": { + "type": "Span", + "start": 207, + "end": 316 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "=" + ], + "message": "Expected token: \"=\"", + "span": { + "type": "Span", + "start": 327, + "end": 327 + } + } + ], + "content": "Continued here\n", + "span": { + "type": "Span", + "start": 317, + "end": 332 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "=" + ], + "message": "Expected token: \"=\"", + "span": { + "type": "Span", + "start": 336, + "end": 336 + } + } + ], + "content": "and even here.\n", + "span": { + "type": "Span", + "start": 332, + "end": 347 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 347 + } +} diff --git a/fluent-syntax/test/fixtures_structure/unclosed.ftl b/fluent-syntax/test/fixtures_structure/unclosed.ftl new file mode 100644 index 000000000..0fe9fd442 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/unclosed.ftl @@ -0,0 +1,12 @@ +err01 = { +key02 = Value 02 + +err03 = { +FUNC( +arg +, +namedArg: "Value" +, +key04 = Value 04 +) +} diff --git a/fluent-syntax/test/fixtures_structure/unclosed.json b/fluent-syntax/test/fixtures_structure/unclosed.json new file mode 100644 index 000000000..f492a5e09 --- /dev/null +++ b/fluent-syntax/test/fixtures_structure/unclosed.json @@ -0,0 +1,154 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 10, + "end": 10 + } + } + ], + "content": "err01 = {\n", + "span": { + "type": "Span", + "start": 0, + "end": 10 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02", + "span": { + "type": "Span", + "start": 10, + "end": 15 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 02", + "span": { + "type": "Span", + "start": 18, + "end": 26 + } + } + ], + "span": { + "type": "Span", + "start": 18, + "end": 26 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 10, + "end": 26 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0021", + "args": [], + "message": "Positional arguments must not follow named arguments", + "span": { + "type": "Span", + "start": 70, + "end": 70 + } + } + ], + "content": "err03 = {\nFUNC(\narg\n,\nnamedArg: \"Value\"\n,\n", + "span": { + "type": "Span", + "start": 28, + "end": 70 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04", + "span": { + "type": "Span", + "start": 70, + "end": 75 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 04", + "span": { + "type": "Span", + "start": 78, + "end": 86 + } + } + ], + "span": { + "type": "Span", + "start": 78, + "end": 86 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 70, + "end": 86 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0002", + "args": [], + "message": "Expected an entry start", + "span": { + "type": "Span", + "start": 87, + "end": 87 + } + } + ], + "content": ")\n}\n", + "span": { + "type": "Span", + "start": 87, + "end": 91 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 91 + } +} diff --git a/fluent-syntax/test/stream_test.js b/fluent-syntax/test/stream_test.js index 25987b7f2..3c4f1fb32 100644 --- a/fluent-syntax/test/stream_test.js +++ b/fluent-syntax/test/stream_test.js @@ -7,93 +7,93 @@ suite("ParserStream", function() { test("next", function() { let ps = new ParserStream("abcd"); - assert.strictEqual("a", ps.current()); - assert.strictEqual(0, ps.getIndex()); + assert.strictEqual("a", ps.currentChar); + assert.strictEqual(0, ps.index); assert.strictEqual("b", ps.next()); - assert.strictEqual("b", ps.current()); - assert.strictEqual(1, ps.getIndex()); + assert.strictEqual("b", ps.currentChar); + assert.strictEqual(1, ps.index); assert.strictEqual("c", ps.next()); - assert.strictEqual("c", ps.current()); - assert.strictEqual(2, ps.getIndex()); + assert.strictEqual("c", ps.currentChar); + assert.strictEqual(2, ps.index); assert.strictEqual("d", ps.next()); - assert.strictEqual("d", ps.current()); - assert.strictEqual(3, ps.getIndex()); + assert.strictEqual("d", ps.currentChar); + assert.strictEqual(3, ps.index); assert.strictEqual(undefined, ps.next()); - assert.strictEqual(undefined, ps.current()); - assert.strictEqual(4, ps.getIndex()); + assert.strictEqual(undefined, ps.currentChar); + assert.strictEqual(4, ps.index); }); test("peek", function() { let ps = new ParserStream("abcd"); - assert.strictEqual("a", ps.currentPeek()); - assert.strictEqual(0, ps.getPeekIndex()); + assert.strictEqual("a", ps.currentPeek); + assert.strictEqual(0, ps.peekOffset); assert.strictEqual("b", ps.peek()); - assert.strictEqual("b", ps.currentPeek()); - assert.strictEqual(1, ps.getPeekIndex()); + assert.strictEqual("b", ps.currentPeek); + assert.strictEqual(1, ps.peekOffset); assert.strictEqual("c", ps.peek()); - assert.strictEqual("c", ps.currentPeek()); - assert.strictEqual(2, ps.getPeekIndex()); + assert.strictEqual("c", ps.currentPeek); + assert.strictEqual(2, ps.peekOffset); assert.strictEqual("d", ps.peek()); - assert.strictEqual("d", ps.currentPeek()); - assert.strictEqual(3, ps.getPeekIndex()); + assert.strictEqual("d", ps.currentPeek); + assert.strictEqual(3, ps.peekOffset); assert.strictEqual(undefined, ps.peek()); - assert.strictEqual(undefined, ps.currentPeek()); - assert.strictEqual(4, ps.getPeekIndex()); + assert.strictEqual(undefined, ps.currentPeek); + assert.strictEqual(4, ps.peekOffset); }); test("peek_and_next", function() { let ps = new ParserStream("abcd"); assert.strictEqual("b", ps.peek()); - assert.strictEqual(1, ps.getPeekIndex()); - assert.strictEqual(0, ps.getIndex()); + assert.strictEqual(1, ps.peekOffset); + assert.strictEqual(0, ps.index); assert.strictEqual("b", ps.next()); - assert.strictEqual(1, ps.getPeekIndex()); - assert.strictEqual(1, ps.getIndex()); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(1, ps.index); assert.strictEqual("c", ps.peek()); - assert.strictEqual(2, ps.getPeekIndex()); - assert.strictEqual(1, ps.getIndex()); + assert.strictEqual(1, ps.peekOffset); + assert.strictEqual(1, ps.index); assert.strictEqual("c", ps.next()); - assert.strictEqual(2, ps.getPeekIndex()); - assert.strictEqual(2, ps.getIndex()); - assert.strictEqual("c", ps.current()); - assert.strictEqual("c", ps.currentPeek()); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(2, ps.index); + assert.strictEqual("c", ps.currentChar); + assert.strictEqual("c", ps.currentPeek); assert.strictEqual("d", ps.peek()); - assert.strictEqual(3, ps.getPeekIndex()); - assert.strictEqual(2, ps.getIndex()); + assert.strictEqual(1, ps.peekOffset); + assert.strictEqual(2, ps.index); assert.strictEqual("d", ps.next()); - assert.strictEqual(3, ps.getPeekIndex()); - assert.strictEqual(3, ps.getIndex()); - assert.strictEqual("d", ps.current()); - assert.strictEqual("d", ps.currentPeek()); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(3, ps.index); + assert.strictEqual("d", ps.currentChar); + assert.strictEqual("d", ps.currentPeek); assert.strictEqual(undefined, ps.peek()); - assert.strictEqual(4, ps.getPeekIndex()); - assert.strictEqual(3, ps.getIndex()); - assert.strictEqual("d", ps.current()); - assert.strictEqual(undefined, ps.currentPeek()); + assert.strictEqual(1, ps.peekOffset); + assert.strictEqual(3, ps.index); + assert.strictEqual("d", ps.currentChar); + assert.strictEqual(undefined, ps.currentPeek); assert.strictEqual(undefined, ps.peek()); - assert.strictEqual(4, ps.getPeekIndex()); - assert.strictEqual(3, ps.getIndex()); + assert.strictEqual(2, ps.peekOffset); + assert.strictEqual(3, ps.index); assert.strictEqual(undefined, ps.next()); - assert.strictEqual(4, ps.getPeekIndex()); - assert.strictEqual(4, ps.getIndex()); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(4, ps.index); }); test("skip_to_peek", function() { @@ -104,24 +104,24 @@ suite("ParserStream", function() { ps.skipToPeek(); - assert.strictEqual("c", ps.current()); - assert.strictEqual("c", ps.currentPeek()); - assert.strictEqual(2, ps.getPeekIndex()); - assert.strictEqual(2, ps.getIndex()); + assert.strictEqual("c", ps.currentChar); + assert.strictEqual("c", ps.currentPeek); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(2, ps.index); ps.peek(); - assert.strictEqual("c", ps.current()); - assert.strictEqual("d", ps.currentPeek()); - assert.strictEqual(3, ps.getPeekIndex()); - assert.strictEqual(2, ps.getIndex()); + assert.strictEqual("c", ps.currentChar); + assert.strictEqual("d", ps.currentPeek); + assert.strictEqual(1, ps.peekOffset); + assert.strictEqual(2, ps.index); ps.next(); - assert.strictEqual("d", ps.current()); - assert.strictEqual("d", ps.currentPeek()); - assert.strictEqual(3, ps.getPeekIndex()); - assert.strictEqual(3, ps.getIndex()); + assert.strictEqual("d", ps.currentChar); + assert.strictEqual("d", ps.currentPeek); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(3, ps.index); }); test("reset_peek", function() { @@ -132,51 +132,35 @@ suite("ParserStream", function() { ps.peek(); ps.resetPeek(); - assert.strictEqual("b", ps.current()); - assert.strictEqual("b", ps.currentPeek()); - assert.strictEqual(1, ps.getPeekIndex()); - assert.strictEqual(1, ps.getIndex()); + assert.strictEqual("b", ps.currentChar); + assert.strictEqual("b", ps.currentPeek); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(1, ps.index); ps.peek(); - assert.strictEqual("b", ps.current()); - assert.strictEqual("c", ps.currentPeek()); - assert.strictEqual(2, ps.getPeekIndex()); - assert.strictEqual(1, ps.getIndex()); + assert.strictEqual("b", ps.currentChar); + assert.strictEqual("c", ps.currentPeek); + assert.strictEqual(1, ps.peekOffset); + assert.strictEqual(1, ps.index); ps.peek(); ps.peek(); ps.peek(); ps.resetPeek(); - assert.strictEqual("b", ps.current()); - assert.strictEqual("b", ps.currentPeek()); - assert.strictEqual(1, ps.getPeekIndex()); - assert.strictEqual(1, ps.getIndex()); + assert.strictEqual("b", ps.currentChar); + assert.strictEqual("b", ps.currentPeek); + assert.strictEqual(0, ps.peekOffset); + assert.strictEqual(1, ps.index); assert.strictEqual("c", ps.peek()); - assert.strictEqual("b", ps.current()); - assert.strictEqual("c", ps.currentPeek()); - assert.strictEqual(2, ps.getPeekIndex()); - assert.strictEqual(1, ps.getIndex()); + assert.strictEqual("b", ps.currentChar); + assert.strictEqual("c", ps.currentPeek); + assert.strictEqual(1, ps.peekOffset); + assert.strictEqual(1, ps.index); assert.strictEqual("d", ps.peek()); assert.strictEqual(undefined, ps.peek()); }); - - test("peek_char_is", function() { - let ps = new ParserStream("abcd"); - - ps.next(); - ps.peek(); - - assert.strictEqual(true, ps.peekCharIs("d")); - - assert.strictEqual("b", ps.current()); - assert.strictEqual("c", ps.currentPeek()); - - ps.skipToPeek(); - - assert.strictEqual("c", ps.current()); - }); }); diff --git a/fluent/test/fixtures_behavior/placeable_in_placeable.json b/fluent/test/fixtures_behavior/placeable_in_placeable.json index 0e54ab88d..824346581 100644 --- a/fluent/test/fixtures_behavior/placeable_in_placeable.json +++ b/fluent/test/fixtures_behavior/placeable_in_placeable.json @@ -4,5 +4,8 @@ }, "key2": { "value": true + }, + "key5": { + "value": true } } diff --git a/fluent/test/fixtures_behavior/unclosed_empty_placeable_error.json b/fluent/test/fixtures_behavior/unclosed_empty_placeable_error.json index 0967ef424..8e7a33617 100644 --- a/fluent/test/fixtures_behavior/unclosed_empty_placeable_error.json +++ b/fluent/test/fixtures_behavior/unclosed_empty_placeable_error.json @@ -1 +1,5 @@ -{} +{ + "bar": { + "value": true + } +} diff --git a/fluent/test/fixtures_structure/multiline_pattern.json b/fluent/test/fixtures_structure/multiline_pattern.json new file mode 100644 index 000000000..55d761b28 --- /dev/null +++ b/fluent/test/fixtures_structure/multiline_pattern.json @@ -0,0 +1,18 @@ +{ + "key01": [ + "Value", + "\n", + "Continued here." + ], + "key02": [ + "Value", + "\n", + "Continued here." + ], + "key03": [ + "Value" + ], + "key04": [ + "Value" + ] +} diff --git a/fluent/test/fixtures_structure/unclosed.json b/fluent/test/fixtures_structure/unclosed.json new file mode 100644 index 000000000..60d0cfe59 --- /dev/null +++ b/fluent/test/fixtures_structure/unclosed.json @@ -0,0 +1,4 @@ +{ + "key02": "Value 02", + "key04": "Value 04" +}