diff --git a/fluent-syntax/src/ast.js b/fluent-syntax/src/ast.js index 585eb7836..6a1a583db 100644 --- a/fluent-syntax/src/ast.js +++ b/fluent-syntax/src/ast.js @@ -19,11 +19,10 @@ class SyntaxNode extends BaseNode { } export class Resource extends SyntaxNode { - constructor(body = [], comment = null) { + constructor(body = []) { super(); this.type = 'Resource'; this.body = body; - this.comment = comment; } } @@ -192,20 +191,31 @@ export class Symbol extends Identifier { } } -export class Comment extends Entry { +export class BaseComment extends Entry { constructor(content) { super(); - this.type = 'Comment'; + this.type = 'BaseComment'; this.content = content; } } -export class Section extends Entry { - constructor(name, comment = null) { - super(); - this.type = 'Section'; - this.name = name; - this.comment = comment; +export class Comment extends BaseComment { + constructor(content) { + super(content); + this.type = 'Comment'; + } +} + +export class GroupComment extends BaseComment { + constructor(content) { + super(content); + this.type = 'GroupComment'; + } +} +export class ResourceComment extends BaseComment { + constructor(content) { + super(content); + this.type = 'ResourceComment'; } } diff --git a/fluent-syntax/src/ftlstream.js b/fluent-syntax/src/ftlstream.js index 108794212..0e72671ca 100644 --- a/fluent-syntax/src/ftlstream.js +++ b/fluent-syntax/src/ftlstream.js @@ -109,7 +109,7 @@ export class FTLParserStream extends ParserStream { return ((cc >= 48 && cc <= 57) || cc === 45); // 0-9 } - isPeekNextLineComment() { + isPeekNextLineZeroFourStyleComment() { if (!this.currentPeekIs('\n')) { return false; } @@ -128,6 +128,39 @@ export class FTLParserStream extends ParserStream { return false; } + // -1 - any + // 0 - comment + // 1 - group comment + // 2 - resource comment + isPeekNextLineComment(level = -1) { + if (!this.currentPeekIs('\n')) { + return false; + } + + let i = 0; + + while (i <= level || (level === -1 && i < 3)) { + this.peek(); + if (!this.currentPeekIs('#')) { + if (i !== level && level !== -1) { + this.resetPeek(); + return false; + } + break; + } + i++; + } + + this.peek(); + if ([' ', '\n'].includes(this.currentPeek())) { + this.resetPeek(); + return true; + } + + this.resetPeek(); + return false; + } + isPeekNextLineVariantStart() { if (!this.currentPeekIs('\n')) { return false; diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js index 6565ffba6..a7e0c906b 100644 --- a/fluent-syntax/src/parser.js +++ b/fluent-syntax/src/parser.js @@ -21,7 +21,7 @@ function withSpan(fn) { } // Spans of Messages and Sections should include the attached Comment. - if (node.type === 'Message' || node.type === 'Section') { + if (node.type === 'Message') { if (node.comment !== null) { start = node.comment.span.start; } @@ -42,7 +42,7 @@ export default class FluentParser { // Poor man's decorators. [ - 'getComment', 'getSection', 'getMessage', 'getAttribute', + 'getComment', 'getMessage', 'getAttribute', 'getIdentifier', 'getVariant', 'getSymbol', 'getNumber', 'getPattern', 'getTextElement', 'getPlaceable', 'getExpression', 'getSelectorExpression', 'getCallArg', 'getString', 'getLiteral', @@ -52,8 +52,6 @@ export default class FluentParser { } parse(source) { - let comment = null; - const ps = new FTLParserStream(source); ps.skipBlankLines(); @@ -62,8 +60,16 @@ export default class FluentParser { while (ps.current()) { const entry = this.getEntryOrJunk(ps); - if (entry.type === 'Comment' && entries.length === 0) { - comment = entry; + if (entry === null) { + // That happens when we get a 0.4 style section + continue; + } + + if (entry.type === 'Comment' && + entry.zeroFourStyle && entries.length === 0) { + const comment = new AST.ResourceComment(entry.content); + comment.span = entry.span; + entries.push(comment); } else { entries.push(entry); } @@ -72,7 +78,7 @@ export default class FluentParser { ps.skipBlankLines(); } - const res = new AST.Resource(entries, comment); + const res = new AST.Resource(entries); if (this.withSpans) { res.addSpan(0, ps.getIndex()); @@ -117,7 +123,7 @@ export default class FluentParser { getEntry(ps) { let comment; - if (ps.currentIs('/')) { + if (ps.currentIs('/') || ps.currentIs('#')) { comment = this.getComment(ps); // The Comment content doesn't include the trailing newline. Consume @@ -127,20 +133,25 @@ export default class FluentParser { } if (ps.currentIs('[')) { - return this.getSection(ps, comment); + this.skipSection(ps); + if (comment) { + return new AST.GroupComment(comment.content); + } + return null; } - if (ps.isIDStart()) { + if (ps.isIDStart() && (!comment || comment.type === 'Comment')) { return this.getMessage(ps, comment); } if (comment) { return comment; } + throw new ParseError('E0002'); } - getComment(ps) { + getZeroFourStyleComment(ps) { ps.expectChar('/'); ps.expectChar('/'); ps.takeCharIf(' '); @@ -153,7 +164,7 @@ export default class FluentParser { content += ch; } - if (ps.isPeekNextLineComment()) { + if (ps.isPeekNextLineZeroFourStyleComment()) { content += '\n'; ps.next(); ps.expectChar('/'); @@ -163,23 +174,80 @@ export default class FluentParser { break; } } - return new AST.Comment(content); + + const comment = new AST.Comment(content); + comment.zeroFourStyle = true; + return comment; } - getSection(ps, comment) { + getComment(ps) { + if (ps.currentIs('/')) { + return this.getZeroFourStyleComment(ps); + } + + // 0 - comment + // 1 - group comment + // 2 - resource comment + let level = -1; + let content = ''; + + while (true) { + let i = -1; + while (ps.currentIs('#') && (i < (level === -1 ? 2 : level))) { + ps.next(); + i++; + } + + if (level === -1) { + level = i; + } + + if (!ps.currentIs('\n')) { + ps.expectChar(' '); + let ch; + while ((ch = ps.takeChar(x => x !== '\n'))) { + content += ch; + } + } + + if (ps.isPeekNextLineComment(level, false)) { + content += '\n'; + ps.next(); + } else { + break; + } + } + + let Comment; + switch (level) { + case 0: + Comment = AST.Comment; + break; + case 1: + Comment = AST.GroupComment; + break; + case 2: + Comment = AST.ResourceComment; + break; + } + return new Comment(content); + } + + skipSection(ps) { ps.expectChar('['); ps.expectChar('['); ps.skipInlineWS(); - const symb = this.getSymbol(ps); + this.getSymbol(ps); ps.skipInlineWS(); ps.expectChar(']'); ps.expectChar(']'); - return new AST.Section(symb, comment); + ps.skipInlineWS(); + ps.next(); } getMessage(ps, comment) { diff --git a/fluent-syntax/src/serializer.js b/fluent-syntax/src/serializer.js index b2c2a0583..3b4eb215f 100644 --- a/fluent-syntax/src/serializer.js +++ b/fluent-syntax/src/serializer.js @@ -14,6 +14,7 @@ function containNewLine(elems) { export default class FluentSerializer { constructor({ withJunk = false } = {}) { this.withJunk = withJunk; + this.hasEntries = false; } serialize(resource) { @@ -23,11 +24,15 @@ export default class FluentSerializer { parts.push( `${serializeComment(resource.comment)}\n\n` ); + this.hasEntries = true; } for (const entry of resource.body) { if (entry.types !== 'Junk' || this.withJunk) { parts.push(this.serializeEntry(entry)); + if (this.hasEntries === false) { + this.hasEntries = true; + } } } @@ -40,10 +45,21 @@ export default class FluentSerializer { return serializeMessage(entry); case 'Section': return serializeSection(entry); - case 'Comment': { - const comment = serializeComment(entry); - return `\n${comment}\n\n`; - } + case 'Comment': + if (this.hasEntries) { + return `\n${serializeComment(entry)}\n\n`; + } + return `${serializeComment(entry)}\n\n`; + case 'GroupComment': + if (this.hasEntries) { + return `\n${serializeGroupComment(entry)}\n\n`; + } + return `${serializeGroupComment(entry)}\n\n`; + case 'ResourceComment': + if (this.hasEntries) { + return `\n${serializeResourceComment(entry)}\n\n`; + } + return `${serializeResourceComment(entry)}\n\n`; case 'Junk': return serializeJunk(entry); default : @@ -55,10 +71,21 @@ export default class FluentSerializer { function serializeComment(comment) { return comment.content.split('\n').map( - line => `// ${line}` + line => line.length ? `# ${line}` : '#' + ).join('\n'); +} + +function serializeGroupComment(comment) { + return comment.content.split('\n').map( + line => line.length ? `## ${line}` : '##' ).join('\n'); } +function serializeResourceComment(comment) { + return comment.content.split('\n').map( + line => line.length ? `### ${line}` : '###' + ).join('\n'); +} function serializeSection(section) { const name = serializeSymbol(section.name); diff --git a/fluent-syntax/test/fixtures_structure/elements_indent.json b/fluent-syntax/test/fixtures_structure/elements_indent.json index 266e9a5a0..131459a34 100644 --- a/fluent-syntax/test/fixtures_structure/elements_indent.json +++ b/fluent-syntax/test/fixtures_structure/elements_indent.json @@ -161,7 +161,6 @@ } } ], - "comment": null, "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json index eab2feeae..072d6168d 100644 --- a/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json +++ b/fluent-syntax/test/fixtures_structure/message_with_empty_pattern.json @@ -31,7 +31,6 @@ } } ], - "comment": null, "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json index 83ad25308..6b441ea43 100644 --- a/fluent-syntax/test/fixtures_structure/placeable_at_eol.json +++ b/fluent-syntax/test/fixtures_structure/placeable_at_eol.json @@ -203,7 +203,6 @@ } } ], - "comment": null, "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/resource_comment.json b/fluent-syntax/test/fixtures_structure/resource_comment.json index a476434ce..3fc128502 100644 --- a/fluent-syntax/test/fixtures_structure/resource_comment.json +++ b/fluent-syntax/test/fixtures_structure/resource_comment.json @@ -1,16 +1,17 @@ { "type": "Resource", - "body": [], - "comment": { - "type": "Comment", - "annotations": [], - "content": "This is a resource wide comment\nIt's multiline", - "span": { - "type": "Span", - "start": 0, - "end": 52 + "body": [ + { + "type": "ResourceComment", + "annotations": [], + "content": "This is a resource wide comment\nIt's multiline", + "span": { + "type": "Span", + "start": 0, + "end": 52 + } } - }, + ], "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.json b/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.json index a91e00b5b..dbe67ae09 100644 --- a/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.json +++ b/fluent-syntax/test/fixtures_structure/resource_comment_trailing_line.json @@ -1,16 +1,17 @@ { "type": "Resource", - "body": [], - "comment": { - "type": "Comment", - "annotations": [], - "content": "This is a comment\nThis comment is multiline\n", - "span": { - "type": "Span", - "start": 0, - "end": 52 + "body": [ + { + "type": "ResourceComment", + "annotations": [], + "content": "This is a comment\nThis comment is multiline\n", + "span": { + "type": "Span", + "start": 0, + "end": 52 + } } - }, + ], "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/section.json b/fluent-syntax/test/fixtures_structure/section.json index 49b000593..0252e37e2 100644 --- a/fluent-syntax/test/fixtures_structure/section.json +++ b/fluent-syntax/test/fixtures_structure/section.json @@ -1,27 +1,6 @@ { "type": "Resource", - "body": [ - { - "type": "Section", - "annotations": [], - "name": { - "type": "Symbol", - "name": "This is a section", - "span": { - "type": "Span", - "start": 4, - "end": 22 - } - }, - "comment": null, - "span": { - "type": "Span", - "start": 1, - "end": 24 - } - } - ], - "comment": null, + "body": [], "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/simple_message.json b/fluent-syntax/test/fixtures_structure/simple_message.json index 425de818a..5cbe78dde 100644 --- a/fluent-syntax/test/fixtures_structure/simple_message.json +++ b/fluent-syntax/test/fixtures_structure/simple_message.json @@ -41,7 +41,6 @@ } } ], - "comment": null, "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/sparse-messages.json b/fluent-syntax/test/fixtures_structure/sparse-messages.json index 97882cbd0..d5b57b48b 100644 --- a/fluent-syntax/test/fixtures_structure/sparse-messages.json +++ b/fluent-syntax/test/fixtures_structure/sparse-messages.json @@ -337,7 +337,6 @@ } } ], - "comment": null, "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/fixtures_structure/standalone_comment.json b/fluent-syntax/test/fixtures_structure/standalone_comment.json index 2775e6031..3140395fe 100644 --- a/fluent-syntax/test/fixtures_structure/standalone_comment.json +++ b/fluent-syntax/test/fixtures_structure/standalone_comment.json @@ -44,6 +44,7 @@ "type": "Comment", "annotations": [], "content": "This is a standalone comment", + "zeroFourStyle": true, "span": { "type": "Span", "start": 13, @@ -51,7 +52,6 @@ } } ], - "comment": null, "span": { "type": "Span", "start": 0, diff --git a/fluent-syntax/test/serializer_test.js b/fluent-syntax/test/serializer_test.js index b51f34d71..71b90c752 100644 --- a/fluent-syntax/test/serializer_test.js +++ b/fluent-syntax/test/serializer_test.js @@ -76,22 +76,10 @@ suite('Serializer', function() { assert.equal(pretty(input), input); }); - test('section', function() { - const input = ftl` - foo = Foo - - - [[ Section Header ]] - - bar = Bar - `; - assert.equal(pretty(input), input); - }); - test('resource comment', function() { const input = ftl` - // A multiline - // resource comment. + ### A multiline + ### resource comment. foo = Foo `; @@ -100,21 +88,21 @@ suite('Serializer', function() { test('message comment', function() { const input = ftl` - // A multiline - // message comment. + # A multiline + # message comment. foo = Foo `; assert.equal(pretty(input), input); }); - test('section comment', function() { + test('group comment', function() { const input = ftl` foo = Foo - - // A multiline - // section comment. - [[ Section Header ]] + ## Comment Header + ## + ## A multiline + ## section comment. bar = Bar `; @@ -125,7 +113,7 @@ suite('Serializer', function() { const input = ftl` foo = Foo - // A Standalone Comment + # A Standalone Comment bar = Bar `; diff --git a/fluent/src/parser.js b/fluent/src/parser.js index bc3efa839..1990ced06 100644 --- a/fluent/src/parser.js +++ b/fluent/src/parser.js @@ -72,7 +72,8 @@ class RuntimeParser { const ch = this._source[this._index]; // We don't care about comments or sections at runtime - if (ch === '/') { + if (ch === '/' || + (ch === '#' && [' ', '#'].includes(this._source[this._index + 1]))) { this.skipComment(); return; } @@ -839,7 +840,9 @@ class RuntimeParser { let eol = this._source.indexOf('\n', this._index); while (eol !== -1 && - this._source[eol + 1] === '/' && this._source[eol + 2] === '/') { + ((this._source[eol + 1] === '/' && this._source[eol + 2] === '/') || + (this._source[eol + 1] === '#' && + [' ', '#'].includes(this._source[eol + 2])))) { this._index = eol + 3; eol = this._source.indexOf('\n', this._index);