Require = after the id (but not really)

In Fluent Syntax 0.5, the identifier must be followed by an equals sign (=)
even if the message doesn't have a value.

    key =
        .attribute = Attribute Value

For some time fluent.js will allow both the new (0.5) and the old 0.4 syntax,
which means that the following will still parse but the Serializer will insert
the = after the key.

    key
        .attribute = Attribute Value

This patch also fixes Bug 1406880 - Decide where multiline Pattern spans should
start. The patterns now start when their first

    key =
        Value
        ^---- Pattern span start

Lastly, this patch forbids null variant and attribute values in the tooling
parser. Empty values should be written with the {""} idiom:

    key =
        {
           *[valid] {""}
            [invalid]
        }

The runtime parser parses them as {val: null} for performance reasons.

Author: stasm

fluent-react/examples/text-input/src/l10n.js

@@ -6,13 +6,13 @@ const MESSAGES_ALL = {
   'pl': `
 hello = Cześć { $username }!
 hello-no-name = Witaj nieznajomy!
-type-name
+type-name =
     .placeholder = Twoje imię
   `,
   'en-US': `
 hello = Hello, { $username }!
 hello-no-name = Hello, stranger!
-type-name
+type-name =
     .placeholder = Your name
   `,
 };

fluent-syntax/src/ftlstream.js

@@ -5,8 +5,18 @@ import { ParseError } from './errors';
 import { includes } from './util';
 
 const INLINE_WS = [' ', '\t'];
+const SPECIAL_LINE_START_CHARS = ['}', '.', '[', '*'];
 
 export class FTLParserStream extends ParserStream {
+  skipInlineWS() {
+    while (this.ch) {
+      if (!includes(INLINE_WS, this.ch)) {
+        break;
+      }
+      this.next();
+    }
+  }
+
   peekInlineWS() {
     let ch = this.currentPeek();
     while (ch) {
@@ -46,13 +56,9 @@ export class FTLParserStream extends ParserStream {
     }
   }
 
-  skipInlineWS() {
-    while (this.ch) {
-      if (!includes(INLINE_WS, this.ch)) {
-        break;
-      }
-      this.next();
-    }
+  skipIndent() {
+    this.skipBlankLines();
+    this.skipInlineWS();
   }
 
   expectChar(ch) {
@@ -125,6 +131,24 @@ export class FTLParserStream extends ParserStream {
     return isDigit;
   }
 
+  isCharPatternStart(ch) {
+    return !includes(SPECIAL_LINE_START_CHARS, ch);
+  }
+
+  isPeekPatternStart() {
+    this.peekInlineWS();
+
+    const ch = this.currentPeek();
+
+    if (ch === '\n') {
+      return this.isPeekNextLinePatternStart();
+    }
+
+    const isPattern = this.isCharPatternStart(this.currentPeek());
+    this.resetPeek();
+    return isPattern;
+  }
+
   isPeekNextLineZeroFourStyleComment() {
     if (!this.currentPeekIs('\n')) {
       return false;
@@ -234,7 +258,7 @@ export class FTLParserStream extends ParserStream {
     return false;
   }
 
-  isPeekNextNonBlankLinePattern() {
+  isPeekNextLinePatternStart() {
     if (!this.currentPeekIs('\n')) {
       return false;
     }
@@ -252,10 +276,7 @@ export class FTLParserStream extends ParserStream {
       return false;
     }
 
-    if (this.currentPeekIs('}') ||
-        this.currentPeekIs('.') ||
-        this.currentPeekIs('[') ||
-        this.currentPeekIs('*')) {
+    if (!this.isCharPatternStart(this.currentPeek())) {
       this.resetPeek();
       return false;
     }

fluent-syntax/src/parser.js

@@ -74,7 +74,6 @@ export default class FluentParser {
         entries.push(entry);
       }
 
-      ps.skipInlineWS();
       ps.skipBlankLines();
     }
 
@@ -258,12 +257,15 @@ export default class FluentParser {
     let pattern;
     let attrs;
 
+    // XXX Syntax 0.4 compatibility.
+    // XXX Replace with ps.expectChar('=').
     if (ps.currentIs('=')) {
       ps.next();
-      ps.skipInlineWS();
-      ps.skipBlankLines();
 
-      pattern = this.getPattern(ps);
+      if (ps.isPeekPatternStart()) {
+        ps.skipIndent();
+        pattern = this.getPattern(ps);
+      }
     }
 
     if (ps.isPeekNextLineAttributeStart()) {
@@ -278,29 +280,27 @@ export default class FluentParser {
   }
 
   getAttribute(ps) {
+    ps.expectIndent();
     ps.expectChar('.');
 
     const key = this.getPublicIdentifier(ps);
 
     ps.skipInlineWS();
     ps.expectChar('=');
-    ps.skipInlineWS();
-
-    const value = this.getPattern(ps);
 
-    if (value === undefined) {
-      throw new ParseError('E0006', 'value');
+    if (ps.isPeekPatternStart()) {
+      ps.skipIndent();
+      const value = this.getPattern(ps);
+      return new AST.Attribute(key, value);
     }
 
-    return new AST.Attribute(key, value);
+    throw new ParseError('E0006', 'value');
   }
 
   getAttributes(ps) {
     const attrs = [];
 
     while (true) {
-      ps.expectIndent();
-
       const attr = this.getAttribute(ps);
       attrs.push(attr);
 
@@ -349,6 +349,8 @@ export default class FluentParser {
   }
 
   getVariant(ps, hasDefault) {
+    ps.expectIndent();
+
     let defaultIndex = false;
 
     if (ps.currentIs('*')) {
@@ -366,24 +368,20 @@ export default class FluentParser {
 
     ps.expectChar(']');
 
-    ps.skipInlineWS();
-
-    const value = this.getPattern(ps);
-
-    if (!value) {
-      throw new ParseError('E0006', 'value');
+    if (ps.isPeekPatternStart()) {
+      ps.skipIndent();
+      const value = this.getPattern(ps);
+      return new AST.Variant(key, value, defaultIndex);
     }
 
-    return new AST.Variant(key, value, defaultIndex);
+    throw new ParseError('E0006', 'value');
   }
 
   getVariants(ps) {
     const variants = [];
     let hasDefault = false;
 
     while (true) {
-      ps.expectIndent();
-
       const variant = this.getVariant(ps, hasDefault);
 
       if (variant.default) {
@@ -459,18 +457,12 @@ export default class FluentParser {
     const elements = [];
     ps.skipInlineWS();
 
-    // Special-case: trim leading whitespace and newlines.
-    if (ps.isPeekNextNonBlankLinePattern()) {
-      ps.skipBlankLines();
-      ps.skipInlineWS();
-    }
-
     let ch;
     while ((ch = ps.current())) {
 
       // The end condition for getPattern's while loop is a newline
       // which is not followed by a valid pattern continuation.
-      if (ch === '\n' && !ps.isPeekNextNonBlankLinePattern()) {
+      if (ch === '\n' && !ps.isPeekNextLinePatternStart()) {
         break;
       }
 
@@ -491,13 +483,12 @@ export default class FluentParser {
 
     let ch;
     while ((ch = ps.current())) {
-
       if (ch === '{') {
         return new AST.TextElement(buffer);
       }
 
       if (ch === '\n') {
-        if (!ps.isPeekNextNonBlankLinePattern()) {
+        if (!ps.isPeekNextLinePatternStart()) {
           return new AST.TextElement(buffer);
         }
 

fluent-syntax/src/serializer.js

@@ -113,9 +113,9 @@ function serializeMessage(message) {
   }
 
   parts.push(serializeIdentifier(message.id));
+  parts.push(' =');
 
   if (message.value) {
-    parts.push(' =');
     parts.push(serializeValue(message.value));
   }
 

fluent-syntax/test/fixtures_behavior/attribute_with_empty_pattern.ftl

@@ -1,2 +1,3 @@
 key = Value
     .label =
+//~ ERROR E0006, pos 24, args "value"

fluent-syntax/test/fixtures_behavior/variant_with_empty_pattern.ftl

@@ -1,3 +1,8 @@
-key = {
+key1 = {
+    *[one] {""}
+    }
+
+err1 = {
     *[one]
     }
+//~ ERROR E0006, pos 51, args "value"

fluent-syntax/test/fixtures_structure/attribute_with_empty_pattern.ftl

@@ -0,0 +1,17 @@
+key1 = Value 1
+    .attr =
+
+key2 =
+    .attr =
+
+key3 =
+    .attr1 = Attr 1
+    .attr2 =
+
+key4 =
+    .attr1 =
+    .attr2 = Attr 2
+
+key5 =
+    .attr1 =
+    .attr2 =