Merge pull request #307 from stasm/zeroeight-part1

Implement Syntax 0.8, part 1 (items 1-5)

Author: stasm

eslint_src.json

@@ -32,17 +32,6 @@
         ],
         "no-implied-eval": 2,
         "no-loop-func": 2,
-        "no-magic-numbers": [
-            1,
-            {
-                "ignore": [
-                    -1,
-                    0,
-                    1,
-                    2
-                ]
-            }
-        ],
         "no-useless-call": 2,
         "no-useless-concat": 2,
         "no-delete-var": 2,

fluent-syntax/.gitattributes

@@ -1,2 +1,3 @@
 test/fixtures_reference/crlf.ftl eol=crlf
+test/fixtures_reference/cr.ftl eol=cr
 test/fixtures_structure/crlf.ftl eol=crlf

fluent-syntax/src/errors.js

@@ -28,7 +28,7 @@ function getErrorMessage(code, args) {
     }
     case "E0006": {
       const [id] = args;
-      return `Expected term "${id}" to have a value`;
+      return `Expected term "-${id}" to have a value`;
     }
     case "E0007":
       return "Keyword cannot end with a whitespace";
@@ -74,6 +74,8 @@ function getErrorMessage(code, args) {
       const [char] = args;
       return `Invalid Unicode escape sequence: \\u${char}.`;
     }
+    case "E0027":
+      return "Unbalanced closing brace in TextElement.";
     default:
       return code;
   }

fluent-syntax/src/parser.js

@@ -6,6 +6,10 @@ import { ParseError } from "./errors";
 
 
 const trailingWSRe = /[ \t\n\r]+$/;
+// The Fluent Syntax spec uses /.*/ to parse comment lines. It matches all
+// characters except the following ones, which are considered line endings by
+// the regex engine.
+const COMMENT_EOL = ["\n", "\r", "\u2028", "\u2029"];
 
 
 function withSpan(fn) {
@@ -39,10 +43,9 @@ export default class FluentParser {
     // Poor man's decorators.
     const methodNames = [
       "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier",
-      "getTermIdentifier", "getVariant", "getNumber",
-      "getValue", "getPattern", "getVariantList", "getTextElement",
-      "getPlaceable", "getExpression", "getSelectorExpression", "getCallArg",
-      "getString", "getLiteral"
+      "getVariant", "getNumber", "getValue", "getPattern", "getVariantList",
+      "getTextElement", "getPlaceable", "getExpression",
+      "getSelectorExpression", "getCallArg", "getString", "getLiteral"
     ];
     for (const name of methodNames) {
       this[name] = withSpan(this[name]);
@@ -189,10 +192,10 @@ export default class FluentParser {
         level = i;
       }
 
-      if (ps.currentChar !== EOL) {
+      if (!COMMENT_EOL.includes(ps.currentChar)) {
         ps.expectChar(" ");
         let ch;
-        while ((ch = ps.takeChar(x => x !== EOL))) {
+        while ((ch = ps.takeChar(x => !COMMENT_EOL.includes(x)))) {
           content += ch;
         }
       }
@@ -242,7 +245,8 @@ export default class FluentParser {
   }
 
   getTerm(ps) {
-    const id = this.getTermIdentifier(ps);
+    ps.expectChar("-");
+    const id = this.getIdentifier(ps);
 
     ps.skipBlankInline();
     ps.expectChar("=");
@@ -301,13 +305,6 @@ export default class FluentParser {
     return new AST.Identifier(name);
   }
 
-  getTermIdentifier(ps) {
-    ps.expectChar("-");
-    const id = this.getIdentifier(ps);
-    return new AST.Identifier(`-${id.name}`);
-
-  }
-
   getVariantKey(ps) {
     const ch = ps.currentChar;
 
@@ -455,6 +452,8 @@ export default class FluentParser {
       if (ch === "{") {
         const element = this.getPlaceable(ps);
         elements.push(element);
+      } else if (ch === "}") {
+        throw new ParseError("E0027");
       } else {
         const element = this.getTextElement(ps);
         elements.push(element);
@@ -478,7 +477,7 @@ export default class FluentParser {
 
     let ch;
     while ((ch = ps.currentChar)) {
-      if (ch === "{") {
+      if (ch === "{" || ch === "}") {
         return new AST.TextElement(buffer);
       }
 
@@ -494,23 +493,17 @@ export default class FluentParser {
         continue;
       }
 
-      if (ch === "\\") {
-        ps.next();
-        buffer += this.getEscapeSequence(ps);
-        continue;
-      }
-
       buffer += ch;
       ps.next();
     }
 
     return new AST.TextElement(buffer);
   }
 
-  getEscapeSequence(ps, specials = ["{", "\\"]) {
+  getEscapeSequence(ps) {
     const next = ps.currentChar;
 
-    if (specials.includes(next)) {
+    if (next === "\\" || next === "\"") {
       ps.next();
       return `\\${next}`;
     }
@@ -738,7 +731,7 @@ export default class FluentParser {
     let ch;
     while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) {
       if (ch === "\\") {
-        val += this.getEscapeSequence(ps, ["{", "\\", "\""]);
+        val += this.getEscapeSequence(ps);
       } else {
         val += ch;
       }
@@ -777,7 +770,8 @@ export default class FluentParser {
     }
 
     if (ch === "-") {
-      const id = this.getTermIdentifier(ps);
+      ps.next();
+      const id = this.getIdentifier(ps);
       return new AST.TermReference(id);
     }
 

fluent-syntax/src/serializer.js

@@ -44,8 +44,9 @@ export default class FluentSerializer {
   serializeEntry(entry, state = 0) {
     switch (entry.type) {
       case "Message":
-      case "Term":
         return serializeMessage(entry);
+      case "Term":
+        return serializeTerm(entry);
       case "Comment":
         if (state & HAS_ENTRIES) {
           return `\n${serializeComment(entry, "#")}\n`;
@@ -95,8 +96,7 @@ function serializeMessage(message) {
     parts.push(serializeComment(message.comment));
   }
 
-  parts.push(serializeIdentifier(message.id));
-  parts.push(" =");
+  parts.push(`${serializeIdentifier(message.id)} =`);
 
   if (message.value) {
     parts.push(serializeValue(message.value));
@@ -111,6 +111,25 @@ function serializeMessage(message) {
 }
 
 
+function serializeTerm(term) {
+  const parts = [];
+
+  if (term.comment) {
+    parts.push(serializeComment(term.comment));
+  }
+
+  parts.push(`-${serializeIdentifier(term.id)} =`);
+  parts.push(serializeValue(term.value));
+
+  for (const attribute of term.attributes) {
+    parts.push(serializeAttribute(attribute));
+  }
+
+  parts.push("\n");
+  return parts.join("");
+}
+
+
 function serializeAttribute(attribute) {
   const id = serializeIdentifier(attribute.id);
   const value = indent(serializeValue(attribute.value));
@@ -202,8 +221,9 @@ function serializeExpression(expr) {
     case "NumberLiteral":
       return serializeNumberLiteral(expr);
     case "MessageReference":
-    case "TermReference":
       return serializeMessageReference(expr);
+    case "TermReference":
+      return serializeTermReference(expr);
     case "VariableReference":
       return serializeVariableReference(expr);
     case "AttributeExpression":
@@ -237,6 +257,11 @@ function serializeMessageReference(expr) {
 }
 
 
+function serializeTermReference(expr) {
+  return `-${serializeIdentifier(expr.id)}`;
+}
+
+
 function serializeVariableReference(expr) {
   return `$${serializeIdentifier(expr.id)}`;
 }

fluent-syntax/test/fixtures_behavior/escape_sequences.ftl

@@ -1,10 +1,16 @@
-# ~ERROR E0025, pos 8, args "A"
-key1 = \A
+## Backslash is a regular character in text elements.
+key01 = \A
+key02 = \u0041
+key03 = \\u0041
+key04 = \u000z
+key05 = \{Value}
 
-# ~ERROR E0026, pos 23, args "000z"
-key2 = \u000z
+key06 = {"Escaped \" quote"}
+key07 = {"Escaped \\ backslash"}
+key08 = {"Escaped \u0041 A"}
 
-key3 = \{Escaped}
-key4 = {"Escaped \" quote"}
-key5 = \u0041
-key6 = \\u0041
+# ~ERROR E0025, pos 232, args "A"
+key09 = {"\A"}
+
+# ~ERROR E0026, pos 252, args "000z"
+key10 = {"\u000z"}

fluent-syntax/test/fixtures_behavior/placeable_in_placeable.ftl

@@ -7,8 +7,9 @@ key2 = {  { foo }  }
 #     { foo }
 #   }
 
-key4 = {  { foo }
 # ~ERROR E0003, pos 96, args "}"
+key4 = {  { foo }
 
 
+# ~ERROR E0027, pos 111
 key5 = { foo } }

fluent-syntax/test/fixtures_behavior/term.ftl

@@ -25,8 +25,8 @@ err4 = { -brand() }
 # ~ERROR E0008, pos 339
 
 -err5 =
-# ~ERROR E0006, pos 351, args "-err5"
+# ~ERROR E0006, pos 351, args "err5"
 
 -err6 =
     .attr = Attribute
-# ~ERROR E0006, pos 360, args "-err6"
+# ~ERROR E0006, pos 360, args "err6"

fluent-syntax/test/fixtures_reference/astral.ftl

@@ -0,0 +1,20 @@
+face-with-tears-of-joy = 😂
+tetragram-for-centre = 𝌆
+
+surrogates-in-text = \uD83D\uDE02
+surrogates-in-string = {"\uD83D\uDE02"}
+surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"}
+
+emoji-in-text = A face 😂 with tears of joy.
+emoji-in-string = {"A face 😂 with tears of joy."}
+
+# ERROR Invalid identifier
+err-😂 = Value
+
+# ERROR Invalid expression
+err-invalid-expression = { 😂 }
+
+# ERROR Invalid variant key
+err-invalid-variant-key = { $sel ->
+    *[😂] Value
+}