Skip to content

Commit eb8f391

Browse files
stasmstasm
committed
Store unescaped content in StringLiteral.value and raw content in .raw
1 parent 504e256 commit eb8f391

17 files changed

+89
-20
lines changed

‎fluent-syntax/src/ast.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,10 @@ export class Placeable extends PatternElement {
9696
export class Expression extends SyntaxNode {}
9797

9898
export class StringLiteral extends Expression {
99-
constructor(value) {
99+
constructor(raw, value) {
100100
super();
101101
this.type = "StringLiteral";
102+
this.raw = raw;
102103
this.value = value;
103104
}
104105
}

‎fluent-syntax/src/parser.js

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -600,7 +600,7 @@ export default class FluentParser {
600600

601601
if (next === "\\" || next === "\"") {
602602
ps.next();
603-
return `\\${next}`;
603+
return [`\\${next}`, next];
604604
}
605605

606606
if (next === "u") {
@@ -617,7 +617,15 @@ export default class FluentParser {
617617
sequence += ch;
618618
}
619619

620-
return `\\u${sequence}`;
620+
const codepoint = parseInt(sequence, 16);
621+
const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint
622+
// It's a Unicode scalar value.
623+
? String.fromCodePoint(codepoint)
624+
// Escape sequences reresenting surrogate code points are well-formed
625+
// but invalid in Fluent. Replace them with U+FFFD REPLACEMENT
626+
// CHARACTER.
627+
: "�";
628+
return [`\\u${sequence}`, unescaped];
621629
}
622630

623631
throw new ParseError("E0025", next);
@@ -804,16 +812,20 @@ export default class FluentParser {
804812
}
805813

806814
getString(ps) {
807-
let val = "";
815+
let raw = "";
816+
let value = "";
808817

809818
ps.expectChar("\"");
810819

811820
let ch;
812821
while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) {
813822
if (ch === "\\") {
814-
val += this.getEscapeSequence(ps);
823+
const [sequence, unescaped] = this.getEscapeSequence(ps);
824+
raw += sequence;
825+
value += unescaped;
815826
} else {
816-
val += ch;
827+
raw += ch;
828+
value += ch;
817829
}
818830
}
819831

@@ -823,8 +835,7 @@ export default class FluentParser {
823835

824836
ps.expectChar("\"");
825837

826-
return new AST.StringLiteral(val);
827-
838+
return new AST.StringLiteral(raw, value);
828839
}
829840

830841
getLiteral(ps) {

‎fluent-syntax/src/serializer.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ function serializePlaceable(placeable) {
211211
function serializeExpression(expr) {
212212
switch (expr.type) {
213213
case "StringLiteral":
214-
return `"${expr.value}"`;
214+
return `"${expr.raw}"`;
215215
case "NumberLiteral":
216216
return expr.value;
217217
case "MessageReference":

‎fluent-syntax/test/fixtures_reference/astral.json

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@
6868
"type": "Placeable",
6969
"expression": {
7070
"type": "StringLiteral",
71-
"value": "\\uD83D\\uDE02"
71+
"raw": "\\uD83D\\uDE02",
72+
"value": "��"
7273
}
7374
}
7475
]
@@ -89,14 +90,16 @@
8990
"type": "Placeable",
9091
"expression": {
9192
"type": "StringLiteral",
92-
"value": "\\uD83D"
93+
"raw": "\\uD83D",
94+
"value": "�"
9395
}
9496
},
9597
{
9698
"type": "Placeable",
9799
"expression": {
98100
"type": "StringLiteral",
99-
"value": "\\uDE02"
101+
"raw": "\\uDE02",
102+
"value": "�"
100103
}
101104
}
102105
]
@@ -135,6 +138,7 @@
135138
"type": "Placeable",
136139
"expression": {
137140
"type": "StringLiteral",
141+
"raw": "A face 😂 with tears of joy.",
138142
"value": "A face 😂 with tears of joy."
139143
}
140144
}

‎fluent-syntax/test/fixtures_reference/call_expressions.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
},
102102
{
103103
"type": "StringLiteral",
104+
"raw": "a",
104105
"value": "a"
105106
},
106107
{
@@ -160,6 +161,7 @@
160161
},
161162
"value": {
162163
"type": "StringLiteral",
164+
"raw": "Y",
163165
"value": "Y"
164166
}
165167
}
@@ -212,6 +214,7 @@
212214
},
213215
"value": {
214216
"type": "StringLiteral",
217+
"raw": "Y",
215218
"value": "Y"
216219
}
217220
}
@@ -250,6 +253,7 @@
250253
},
251254
{
252255
"type": "StringLiteral",
256+
"raw": "a",
253257
"value": "a"
254258
},
255259
{
@@ -280,6 +284,7 @@
280284
},
281285
"value": {
282286
"type": "StringLiteral",
287+
"raw": "Y",
283288
"value": "Y"
284289
}
285290
}
@@ -336,6 +341,7 @@
336341
"positional": [
337342
{
338343
"type": "StringLiteral",
344+
"raw": "a",
339345
"value": "a"
340346
},
341347
{
@@ -418,6 +424,7 @@
418424
"positional": [
419425
{
420426
"type": "StringLiteral",
427+
"raw": "a",
421428
"value": "a"
422429
},
423430
{
@@ -471,6 +478,7 @@
471478
"positional": [
472479
{
473480
"type": "StringLiteral",
481+
"raw": "a",
474482
"value": "a"
475483
},
476484
{
@@ -587,6 +595,7 @@
587595
"positional": [
588596
{
589597
"type": "StringLiteral",
598+
"raw": "a",
590599
"value": "a"
591600
}
592601
],

‎fluent-syntax/test/fixtures_reference/escaped_characters.json

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@
122122
"type": "Placeable",
123123
"expression": {
124124
"type": "StringLiteral",
125-
"value": "\\\""
125+
"raw": "\\\"",
126+
"value": "\""
126127
}
127128
}
128129
]
@@ -143,7 +144,8 @@
143144
"type": "Placeable",
144145
"expression": {
145146
"type": "StringLiteral",
146-
"value": "\\\\"
147+
"raw": "\\\\",
148+
"value": "\\"
147149
}
148150
}
149151
]
@@ -186,7 +188,8 @@
186188
"type": "Placeable",
187189
"expression": {
188190
"type": "StringLiteral",
189-
"value": "\\u0041"
191+
"raw": "\\u0041",
192+
"value": "A"
190193
}
191194
}
192195
]
@@ -207,7 +210,8 @@
207210
"type": "Placeable",
208211
"expression": {
209212
"type": "StringLiteral",
210-
"value": "\\\\u0041"
213+
"raw": "\\\\u0041",
214+
"value": "\\u0041"
211215
}
212216
}
213217
]
@@ -236,6 +240,7 @@
236240
"type": "Placeable",
237241
"expression": {
238242
"type": "StringLiteral",
243+
"raw": "{",
239244
"value": "{"
240245
}
241246
},
@@ -265,6 +270,7 @@
265270
"type": "Placeable",
266271
"expression": {
267272
"type": "StringLiteral",
273+
"raw": "}",
268274
"value": "}"
269275
}
270276
},

‎fluent-syntax/test/fixtures_reference/leading_dots.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
"type": "Placeable",
5151
"expression": {
5252
"type": "StringLiteral",
53+
"raw": ".",
5354
"value": "."
5455
}
5556
},
@@ -75,6 +76,7 @@
7576
"type": "Placeable",
7677
"expression": {
7778
"type": "StringLiteral",
79+
"raw": ".",
7880
"value": "."
7981
}
8082
},
@@ -104,6 +106,7 @@
104106
"type": "Placeable",
105107
"expression": {
106108
"type": "StringLiteral",
109+
"raw": ".",
107110
"value": "."
108111
}
109112
},
@@ -133,6 +136,7 @@
133136
"type": "Placeable",
134137
"expression": {
135138
"type": "StringLiteral",
139+
"raw": ".",
136140
"value": "."
137141
}
138142
},
@@ -229,6 +233,7 @@
229233
"type": "Placeable",
230234
"expression": {
231235
"type": "StringLiteral",
236+
"raw": ".",
232237
"value": "."
233238
}
234239
},
@@ -316,6 +321,7 @@
316321
"type": "Placeable",
317322
"expression": {
318323
"type": "StringLiteral",
324+
"raw": ".",
319325
"value": "."
320326
}
321327
},
@@ -377,6 +383,7 @@
377383
"type": "Placeable",
378384
"expression": {
379385
"type": "StringLiteral",
386+
"raw": ".",
380387
"value": "."
381388
}
382389
},
@@ -463,6 +470,7 @@
463470
"type": "Placeable",
464471
"expression": {
465472
"type": "StringLiteral",
473+
"raw": ".",
466474
"value": "."
467475
}
468476
},

‎fluent-syntax/test/fixtures_reference/literal_expressions.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"type": "Placeable",
1515
"expression": {
1616
"type": "StringLiteral",
17+
"raw": "abc",
1718
"value": "abc"
1819
}
1920
}

‎fluent-syntax/test/fixtures_reference/messages.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@
218218
"type": "Placeable",
219219
"expression": {
220220
"type": "StringLiteral",
221+
"raw": "",
221222
"value": ""
222223
}
223224
}

‎fluent-syntax/test/fixtures_reference/multiline_values.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@
126126
"type": "Placeable",
127127
"expression": {
128128
"type": "StringLiteral",
129+
"raw": "placeables",
129130
"value": "placeables"
130131
}
131132
},
@@ -137,6 +138,7 @@
137138
"type": "Placeable",
138139
"expression": {
139140
"type": "StringLiteral",
141+
"raw": "at",
140142
"value": "at"
141143
}
142144
},
@@ -148,13 +150,15 @@
148150
"type": "Placeable",
149151
"expression": {
150152
"type": "StringLiteral",
153+
"raw": "of lines",
151154
"value": "of lines"
152155
}
153156
},
154157
{
155158
"type": "Placeable",
156159
"expression": {
157160
"type": "StringLiteral",
161+
"raw": ".",
158162
"value": "."
159163
}
160164
}
@@ -176,6 +180,7 @@
176180
"type": "Placeable",
177181
"expression": {
178182
"type": "StringLiteral",
183+
"raw": "A multiline value",
179184
"value": "A multiline value"
180185
}
181186
},
@@ -187,6 +192,7 @@
187192
"type": "Placeable",
188193
"expression": {
189194
"type": "StringLiteral",
195+
"raw": "with a placeable",
190196
"value": "with a placeable"
191197
}
192198
}
@@ -280,6 +286,7 @@
280286
"type": "Placeable",
281287
"expression": {
282288
"type": "StringLiteral",
289+
"raw": ".",
283290
"value": "."
284291
}
285292
},
@@ -309,6 +316,7 @@
309316
"type": "Placeable",
310317
"expression": {
311318
"type": "StringLiteral",
319+
"raw": ".",
312320
"value": "."
313321
}
314322
}

0 commit comments

Comments
 (0)