Skip to content

Commit 0061407

Browse files
zbranieckizbraniecki
authored
Port blankline changes to python parser (#20)
1 parent e881c0c commit 0061407

11 files changed

+1074
-28
lines changed

‎fluent/syntax/ftlstream.py

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,18 @@ def skip_blank_lines(self):
2525
self.reset_peek()
2626
break
2727

28+
def peek_blank_lines(self):
29+
while True:
30+
line_start = self.get_peek_index()
31+
32+
self.peek_inline_ws()
33+
34+
if self.current_peek_is('\n'):
35+
self.peek()
36+
else:
37+
self.reset_peek(line_start)
38+
break
39+
2840
def skip_inline_ws(self):
2941
while self.ch:
3042
if self.ch not in INLINE_WS:
@@ -42,6 +54,12 @@ def expect_char(self, ch):
4254

4355
raise ParseError('E0003', ch)
4456

57+
def expect_indent(self):
58+
self.expect_char('\n')
59+
self.skip_blank_lines()
60+
self.expect_char(' ')
61+
self.skip_inline_ws()
62+
4563
def take_char_if(self, ch):
4664
if self.ch == ch:
4765
self.next()
@@ -63,7 +81,7 @@ def is_id_start(self):
6381

6482
return (cc >= 97 and cc <= 122) or \
6583
(cc >= 65 and cc <= 90) or \
66-
cc == 95
84+
cc == 95
6785

6886
def is_number_start(self):
6987
cc = ord(self.ch)
@@ -90,6 +108,8 @@ def is_peek_next_line_variant_start(self):
90108

91109
self.peek()
92110

111+
self.peek_blank_lines()
112+
93113
ptr = self.get_peek_index()
94114

95115
self.peek_inline_ws()
@@ -114,6 +134,8 @@ def is_peek_next_line_attribute_start(self):
114134

115135
self.peek()
116136

137+
self.peek_blank_lines()
138+
117139
ptr = self.get_peek_index()
118140

119141
self.peek_inline_ws()
@@ -129,12 +151,14 @@ def is_peek_next_line_attribute_start(self):
129151
self.reset_peek()
130152
return False
131153

132-
def is_peek_next_line_pattern(self):
154+
def is_peek_next_non_blank_line_pattern(self):
133155
if not self.current_peek_is('\n'):
134156
return False
135157

136158
self.peek()
137159

160+
self.peek_blank_lines()
161+
138162
ptr = self.get_peek_index()
139163

140164
self.peek_inline_ws()
@@ -161,6 +185,8 @@ def is_peek_next_line_tag_start(self):
161185

162186
self.peek()
163187

188+
self.peek_blank_lines()
189+
164190
ptr = self.get_peek_index()
165191

166192
self.peek_inline_ws()
@@ -212,7 +238,7 @@ def closure(ch):
212238
return (cc >= 97 and cc <= 122) or \
213239
(cc >= 65 and cc <= 90) or \
214240
(cc >= 48 and cc <= 57) or \
215-
cc == 95 or cc == 45 or cc == 32
241+
cc == 95 or cc == 45 or cc == 32
216242
return self.take_char(closure)
217243

218244
def take_digit(self):

‎fluent/syntax/ftlstream.py.orig

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
from __future__ import unicode_literals
2+
from .stream import ParserStream
3+
from .errors import ParseError
4+
5+
6+
INLINE_WS = (' ', '\t')
7+
8+
9+
class FTLParserStream(ParserStream):
10+
def peek_inline_ws(self):
11+
ch = self.current_peek()
12+
while ch:
13+
if ch not in INLINE_WS:
14+
break
15+
ch = self.peek()
16+
17+
def skip_blank_lines(self):
18+
while True:
19+
self.peek_inline_ws()
20+
21+
if self.current_peek_is('\n'):
22+
self.skip_to_peek()
23+
self.next()
24+
else:
25+
self.reset_peek()
26+
break
27+
28+
def skip_inline_ws(self):
29+
while self.ch:
30+
if self.ch not in INLINE_WS:
31+
break
32+
self.next()
33+
34+
def expect_char(self, ch):
35+
if self.ch == ch:
36+
self.next()
37+
return True
38+
39+
if ch == '\n':
40+
# Unicode Character 'SYMBOL FOR NEWLINE' (U+2424)
41+
raise ParseError('E0003', '\u2424')
42+
43+
raise ParseError('E0003', ch)
44+
45+
def take_char_if(self, ch):
46+
if self.ch == ch:
47+
self.next()
48+
return True
49+
return False
50+
51+
def take_char(self, f):
52+
ch = self.ch
53+
if ch is not None and f(ch):
54+
self.next()
55+
return ch
56+
return None
57+
58+
def is_id_start(self):
59+
if self.ch is None:
60+
return False
61+
62+
cc = ord(self.ch)
63+
64+
return (cc >= 97 and cc <= 122) or \
65+
(cc >= 65 and cc <= 90) or \
66+
cc == 95
67+
68+
def is_number_start(self):
69+
cc = ord(self.ch)
70+
71+
return (cc >= 48 and cc <= 57) or cc == 45
72+
73+
def is_peek_next_line_comment(self):
74+
if not self.current_peek_is('\n'):
75+
return False
76+
77+
self.peek()
78+
if self.current_peek_is('/'):
79+
self.peek()
80+
if self.current_peek_is('/'):
81+
self.reset_peek()
82+
return True
83+
84+
self.reset_peek()
85+
return False
86+
87+
def is_peek_next_line_variant_start(self):
88+
if not self.current_peek_is('\n'):
89+
return False
90+
91+
self.peek()
92+
93+
ptr = self.get_peek_index()
94+
95+
self.peek_inline_ws()
96+
97+
if (self.get_peek_index() - ptr == 0):
98+
self.reset_peek()
99+
return False
100+
101+
if self.current_peek_is('*'):
102+
self.peek()
103+
104+
if self.current_peek_is('[') and not self.peek_char_is('['):
105+
self.reset_peek()
106+
return True
107+
108+
self.reset_peek()
109+
return False
110+
111+
def is_peek_next_line_attribute_start(self):
112+
if not self.current_peek_is('\n'):
113+
return False
114+
115+
self.peek()
116+
117+
ptr = self.get_peek_index()
118+
119+
self.peek_inline_ws()
120+
121+
if (self.get_peek_index() - ptr == 0):
122+
self.reset_peek()
123+
return False
124+
125+
if self.current_peek_is('.'):
126+
self.reset_peek()
127+
return True
128+
129+
self.reset_peek()
130+
return False
131+
132+
def is_peek_next_line_pattern(self):
133+
if not self.current_peek_is('\n'):
134+
return False
135+
136+
self.peek()
137+
138+
ptr = self.get_peek_index()
139+
140+
self.peek_inline_ws()
141+
142+
if (self.get_peek_index() - ptr == 0):
143+
self.reset_peek()
144+
return False
145+
146+
if (self.current_peek_is('}') or
147+
self.current_peek_is('.') or
148+
self.current_peek_is('#') or
149+
self.current_peek_is('[') or
150+
self.current_peek_is('*')):
151+
self.reset_peek()
152+
return False
153+
154+
self.reset_peek()
155+
return True
156+
157+
def is_peek_next_line_tag_start(self):
158+
159+
if not self.current_peek_is('\n'):
160+
return False
161+
162+
self.peek()
163+
164+
ptr = self.get_peek_index()
165+
166+
self.peek_inline_ws()
167+
168+
if (self.get_peek_index() - ptr == 0):
169+
self.reset_peek()
170+
return False
171+
172+
if self.current_peek_is('#'):
173+
self.reset_peek()
174+
return True
175+
176+
self.reset_peek()
177+
return False
178+
179+
def skip_to_next_entry_start(self):
180+
while self.ch:
181+
if self.current_is('\n') and not self.peek_char_is('\n'):
182+
self.next()
183+
184+
if self.ch is None or self.is_id_start() or \
185+
(self.current_is('/') and self.peek_char_is('/')) or \
186+
(self.current_is('[') and self.peek_char_is('[')):
187+
break
188+
self.next()
189+
190+
def take_id_start(self):
191+
if self.is_id_start():
192+
ret = self.ch
193+
self.next()
194+
return ret
195+
196+
raise ParseError('E0004', 'a-zA-Z_')
197+
198+
def take_id_char(self):
199+
def closure(ch):
200+
cc = ord(ch)
201+
return ((cc >= 97 and cc <= 122) or
202+
(cc >= 65 and cc <= 90) or
203+
(cc >= 48 and cc <= 57) or
204+
cc == 95 or cc == 45)
205+
return self.take_char(closure)
206+
207+
def take_symb_char(self):
208+
def closure(ch):
209+
if ch is None:
210+
return False
211+
cc = ord(ch)
212+
return (cc >= 97 and cc <= 122) or \
213+
(cc >= 65 and cc <= 90) or \
214+
(cc >= 48 and cc <= 57) or \
215+
cc == 95 or cc == 45 or cc == 32
216+
return self.take_char(closure)
217+
218+
def take_digit(self):
219+
def closure(ch):
220+
cc = ord(ch)
221+
return (cc >= 48 and cc <= 57)
222+
return self.take_char(closure)

0 commit comments

Comments
 (0)