Skip to content

Commit fd73493

Browse files
pablogsalpablogsal
committed
Highlight invalid ranges in SyntaxErrors
1 parent 2a3f489 commit fd73493

File tree

13 files changed

+874
-585
lines changed

13 files changed

+874
-585
lines changed

‎Grammar/python.gram

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -779,32 +779,32 @@ t_atom[expr_ty]:
779779

780780
# From here on, there are rules for invalid syntax with specialised error messages
781781
invalid_arguments:
782-
| args ',' '*' { RAISE_SYNTAX_ERROR("iterable argument unpacking follows keyword argument unpacking") }
783-
| a=expression for_if_clauses ',' [args | expression for_if_clauses] {
784-
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") }
782+
| a=args ',' '*' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable argument unpacking follows keyword argument unpacking") }
783+
| a=expression b=for_if_clauses ',' [args | expression for_if_clauses] {
784+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") }
785785
| a=args for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a) }
786-
| args ',' a=expression for_if_clauses {
787-
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") }
786+
| args ',' a=expression b=for_if_clauses {
787+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") }
788788
| a=args ',' args { _PyPegen_arguments_parsing_error(p, a) }
789789
invalid_kwarg:
790-
| expression a='=' {
791-
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
792-
a, "expression cannot contain assignment, perhaps you meant \"==\"?") }
790+
| a=expression b='=' {
791+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(
792+
a, b, "expression cannot contain assignment, perhaps you meant \"==\"?") }
793793

794794
invalid_expression:
795795
# !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf"
796796
# Soft keywords need to also be ignored because they can be parsed as NAME NAME
797-
| !(NAME STRING | SOFT_KEYWORD) a=disjunction expression {
798-
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, "invalid syntax. Perhaps you forgot a comma?") }
797+
| !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression {
798+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") }
799799

800800
invalid_named_expression:
801801
| a=expression ':=' expression {
802802
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
803803
a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) }
804-
| a=NAME b='=' bitwise_or !('='|':='|',') {
805-
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?") }
804+
| a=NAME '=' b=bitwise_or !('='|':='|',') {
805+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?") }
806806
| !(list|tuple|genexp|'True'|'None'|'False') a=bitwise_or b='=' bitwise_or !('='|':='|',') {
807-
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "cannot assign to %s here. Maybe you meant '==' instead of '='?",
807+
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot assign to %s here. Maybe you meant '==' instead of '='?",
808808
_PyPegen_get_expr_name(a)) }
809809

810810
invalid_assignment:
@@ -841,25 +841,28 @@ invalid_primary:
841841
invalid_comprehension:
842842
| ('[' | '(' | '{') a=starred_expression for_if_clauses {
843843
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") }
844-
| ('[' | '{') a=star_named_expression ',' [star_named_expressions] for_if_clauses {
845-
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "did you forget parentheses around the comprehension target?") }
844+
| ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses {
845+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, (expr_ty)_PyPegen_seq_last_item((asdl_seq*)b),
846+
"did you forget parentheses around the comprehension target?") }
847+
| ('[' | '{') a=star_named_expression b=',' for_if_clauses {
848+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget parentheses around the comprehension target?") }
846849
invalid_dict_comprehension:
847850
| '{' a='**' bitwise_or for_if_clauses '}' {
848851
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") }
849852
invalid_parameters:
850-
| param_no_default* invalid_parameters_helper param_no_default {
851-
RAISE_SYNTAX_ERROR("non-default argument follows default argument") }
853+
| param_no_default* invalid_parameters_helper a=param_no_default {
854+
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "non-default argument follows default argument") }
852855
invalid_parameters_helper: # This is only there to avoid type errors
853856
| a=slash_with_default { _PyPegen_singleton_seq(p, a) }
854857
| param_with_default+
855858
invalid_lambda_parameters:
856-
| lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default {
857-
RAISE_SYNTAX_ERROR("non-default argument follows default argument") }
859+
| lambda_param_no_default* invalid_lambda_parameters_helper a=lambda_param_no_default {
860+
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "non-default argument follows default argument") }
858861
invalid_lambda_parameters_helper:
859862
| a=lambda_slash_with_default { _PyPegen_singleton_seq(p, a) }
860863
| lambda_param_with_default+
861864
invalid_star_etc:
862-
| '*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") }
865+
| a='*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "named arguments must follow bare *") }
863866
| '*' ',' TYPE_COMMENT { RAISE_SYNTAX_ERROR("bare * has associated type comment") }
864867
invalid_lambda_star_etc:
865868
| '*' (':' | ',' (':' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") }
@@ -897,7 +900,7 @@ invalid_try_stmt:
897900
RAISE_INDENTATION_ERROR("expected an indented block after 'try' statement on line %d", a->lineno) }
898901
invalid_except_stmt:
899902
| 'except' a=expression ',' expressions ['as' NAME ] ':' {
900-
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "exception group must be parenthesized") }
903+
RAISE_SYNTAX_ERROR_STARTING_FROM(a, "exception group must be parenthesized") }
901904
| a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
902905
| a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
903906
invalid_finally_stmt:
@@ -942,10 +945,10 @@ invalid_class_def_raw:
942945

943946
invalid_double_starred_kvpairs:
944947
| ','.double_starred_kvpair+ ',' invalid_kvpair
945-
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use a starred expression in a dictionary value") }
948+
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") }
946949
| expression a=':' &('}'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
947950
invalid_kvpair:
948951
| a=expression !(':') {
949-
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, "':' expected after dictionary key") }
950-
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use a starred expression in a dictionary value") }
952+
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") }
953+
| expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") }
951954
| expression a=':' {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }

‎Include/cpython/pyerrors.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ typedef struct {
2020
PyObject *filename;
2121
PyObject *lineno;
2222
PyObject *offset;
23+
PyObject *end_lineno;
24+
PyObject *end_offset;
2325
PyObject *text;
2426
PyObject *print_file_and_line;
2527
} PySyntaxErrorObject;
@@ -148,6 +150,13 @@ PyAPI_FUNC(void) PyErr_SyntaxLocationObject(
148150
int lineno,
149151
int col_offset);
150152

153+
PyAPI_FUNC(void) PyErr_RangedSyntaxLocationObject(
154+
PyObject *filename,
155+
int lineno,
156+
int col_offset,
157+
int end_lineno,
158+
int end_col_offset);
159+
151160
PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject(
152161
PyObject *filename,
153162
int lineno);

‎Include/internal/pycore_symtable.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ typedef struct _symtable_entry {
6262
int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
6363
int ste_lineno; /* first line of block */
6464
int ste_col_offset; /* offset of first line of block */
65+
int ste_end_lineno; /* end line of block */
66+
int ste_end_col_offset; /* end offset of first line of block */
6567
int ste_opt_lineno; /* lineno of last exec or import * */
6668
int ste_opt_col_offset; /* offset of last exec or import * */
6769
struct symtable *ste_table;

‎Lib/test/test_exceptions.py

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -255,13 +255,13 @@ def baz():
255255
check('from __future__ import doesnt_exist', 1, 1)
256256
check('from __future__ import braces', 1, 1)
257257
check('x=1\nfrom __future__ import division', 2, 1)
258-
check('foo(1=2)', 1, 6)
258+
check('foo(1=2)', 1, 5)
259259
check('def f():\n x, y: int', 2, 3)
260260
check('[*x for x in xs]', 1, 2)
261261
check('foo(x for x in range(10), 100)', 1, 5)
262262
check('for 1 in []: pass', 1, 5)
263-
check('(yield i) = 2', 1, 11)
264-
check('def f(*):\n pass', 1, 8)
263+
check('(yield i) = 2', 1, 2)
264+
check('def f(*):\n pass', 1, 7)
265265

266266
@cpython_only
267267
def testSettingException(self):
@@ -395,25 +395,31 @@ def testAttributes(self):
395395
'filename' : 'filenameStr', 'filename2' : None}),
396396
(SyntaxError, (), {'msg' : None, 'text' : None,
397397
'filename' : None, 'lineno' : None, 'offset' : None,
398-
'print_file_and_line' : None}),
398+
'end_offset': None, 'print_file_and_line' : None}),
399399
(SyntaxError, ('msgStr',),
400400
{'args' : ('msgStr',), 'text' : None,
401401
'print_file_and_line' : None, 'msg' : 'msgStr',
402-
'filename' : None, 'lineno' : None, 'offset' : None}),
402+
'filename' : None, 'lineno' : None, 'offset' : None,
403+
'end_offset': None}),
403404
(SyntaxError, ('msgStr', ('filenameStr', 'linenoStr', 'offsetStr',
404-
'textStr')),
405+
'textStr', 'endLinenoStr', 'endOffsetStr')),
405406
{'offset' : 'offsetStr', 'text' : 'textStr',
406407
'args' : ('msgStr', ('filenameStr', 'linenoStr',
407-
'offsetStr', 'textStr')),
408+
'offsetStr', 'textStr',
409+
'endLinenoStr', 'endOffsetStr')),
408410
'print_file_and_line' : None, 'msg' : 'msgStr',
409-
'filename' : 'filenameStr', 'lineno' : 'linenoStr'}),
411+
'filename' : 'filenameStr', 'lineno' : 'linenoStr',
412+
'end_lineno': 'endLinenoStr', 'end_offset': 'endOffsetStr'}),
410413
(SyntaxError, ('msgStr', 'filenameStr', 'linenoStr', 'offsetStr',
411-
'textStr', 'print_file_and_lineStr'),
414+
'textStr', 'endLinenoStr', 'endOffsetStr',
415+
'print_file_and_lineStr'),
412416
{'text' : None,
413417
'args' : ('msgStr', 'filenameStr', 'linenoStr', 'offsetStr',
414-
'textStr', 'print_file_and_lineStr'),
418+
'textStr', 'endLinenoStr', 'endOffsetStr',
419+
'print_file_and_lineStr'),
415420
'print_file_and_line' : None, 'msg' : 'msgStr',
416-
'filename' : None, 'lineno' : None, 'offset' : None}),
421+
'filename' : None, 'lineno' : None, 'offset' : None,
422+
'end_lineno': None, 'end_offset': None}),
417423
(UnicodeError, (), {'args' : (),}),
418424
(UnicodeEncodeError, ('ascii', 'a', 0, 1,
419425
'ordinal not in range'),
@@ -459,7 +465,7 @@ def testAttributes(self):
459465
e = exc(*args)
460466
except:
461467
print("\nexc=%r, args=%r" % (exc, args), file=sys.stderr)
462-
raise
468+
# raise
463469
else:
464470
# Verify module name
465471
if not type(e).__name__.endswith('NaiveException'):

‎Lib/test/test_syntax.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1212,7 +1212,7 @@ def test_expression_with_assignment(self):
12121212
self._check_error(
12131213
"print(end1 + end2 = ' ')",
12141214
'expression cannot contain assignment, perhaps you meant "=="?',
1215-
offset=19
1215+
offset=7
12161216
)
12171217

12181218
def test_curly_brace_after_primary_raises_immediately(self):

‎Objects/exceptions.c

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1497,25 +1497,22 @@ SyntaxError_init(PySyntaxErrorObject *self, PyObject *args, PyObject *kwds)
14971497
if (!info)
14981498
return -1;
14991499

1500-
if (PyTuple_GET_SIZE(info) != 4) {
1501-
/* not a very good error message, but it's what Python 2.4 gives */
1502-
PyErr_SetString(PyExc_IndexError, "tuple index out of range");
1500+
self->end_lineno = NULL;
1501+
self->end_offset = NULL;
1502+
if (!PyArg_ParseTuple(info, "OOOO|OO",
1503+
&self->filename, &self->lineno,
1504+
&self->offset, &self->text,
1505+
&self->end_lineno, &self->end_offset)) {
15031506
Py_DECREF(info);
15041507
return -1;
15051508
}
15061509

1507-
Py_INCREF(PyTuple_GET_ITEM(info, 0));
1508-
Py_XSETREF(self->filename, PyTuple_GET_ITEM(info, 0));
1509-
1510-
Py_INCREF(PyTuple_GET_ITEM(info, 1));
1511-
Py_XSETREF(self->lineno, PyTuple_GET_ITEM(info, 1));
1512-
1513-
Py_INCREF(PyTuple_GET_ITEM(info, 2));
1514-
Py_XSETREF(self->offset, PyTuple_GET_ITEM(info, 2));
1515-
1516-
Py_INCREF(PyTuple_GET_ITEM(info, 3));
1517-
Py_XSETREF(self->text, PyTuple_GET_ITEM(info, 3));
1518-
1510+
Py_INCREF(self->filename);
1511+
Py_INCREF(self->lineno);
1512+
Py_INCREF(self->offset);
1513+
Py_INCREF(self->text);
1514+
Py_XINCREF(self->end_lineno);
1515+
Py_XINCREF(self->end_offset);
15191516
Py_DECREF(info);
15201517

15211518
/*
@@ -1540,6 +1537,8 @@ SyntaxError_clear(PySyntaxErrorObject *self)
15401537
Py_CLEAR(self->filename);
15411538
Py_CLEAR(self->lineno);
15421539
Py_CLEAR(self->offset);
1540+
Py_CLEAR(self->end_lineno);
1541+
Py_CLEAR(self->end_offset);
15431542
Py_CLEAR(self->text);
15441543
Py_CLEAR(self->print_file_and_line);
15451544
return BaseException_clear((PyBaseExceptionObject *)self);
@@ -1560,6 +1559,8 @@ SyntaxError_traverse(PySyntaxErrorObject *self, visitproc visit, void *arg)
15601559
Py_VISIT(self->filename);
15611560
Py_VISIT(self->lineno);
15621561
Py_VISIT(self->offset);
1562+
Py_VISIT(self->end_lineno);
1563+
Py_VISIT(self->end_offset);
15631564
Py_VISIT(self->text);
15641565
Py_VISIT(self->print_file_and_line);
15651566
return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg);
@@ -1650,6 +1651,10 @@ static PyMemberDef SyntaxError_members[] = {
16501651
PyDoc_STR("exception offset")},
16511652
{"text", T_OBJECT, offsetof(PySyntaxErrorObject, text), 0,
16521653
PyDoc_STR("exception text")},
1654+
{"end_lineno", T_OBJECT, offsetof(PySyntaxErrorObject, end_lineno), 0,
1655+
PyDoc_STR("exception end lineno")},
1656+
{"end_offset", T_OBJECT, offsetof(PySyntaxErrorObject, end_offset), 0,
1657+
PyDoc_STR("exception end offset")},
16531658
{"print_file_and_line", T_OBJECT,
16541659
offsetof(PySyntaxErrorObject, print_file_and_line), 0,
16551660
PyDoc_STR("exception print_file_and_line")},

0 commit comments

Comments
 (0)