Skip to content
This repository was archived by the owner on Jul 5, 2023. It is now read-only.

Preserve string kind modifiers #49

Merged
merged 2 commits into from
Sep 15, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ast27/Include/Python-ast.h
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ struct _expr {

struct {
string s;
int has_b;
string kind;
} Str;

struct {
Expand Down Expand Up @@ -505,7 +505,7 @@ expr_ty _Ta27_Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Num(a0, a1, a2, a3) _Ta27_Num(a0, a1, a2, a3)
expr_ty _Ta27_Num(object n, int lineno, int col_offset, PyArena *arena);
#define Str(a0, a1, a2, a3, a4) _Ta27_Str(a0, a1, a2, a3, a4)
expr_ty _Ta27_Str(string s, int has_b, int lineno, int col_offset, PyArena *arena);
expr_ty _Ta27_Str(string s, string kind, int lineno, int col_offset, PyArena *arena);
#define Attribute(a0, a1, a2, a3, a4, a5) _Ta27_Attribute(a0, a1, a2, a3, a4, a5)
expr_ty _Ta27_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena);
Expand Down
2 changes: 1 addition & 1 deletion ast27/Parser/Python.asdl
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ module Python version "$Revision$"
expr? starargs, expr? kwargs)
| Repr(expr value)
| Num(object n) -- a number as a PyObject.
| Str(string s, int? has_b) -- need to specify raw, unicode, etc?
| Str(string s, string kind)
-- other literals? bools?

-- the following expression can appear in assignment context
Expand Down
28 changes: 17 additions & 11 deletions ast27/Python/Python-ast.c
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ static char *Num_fields[]={
static PyTypeObject *Str_type;
static char *Str_fields[]={
"s",
"has_b",
"kind",
};
static PyTypeObject *Attribute_type;
static char *Attribute_fields[]={
Expand Down Expand Up @@ -1850,20 +1850,25 @@ Num(object n, int lineno, int col_offset, PyArena *arena)
}

expr_ty
Str(string s, int has_b, int lineno, int col_offset, PyArena *arena)
Str(string s, string kind, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!s) {
PyErr_SetString(PyExc_ValueError,
"field s is required for Str");
return NULL;
}
if (!kind) {
PyErr_SetString(PyExc_ValueError,
"field kind is required for Str");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = Str_kind;
p->v.Str.s = s;
p->v.Str.has_b = has_b;
p->v.Str.kind = kind;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
Expand Down Expand Up @@ -2889,9 +2894,9 @@ ast2obj_expr(void* _o)
if (PyObject_SetAttrString(result, "s", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_int(o->v.Str.has_b);
value = ast2obj_string(o->v.Str.kind);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "has_b", value) == -1)
if (PyObject_SetAttrString(result, "kind", value) == -1)
goto failed;
Py_DECREF(value);
break;
Expand Down Expand Up @@ -5714,7 +5719,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
if (isinstance) {
string s;
int has_b;
string kind;

if (PyObject_HasAttrString(obj, "s")) {
int res;
Expand All @@ -5728,18 +5733,19 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str");
return 1;
}
if (PyObject_HasAttrString(obj, "has_b")) {
if (PyObject_HasAttrString(obj, "kind")) {
int res;
tmp = PyObject_GetAttrString(obj, "has_b");
tmp = PyObject_GetAttrString(obj, "kind");
if (tmp == NULL) goto failed;
res = obj2ast_int(tmp, &has_b, arena);
res = obj2ast_string(tmp, &kind, arena);
if (res != 0) goto failed;
Py_XDECREF(tmp);
tmp = NULL;
} else {
has_b = 0;
PyErr_SetString(PyExc_TypeError, "required field \"kind\" missing from Str");
return 1;
}
*out = Str(s, has_b, lineno, col_offset, arena);
*out = Str(s, kind, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
Expand Down
21 changes: 14 additions & 7 deletions ast27/Python/ast.c
Original file line number Diff line number Diff line change
Expand Up @@ -1498,10 +1498,20 @@ ast_for_atom(struct compiling *c, const node *n)
return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
case STRING: {
PyObject *str = parsestrplus(c, n);
const char *s = STR(CHILD(n, 0));
PyObject *kind, *str = parsestrplus(c, n);
const char *raw, *s = STR(CHILD(n, 0));
int quote = Py_CHARMASK(*s);
int has_b = 0;
/* currently Python allows up to 2 string modifiers */
char *ch, s_kind[3] = {0, 0, 0};
ch = s_kind;
raw = s;
while (*raw && *raw != '\'' && *raw != '"') {
*ch++ = *raw++;
}
kind = PyUnicode_FromString(s_kind);
if (!kind) {
return NULL;
}
if (!str) {
#ifdef Py_USING_UNICODE
if (PyErr_ExceptionMatches(PyExc_UnicodeError)){
Expand All @@ -1526,10 +1536,7 @@ ast_for_atom(struct compiling *c, const node *n)
return NULL;
}
PyArena_AddPyObject(c->c_arena, str);
if (quote == 'b' || quote == 'B') {
has_b = 1;
}
return Str(str, has_b, LINENO(n), n->n_col_offset, c->c_arena);
return Str(str, kind, LINENO(n), n->n_col_offset, c->c_arena);
}
case NUMBER: {
PyObject *pynum = parsenumber(c, STR(ch));
Expand Down
6 changes: 4 additions & 2 deletions ast3/Include/Python-ast.h
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ struct _expr {

struct {
string s;
string kind;
} Str;

struct {
Expand Down Expand Up @@ -601,8 +602,9 @@ expr_ty _Ta3_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
lineno, int col_offset, PyArena *arena);
#define Num(a0, a1, a2, a3) _Ta3_Num(a0, a1, a2, a3)
expr_ty _Ta3_Num(object n, int lineno, int col_offset, PyArena *arena);
#define Str(a0, a1, a2, a3) _Ta3_Str(a0, a1, a2, a3)
expr_ty _Ta3_Str(string s, int lineno, int col_offset, PyArena *arena);
#define Str(a0, a1, a2, a3, a4) _Ta3_Str(a0, a1, a2, a3, a4)
expr_ty _Ta3_Str(string s, string kind, int lineno, int col_offset, PyArena
*arena);
#define FormattedValue(a0, a1, a2, a3, a4, a5) _Ta3_FormattedValue(a0, a1, a2, a3, a4, a5)
expr_ty _Ta3_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
int lineno, int col_offset, PyArena *arena);
Expand Down
2 changes: 1 addition & 1 deletion ast3/Parser/Python.asdl
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ module Python
| Compare(expr left, cmpop* ops, expr* comparators)
| Call(expr func, expr* args, keyword* keywords)
| Num(object n) -- a number as a PyObject.
| Str(string s) -- need to specify raw, unicode, etc?
| Str(string s, string kind)
| FormattedValue(expr value, int? conversion, expr? format_spec)
| JoinedStr(expr* values)
| Bytes(bytes s)
Expand Down
31 changes: 28 additions & 3 deletions ast3/Python/Python-ast.c
Original file line number Diff line number Diff line change
Expand Up @@ -307,8 +307,10 @@ static char *Num_fields[]={
};
static PyTypeObject *Str_type;
_Py_IDENTIFIER(s);
_Py_IDENTIFIER(kind);
static char *Str_fields[]={
"s",
"kind",
};
static PyTypeObject *FormattedValue_type;
_Py_IDENTIFIER(conversion);
Expand Down Expand Up @@ -983,7 +985,7 @@ static int init_types(void)
if (!Call_type) return 0;
Num_type = make_type("Num", expr_type, Num_fields, 1);
if (!Num_type) return 0;
Str_type = make_type("Str", expr_type, Str_fields, 1);
Str_type = make_type("Str", expr_type, Str_fields, 2);
if (!Str_type) return 0;
FormattedValue_type = make_type("FormattedValue", expr_type,
FormattedValue_fields, 3);
Expand Down Expand Up @@ -2181,19 +2183,25 @@ Num(object n, int lineno, int col_offset, PyArena *arena)
}

expr_ty
Str(string s, int lineno, int col_offset, PyArena *arena)
Str(string s, string kind, int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
if (!s) {
PyErr_SetString(PyExc_ValueError,
"field s is required for Str");
return NULL;
}
if (!kind) {
PyErr_SetString(PyExc_ValueError,
"field kind is required for Str");
return NULL;
}
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
if (!p)
return NULL;
p->kind = Str_kind;
p->v.Str.s = s;
p->v.Str.kind = kind;
p->lineno = lineno;
p->col_offset = col_offset;
return p;
Expand Down Expand Up @@ -3448,6 +3456,11 @@ ast2obj_expr(void* _o)
if (_PyObject_SetAttrId(result, &PyId_s, value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_string(o->v.Str.kind);
if (!value) goto failed;
if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1)
goto failed;
Py_DECREF(value);
break;
case FormattedValue_kind:
result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
Expand Down Expand Up @@ -6766,6 +6779,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
if (isinstance) {
string s;
string kind;

if (_PyObject_HasAttrId(obj, &PyId_s)) {
int res;
Expand All @@ -6778,7 +6792,18 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str");
return 1;
}
*out = Str(s, lineno, col_offset, arena);
if (_PyObject_HasAttrId(obj, &PyId_kind)) {
int res;
tmp = _PyObject_GetAttrId(obj, &PyId_kind);
if (tmp == NULL) goto failed;
res = obj2ast_string(tmp, &kind, arena);
if (res != 0) goto failed;
Py_CLEAR(tmp);
} else {
PyErr_SetString(PyExc_TypeError, "required field \"kind\" missing from Str");
return 1;
}
*out = Str(s, kind, lineno, col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
Expand Down
15 changes: 13 additions & 2 deletions ast3/Python/ast.c
Original file line number Diff line number Diff line change
Expand Up @@ -4971,14 +4971,25 @@ FstringParser_Dealloc(FstringParser *state)
static expr_ty
make_str_node_and_del(PyObject **str, struct compiling *c, const node* n)
{
PyObject *s = *str;
PyObject *kind, *s = *str;
const char *raw = STR(CHILD(n, 0));
/* currently Python allows up to 2 string modifiers */
char *ch, s_kind[3] = {0, 0, 0};
ch = s_kind;
while (*raw && *raw != '\'' && *raw != '"') {
*ch++ = *raw++;
}
kind = PyUnicode_FromString(s_kind);
if (!kind) {
return NULL;
}
*str = NULL;
assert(PyUnicode_CheckExact(s));
if (PyArena_AddPyObject(c->c_arena, s) < 0) {
Py_DECREF(s);
return NULL;
}
return Str(s, LINENO(n), n->n_col_offset, c->c_arena);
return Str(s, kind, LINENO(n), n->n_col_offset, c->c_arena);
}

/* Add a non-f-string (that is, a regular literal string). str is
Expand Down