Preserve string kind modifiers

ast27/Include/Python-ast.h

@@ -286,7 +286,7 @@ struct _expr {
 
                 struct {
                         string s;
-                        int has_b;
+                        string kind;
                 } Str;
 
                 struct {
@@ -505,7 +505,7 @@ expr_ty _Ta27_Repr(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Num(a0, a1, a2, a3) _Ta27_Num(a0, a1, a2, a3)
 expr_ty _Ta27_Num(object n, int lineno, int col_offset, PyArena *arena);
 #define Str(a0, a1, a2, a3, a4) _Ta27_Str(a0, a1, a2, a3, a4)
-expr_ty _Ta27_Str(string s, int has_b, int lineno, int col_offset, PyArena *arena);
+expr_ty _Ta27_Str(string s, string kind, int lineno, int col_offset, PyArena *arena);
 #define Attribute(a0, a1, a2, a3, a4, a5) _Ta27_Attribute(a0, a1, a2, a3, a4, a5)
 expr_ty _Ta27_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
                         col_offset, PyArena *arena);

ast27/Parser/Python.asdl

@@ -71,7 +71,7 @@ module Python version "$Revision$"
 			 expr? starargs, expr? kwargs)
 	     | Repr(expr value)
 	     | Num(object n) -- a number as a PyObject.
-	     | Str(string s, int? has_b) -- need to specify raw, unicode, etc?
+	     | Str(string s, string kind)
 	     -- other literals? bools?
 
 	     -- the following expression can appear in assignment context

ast27/Python/Python-ast.c

@@ -252,7 +252,7 @@ static char *Num_fields[]={
 static PyTypeObject *Str_type;
 static char *Str_fields[]={
         "s",
-        "has_b",
+        "kind",
 };
 static PyTypeObject *Attribute_type;
 static char *Attribute_fields[]={
@@ -1850,20 +1850,25 @@ Num(object n, int lineno, int col_offset, PyArena *arena)
 }
 
 expr_ty
-Str(string s, int has_b, int lineno, int col_offset, PyArena *arena)
+Str(string s, string kind, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         if (!s) {
                 PyErr_SetString(PyExc_ValueError,
                                 "field s is required for Str");
                 return NULL;
         }
+        if (!kind) {
+                PyErr_SetString(PyExc_ValueError,
+                                "field kind is required for Str");
+                return NULL;
+        }
         p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
         if (!p)
                 return NULL;
         p->kind = Str_kind;
         p->v.Str.s = s;
-        p->v.Str.has_b = has_b;
+        p->v.Str.kind = kind;
         p->lineno = lineno;
         p->col_offset = col_offset;
         return p;
@@ -2889,9 +2894,9 @@ ast2obj_expr(void* _o)
                 if (PyObject_SetAttrString(result, "s", value) == -1)
                         goto failed;
                 Py_DECREF(value);
-                value = ast2obj_int(o->v.Str.has_b);
+                value = ast2obj_string(o->v.Str.kind);
                 if (!value) goto failed;
-                if (PyObject_SetAttrString(result, "has_b", value) == -1)
+                if (PyObject_SetAttrString(result, "kind", value) == -1)
                         goto failed;
                 Py_DECREF(value);
                 break;
@@ -5714,7 +5719,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         }
         if (isinstance) {
                 string s;
-                int has_b;
+                string kind;
 
                 if (PyObject_HasAttrString(obj, "s")) {
                         int res;
@@ -5728,18 +5733,19 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
                         PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str");
                         return 1;
                 }
-                if (PyObject_HasAttrString(obj, "has_b")) {
+                if (PyObject_HasAttrString(obj, "kind")) {
                         int res;
-                        tmp = PyObject_GetAttrString(obj, "has_b");
+                        tmp = PyObject_GetAttrString(obj, "kind");
                         if (tmp == NULL) goto failed;
-                        res = obj2ast_int(tmp, &has_b, arena);
+                        res = obj2ast_string(tmp, &kind, arena);
                         if (res != 0) goto failed;
                         Py_XDECREF(tmp);
                         tmp = NULL;
                 } else {
-                        has_b = 0;
+                        PyErr_SetString(PyExc_TypeError, "required field \"kind\" missing from Str");
+                        return 1;
                 }
-                *out = Str(s, has_b, lineno, col_offset, arena);
+                *out = Str(s, kind, lineno, col_offset, arena);
                 if (*out == NULL) goto failed;
                 return 0;
         }

ast27/Python/ast.c

@@ -1498,10 +1498,20 @@ ast_for_atom(struct compiling *c, const node *n)
         return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
     }
     case STRING: {
-        PyObject *str = parsestrplus(c, n);
-        const char *s = STR(CHILD(n, 0));
+        PyObject *kind, *str = parsestrplus(c, n);
+        const char *raw, *s = STR(CHILD(n, 0));
         int quote = Py_CHARMASK(*s);
-        int has_b = 0;
+        /* currently Python allows up to 2 string modifiers */
+        char *ch, s_kind[3] = {0, 0, 0};
+        ch = s_kind;
+        raw = s;
+        while (*raw && *raw != '\'' && *raw != '"') {
+            *ch++ = *raw++;
+        }
+        kind = PyUnicode_FromString(s_kind);
+        if (!kind) {
+            return NULL;
+        }
         if (!str) {
 #ifdef Py_USING_UNICODE
             if (PyErr_ExceptionMatches(PyExc_UnicodeError)){
@@ -1526,10 +1536,7 @@ ast_for_atom(struct compiling *c, const node *n)
             return NULL;
         }
         PyArena_AddPyObject(c->c_arena, str);
-        if (quote == 'b' || quote == 'B') {
-            has_b = 1;
-        }
-        return Str(str, has_b, LINENO(n), n->n_col_offset, c->c_arena);
+        return Str(str, kind, LINENO(n), n->n_col_offset, c->c_arena);
     }
     case NUMBER: {
         PyObject *pynum = parsenumber(c, STR(ch));

ast3/Include/Python-ast.h

@@ -318,6 +318,7 @@ struct _expr {
 
         struct {
             string s;
+            string kind;
         } Str;
 
         struct {
@@ -601,8 +602,9 @@ expr_ty _Ta3_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
                   lineno, int col_offset, PyArena *arena);
 #define Num(a0, a1, a2, a3) _Ta3_Num(a0, a1, a2, a3)
 expr_ty _Ta3_Num(object n, int lineno, int col_offset, PyArena *arena);
-#define Str(a0, a1, a2, a3) _Ta3_Str(a0, a1, a2, a3)
-expr_ty _Ta3_Str(string s, int lineno, int col_offset, PyArena *arena);
+#define Str(a0, a1, a2, a3, a4) _Ta3_Str(a0, a1, a2, a3, a4)
+expr_ty _Ta3_Str(string s, string kind, int lineno, int col_offset, PyArena
+                 *arena);
 #define FormattedValue(a0, a1, a2, a3, a4, a5) _Ta3_FormattedValue(a0, a1, a2, a3, a4, a5)
 expr_ty _Ta3_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
                             int lineno, int col_offset, PyArena *arena);

ast3/Parser/Python.asdl

@@ -77,7 +77,7 @@ module Python
          | Compare(expr left, cmpop* ops, expr* comparators)
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
-         | Str(string s) -- need to specify raw, unicode, etc?
+         | Str(string s, string kind)
          | FormattedValue(expr value, int? conversion, expr? format_spec)
          | JoinedStr(expr* values)
          | Bytes(bytes s)

ast3/Python/Python-ast.c

@@ -307,8 +307,10 @@ static char *Num_fields[]={
 };
 static PyTypeObject *Str_type;
 _Py_IDENTIFIER(s);
+_Py_IDENTIFIER(kind);
 static char *Str_fields[]={
     "s",
+    "kind",
 };
 static PyTypeObject *FormattedValue_type;
 _Py_IDENTIFIER(conversion);
@@ -983,7 +985,7 @@ static int init_types(void)
     if (!Call_type) return 0;
     Num_type = make_type("Num", expr_type, Num_fields, 1);
     if (!Num_type) return 0;
-    Str_type = make_type("Str", expr_type, Str_fields, 1);
+    Str_type = make_type("Str", expr_type, Str_fields, 2);
     if (!Str_type) return 0;
     FormattedValue_type = make_type("FormattedValue", expr_type,
                                     FormattedValue_fields, 3);
@@ -2181,19 +2183,25 @@ Num(object n, int lineno, int col_offset, PyArena *arena)
 }
 
 expr_ty
-Str(string s, int lineno, int col_offset, PyArena *arena)
+Str(string s, string kind, int lineno, int col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!s) {
         PyErr_SetString(PyExc_ValueError,
                         "field s is required for Str");
         return NULL;
     }
+    if (!kind) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field kind is required for Str");
+        return NULL;
+    }
     p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
     if (!p)
         return NULL;
     p->kind = Str_kind;
     p->v.Str.s = s;
+    p->v.Str.kind = kind;
     p->lineno = lineno;
     p->col_offset = col_offset;
     return p;
@@ -3448,6 +3456,11 @@ ast2obj_expr(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_s, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.Str.kind);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case FormattedValue_kind:
         result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
@@ -6766,6 +6779,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
     }
     if (isinstance) {
         string s;
+        string kind;
 
         if (_PyObject_HasAttrId(obj, &PyId_s)) {
             int res;
@@ -6778,7 +6792,18 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str");
             return 1;
         }
-        *out = Str(s, lineno, col_offset, arena);
+        if (_PyObject_HasAttrId(obj, &PyId_kind)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_kind);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_string(tmp, &kind, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"kind\" missing from Str");
+            return 1;
+        }
+        *out = Str(s, kind, lineno, col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }

ast3/Python/ast.c

@@ -4971,14 +4971,25 @@ FstringParser_Dealloc(FstringParser *state)
 static expr_ty
 make_str_node_and_del(PyObject **str, struct compiling *c, const node* n)
 {
-    PyObject *s = *str;
+    PyObject *kind, *s = *str;
+    const char *raw = STR(CHILD(n, 0));
+    /* currently Python allows up to 2 string modifiers */
+    char *ch, s_kind[3] = {0, 0, 0};
+    ch = s_kind;
+    while (*raw && *raw != '\'' && *raw != '"') {
+        *ch++ = *raw++;
+    }
+    kind = PyUnicode_FromString(s_kind);
+    if (!kind) {
+        return NULL;
+    }
     *str = NULL;
     assert(PyUnicode_CheckExact(s));
     if (PyArena_AddPyObject(c->c_arena, s) < 0) {
         Py_DECREF(s);
         return NULL;
     }
-    return Str(s, LINENO(n), n->n_col_offset, c->c_arena);
+    return Str(s, kind, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 /* Add a non-f-string (that is, a regular literal string). str is