gh-74917: Raise TypeError in the JSON encoder if the default function returns the same type

Doc/library/json.rst

@@ -486,6 +486,10 @@ Encoders and Decoders
    .. versionchanged:: 3.6
       All parameters are now :ref:`keyword-only <keyword-only_parameter>`.
 
+   .. versionchanged:: 3.13
+      :exc:`TypeError` is raised if the default serialization function
+      returns a result of the same type as the input.
+
 
    .. method:: default(o)
 

Doc/whatsnew/3.13.rst

@@ -1163,6 +1163,10 @@ Changes in the Python API
   other "private" attributes.
   (See :gh:`112826`.)
 
+* :exc:`TypeError` is now raised in the :mod:`json` encoder if the default
+  serialization function returns a result of the same type as the input.
+  (Contributed by Siddharth Velankar and Serhiy Storchaka in :gh:`74917`.)
+
 
 Build Changes
 =============

Lib/json/encoder.py

@@ -436,8 +436,11 @@ def _iterencode(o, _current_indent_level):
                 if markerid in markers:
                     raise ValueError("Circular reference detected")
                 markers[markerid] = o
-            o = _default(o)
-            yield from _iterencode(o, _current_indent_level)
+            new = _default(o)
+            if type(new) is type(o):
+                raise TypeError("Default serialization function "
+                                "returning same type")
+            yield from _iterencode(new, _current_indent_level)
             if markers is not None:
                 del markers[markerid]
     return _iterencode

Lib/test/test_json/test_recursion.py

@@ -65,6 +65,9 @@ def default(self, o):
         else:
             self.fail("didn't raise ValueError on default recursion")
 
+    def test_default_returns_same_type(self):
+        with self.assertRaisesRegex(TypeError, ".*same type"):
+            self.dumps(JSONTestObject(), default=lambda o: JSONTestObject())
 
     def test_highly_nested_objects_decoding(self):
         # test that loading highly-nested objects doesn't segfault when C

Misc/NEWS.d/next/Library/2023-12-24-15-39-00.gh-issue-74917.48uE7k.rst

@@ -0,0 +1,2 @@
+:exc:`TypeError` is now raised in the :mod:`json` encoder if the default
+serialization function returns a result of the same type as the input.

Modules/_json.c

@@ -1430,6 +1430,13 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer,
             Py_XDECREF(ident);
             return -1;
         }
+        if (Py_TYPE(newobj) == Py_TYPE(obj)) {
+            PyErr_SetString(PyExc_TypeError, "Default serialization function "
+                            "returning same type");
+            Py_DECREF(newobj);
+            Py_XDECREF(ident);
+            return -1;
+        }
 
         if (_Py_EnterRecursiveCall(" while encoding a JSON object")) {
             Py_DECREF(newobj);