bpo-43950: Add option to opt-out of PEP-657 (GH-27023)

Co-authored-by: Pablo Galindo <[email protected]>
Co-authored-by: Batuhan Taskaya <[email protected]>
Co-authored-by: Ammar Askar <[email protected]>

Author: 3 people

Doc/c-api/init_config.rst

@@ -596,6 +596,16 @@ PyConfig
 
       .. versionadded:: 3.10
 
+   .. c:member:: int no_debug_ranges
+
+      If equals to ``1``, disables the inclusion of the end line and column
+      mappings in code objects. Also disables traceback printing carets to
+      specific error locations.
+
+      Default: ``0``.
+
+      .. versionadded:: 3.11
+
    .. c:member:: wchar_t* check_hash_pycs_mode
 
       Control the validation behavior of hash-based ``.pyc`` files:

Doc/using/cmdline.rst

@@ -474,6 +474,12 @@ Miscellaneous options
    * ``-X warn_default_encoding`` issues a :class:`EncodingWarning` when the
      locale-specific default encoding is used for opening files.
      See also :envvar:`PYTHONWARNDEFAULTENCODING`.
+   * ``-X no_debug_ranges`` disables the inclusion of the tables mapping extra
+     location information (end line, start column offset and end column offset)
+     to every instruction in code objects. This is useful when smaller code
+     objects and pyc files are desired as well as supressing the extra visual
+     location indicators when the interpreter displays tracebacks. See also
+     :envvar:`PYTHONNODEBUGRANGES`.
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -509,6 +515,9 @@ Miscellaneous options
    .. deprecated-removed:: 3.9 3.10
       The ``-X oldparser`` option.
 
+   .. versionadded:: 3.11
+      The ``-X no_debug_ranges`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -936,6 +945,17 @@ conflict.
 
    .. versionadded:: 3.10
 
+.. envvar:: PYTHONNODEBUGRANGES
+
+   If this variable is set, it disables the inclusion of the tables mapping
+   extra location information (end line, start column offset and end column
+   offset) to every instruction in code objects. This is useful when smaller
+   code objects and pyc files are desired as well as supressing the extra visual
+   location indicators when the interpreter displays tracebacks.
+
+   .. versionadded:: 3.11
+
+
 
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~

Include/cpython/initconfig.h

@@ -140,6 +140,7 @@ typedef struct PyConfig {
     int faulthandler;
     int tracemalloc;
     int import_time;
+    int no_debug_ranges;
     int show_ref_count;
     int dump_refs;
     int malloc_stats;

Lib/idlelib/idle_test/test_run.py

@@ -3,7 +3,7 @@
 from idlelib import run
 import io
 import sys
-from test.support import captured_output, captured_stderr
+from test.support import captured_output, captured_stderr, has_no_debug_ranges
 import unittest
 from unittest import mock
 import idlelib
@@ -33,9 +33,14 @@ def __eq__(self, other):
                         run.print_exception()
 
         tb = output.getvalue().strip().splitlines()
-        self.assertEqual(13, len(tb))
-        self.assertIn('UnhashableException: ex2', tb[4])
-        self.assertIn('UnhashableException: ex1', tb[12])
+        if has_no_debug_ranges():
+            self.assertEqual(11, len(tb))
+            self.assertIn('UnhashableException: ex2', tb[3])
+            self.assertIn('UnhashableException: ex1', tb[10])
+        else:
+            self.assertEqual(13, len(tb))
+            self.assertIn('UnhashableException: ex2', tb[4])
+            self.assertIn('UnhashableException: ex1', tb[12])
 
     data = (('1/0', ZeroDivisionError, "division by zero\n"),
             ('abc', NameError, "name 'abc' is not defined. "

Lib/test/_test_embed_set_config.py

@@ -61,6 +61,7 @@ def test_set_invalid(self):
             'faulthandler',
             'tracemalloc',
             'import_time',
+            'no_debug_ranges',
             'show_ref_count',
             'dump_refs',
             'malloc_stats',

Lib/test/support/__init__.py

@@ -415,6 +415,14 @@ def requires_lzma(reason='requires lzma'):
         lzma = None
     return unittest.skipUnless(lzma, reason)
 
+def has_no_debug_ranges():
+    import _testinternalcapi
+    config = _testinternalcapi.get_config()
+    return bool(config['no_debug_ranges'])
+
+def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
+    return unittest.skipIf(has_no_debug_ranges(), reason)
+
 requires_legacy_unicode_capi = unittest.skipUnless(unicode_legacy_string,
                         'requires legacy Unicode C API')
 

Lib/test/test_code.py

@@ -137,7 +137,8 @@
 except ImportError:
     ctypes = None
 from test.support import (run_doctest, run_unittest, cpython_only,
-                          check_impl_detail)
+                          check_impl_detail, requires_debug_ranges)
+from test.support.script_helper import assert_python_ok
 
 
 def consts(t):
@@ -325,6 +326,7 @@ def func():
         new_code = code = func.__code__.replace(co_linetable=b'')
         self.assertEqual(list(new_code.co_lines()), [])
 
+    @requires_debug_ranges()
     def test_co_positions_artificial_instructions(self):
         import dis
 
@@ -372,8 +374,32 @@ def test_co_positions_artificial_instructions(self):
             ]
         )
 
+    def test_endline_and_columntable_none_when_no_debug_ranges(self):
+        # Make sure that if `-X no_debug_ranges` is used, the endlinetable and
+        # columntable are None.
+        code = textwrap.dedent("""
+            def f():
+                pass
+
+            assert f.__code__.co_endlinetable is None
+            assert f.__code__.co_columntable is None
+            """)
+        assert_python_ok('-X', 'no_debug_ranges', '-c', code, __cleanenv=True)
+
+    def test_endline_and_columntable_none_when_no_debug_ranges_env(self):
+        # Same as above but using the environment variable opt out.
+        code = textwrap.dedent("""
+            def f():
+                pass
+
+            assert f.__code__.co_endlinetable is None
+            assert f.__code__.co_columntable is None
+            """)
+        assert_python_ok('-c', code, PYTHONNODEBUGRANGES='1', __cleanenv=True)
+
     # co_positions behavior when info is missing.
 
+    @requires_debug_ranges()
     def test_co_positions_empty_linetable(self):
         def func():
             x = 1
@@ -382,6 +408,7 @@ def func():
             self.assertIsNone(line)
             self.assertEqual(end_line, new_code.co_firstlineno + 1)
 
+    @requires_debug_ranges()
     def test_co_positions_empty_endlinetable(self):
         def func():
             x = 1
@@ -390,6 +417,7 @@ def func():
             self.assertEqual(line, new_code.co_firstlineno + 1)
             self.assertIsNone(end_line)
 
+    @requires_debug_ranges()
     def test_co_positions_empty_columntable(self):
         def func():
             x = 1

Lib/test/test_compile.py

@@ -9,7 +9,7 @@
 import types
 import textwrap
 from test import support
-from test.support import script_helper
+from test.support import script_helper, requires_debug_ranges
 from test.support.os_helper import FakePath
 
 
@@ -985,7 +985,7 @@ def if_else_break():
             elif instr.opname in HANDLED_JUMPS:
                 self.assertNotEqual(instr.arg, (line + 1)*INSTR_SIZE)
 
-
+@requires_debug_ranges()
 class TestSourcePositions(unittest.TestCase):
     # Ensure that compiled code snippets have correct line and column numbers
     # in `co_positions()`.

Lib/test/test_dis.py

@@ -1,6 +1,6 @@
 # Minimal tests for dis module
 
-from test.support import captured_stdout
+from test.support import captured_stdout, requires_debug_ranges
 from test.support.bytecode_helper import BytecodeTestCase
 import unittest
 import sys
@@ -1192,6 +1192,7 @@ def test_jumpy(self):
         actual = dis.get_instructions(jumpy, first_line=expected_jumpy_line)
         self.assertInstructionsEqual(list(actual), expected_opinfo_jumpy)
 
+    @requires_debug_ranges()
     def test_co_positions(self):
         code = compile('f(\n  x, y, z\n)', '<test>', 'exec')
         positions = [
@@ -1210,6 +1211,7 @@ def test_co_positions(self):
         ]
         self.assertEqual(positions, expected)
 
+    @requires_debug_ranges()
     def test_co_positions_missing_info(self):
         code = compile('x, y, z', '<test>', 'exec')
         code_without_column_table = code.replace(co_columntable=b'')

Lib/test/test_doctest.py

@@ -2808,10 +2808,12 @@ def test_testmod(): r"""
 
 try:
     os.fsencode("foo-bä[email protected]")
+    supports_unicode = True
 except UnicodeEncodeError:
     # Skip the test: the filesystem encoding is unable to encode the filename
-    pass
-else:
+    supports_unicode = False
+
+if supports_unicode and not support.has_no_debug_ranges():
     def test_unicode(): """
 Check doctest with a non-ascii filename:
 

Lib/test/test_embed.py

@@ -369,6 +369,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'faulthandler': 0,
         'tracemalloc': 0,
         'import_time': 0,
+        'no_debug_ranges': 0,
         'show_ref_count': 0,
         'dump_refs': 0,
         'malloc_stats': 0,
@@ -798,6 +799,7 @@ def test_init_from_config(self):
             'hash_seed': 123,
             'tracemalloc': 2,
             'import_time': 1,
+            'no_debug_ranges': 1,
             'show_ref_count': 1,
             'malloc_stats': 1,
 
@@ -858,6 +860,7 @@ def test_init_compat_env(self):
             'hash_seed': 42,
             'tracemalloc': 2,
             'import_time': 1,
+            'no_debug_ranges': 1,
             'malloc_stats': 1,
             'inspect': 1,
             'optimization_level': 2,
@@ -887,6 +890,7 @@ def test_init_python_env(self):
             'hash_seed': 42,
             'tracemalloc': 2,
             'import_time': 1,
+            'no_debug_ranges': 1,
             'malloc_stats': 1,
             'inspect': 1,
             'optimization_level': 2,

Lib/test/test_marshal.py

@@ -1,12 +1,14 @@
 from test import support
-from test.support import os_helper
+from test.support import os_helper, requires_debug_ranges
+from test.support.script_helper import assert_python_ok
 import array
 import io
 import marshal
 import sys
 import unittest
 import os
 import types
+import textwrap
 
 try:
     import _testcapi
@@ -126,6 +128,31 @@ def test_different_filenames(self):
         self.assertEqual(co1.co_filename, "f1")
         self.assertEqual(co2.co_filename, "f2")
 
+    @requires_debug_ranges()
+    def test_no_columntable_and_endlinetable_with_no_debug_ranges(self):
+        # Make sure when demarshalling objects with `-X no_debug_ranges`
+        # that the columntable and endlinetable are None.
+        co = ExceptionTestCase.test_exceptions.__code__
+        code = textwrap.dedent("""
+        import sys
+        import marshal
+        with open(sys.argv[1], 'rb') as f:
+            co = marshal.load(f)
+
+            assert co.co_endlinetable is None
+            assert co.co_columntable is None
+        """)
+
+        try:
+            with open(os_helper.TESTFN, 'wb') as f:
+                marshal.dump(co, f)
+
+            assert_python_ok('-X', 'no_debug_ranges',
+                             '-c', code, os_helper.TESTFN,
+                             __cleanenv=True)
+        finally:
+            os_helper.unlink(os_helper.TESTFN)
+
     @support.cpython_only
     def test_same_filename_used(self):
         s = """def f(): pass\ndef g(): pass"""