Produce cleaner bytecode for 'with' and 'async with' by generating separate code for normal and exceptional paths. (#6641)

Remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY and POP_FINALLY bytecodes. Implement finally blocks by code duplication.
Reimplement frame.lineno setter using line numbers rather than bytecode offsets.

Author: markshannon

Doc/library/dis.rst

@@ -706,50 +706,21 @@ iterations of the loop.
    popped values are used to restore the exception state.
 
 
-.. opcode:: POP_FINALLY (preserve_tos)
+.. opcode:: RERAISE
 
-   Cleans up the value stack and the block stack.  If *preserve_tos* is not
-   ``0`` TOS first is popped from the stack and pushed on the stack after
-   performing other stack operations:
+    Re-raises the exception currently on top of the stack.
 
-   * If TOS is ``NULL`` or an integer (pushed by :opcode:`BEGIN_FINALLY`
-     or :opcode:`CALL_FINALLY`) it is popped from the stack.
-   * If TOS is an exception type (pushed when an exception has been raised)
-     6 values are popped from the stack, the last three popped values are
-     used to restore the exception state.  An exception handler block is
-     removed from the block stack.
+    .. versionadded:: 3.8
 
-   It is similar to :opcode:`END_FINALLY`, but doesn't change the bytecode
-   counter nor raise an exception.  Used for implementing :keyword:`break`,
-   :keyword:`continue` and :keyword:`return` in the :keyword:`finally` block.
 
-   .. versionadded:: 3.8
-
-
-.. opcode:: BEGIN_FINALLY
-
-   Pushes ``NULL`` onto the stack for using it in :opcode:`END_FINALLY`,
-   :opcode:`POP_FINALLY`, :opcode:`WITH_CLEANUP_START` and
-   :opcode:`WITH_CLEANUP_FINISH`.  Starts the :keyword:`finally` block.
-
-   .. versionadded:: 3.8
+.. opcode:: WITH_EXCEPT_START
 
+    Calls the function in position 7 on the stack with the top three
+    items on the stack as arguments.
+    Used to implement the call ``context_manager.__exit__(*exc_info())`` when an exception
+    has occurred in a :keyword:`with` statement.
 
-.. opcode:: END_FINALLY
-
-   Terminates a :keyword:`finally` clause.  The interpreter recalls whether the
-   exception has to be re-raised or execution has to be continued depending on
-   the value of TOS.
-
-   * If TOS is ``NULL`` (pushed by :opcode:`BEGIN_FINALLY`) continue from
-     the next instruction. TOS is popped.
-   * If TOS is an integer (pushed by :opcode:`CALL_FINALLY`), sets the
-     bytecode counter to TOS.  TOS is popped.
-   * If TOS is an exception type (pushed when an exception has been raised)
-     6 values are popped from the stack, the first three popped values are
-     used to re-raise the exception and the last three popped values are used
-     to restore the exception state.  An exception handler block is removed
-     from the block stack.
+    .. versionadded:: 3.8
 
 
 .. opcode:: LOAD_ASSERTION_ERROR
@@ -780,35 +751,6 @@ iterations of the loop.
    .. versionadded:: 3.2
 
 
-.. opcode:: WITH_CLEANUP_START
-
-   Starts cleaning up the stack when a :keyword:`with` statement block exits.
-
-   At the top of the stack are either ``NULL`` (pushed by
-   :opcode:`BEGIN_FINALLY`) or 6 values pushed if an exception has been
-   raised in the with block.  Below is the context manager's
-   :meth:`~object.__exit__` or :meth:`~object.__aexit__` bound method.
-
-   If TOS is ``NULL``, calls ``SECOND(None, None, None)``,
-   removes the function from the stack, leaving TOS, and pushes ``None``
-   to the stack.  Otherwise calls ``SEVENTH(TOP, SECOND, THIRD)``,
-   shifts the bottom 3 values of the stack down, replaces the empty spot
-   with ``NULL`` and pushes TOS.  Finally pushes the result of the call.
-
-
-.. opcode:: WITH_CLEANUP_FINISH
-
-   Finishes cleaning up the stack when a :keyword:`with` statement block exits.
-
-   TOS is result of ``__exit__()`` or ``__aexit__()`` function call pushed
-   by :opcode:`WITH_CLEANUP_START`.  SECOND is ``None`` or an exception type
-   (pushed when an exception has been raised).
-
-   Pops two values from the stack.  If SECOND is not None and TOS is true
-   unwinds the EXCEPT_HANDLER block which was created when the exception
-   was caught and pushes ``NULL`` to the stack.
-
-
 All of the following opcodes use their arguments.
 
 .. opcode:: STORE_NAME (namei)
@@ -1060,15 +1002,6 @@ All of the following opcodes use their arguments.
    stack.  *delta* points to the finally block or the first except block.
 
 
-.. opcode:: CALL_FINALLY (delta)
-
-   Pushes the address of the next instruction onto the stack and increments
-   bytecode counter by *delta*.  Used for calling the finally block as a
-   "subroutine".
-
-   .. versionadded:: 3.8
-
-
 .. opcode:: LOAD_FAST (var_num)
 
    Pushes a reference to the local ``co_varnames[var_num]`` onto the stack.

Include/opcode.h

@@ -272,6 +272,8 @@ def _write_atomic(path, data, mode=0o666):
 #                         only args in ast.arguments #37593)
 #     Python 3.8b4  3413 (Fix "break" and "continue" in "finally" #37830)
 #     Python 3.9a0  3420 (add LOAD_ASSERTION_ERROR #34880)
+#     Python 3.9a0  3421 (simplified bytecode for with blocks #32949)
+#     Python 3.9a0  3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -280,7 +282,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3420).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3422).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/importlib/_bootstrap_external.py

@@ -84,10 +84,12 @@ def jabs_op(name, op):
 def_op('INPLACE_FLOOR_DIVIDE', 28)
 def_op('INPLACE_TRUE_DIVIDE', 29)
 
+def_op('RERAISE', 48)
+def_op('WITH_EXCEPT_START', 49)
 def_op('GET_AITER', 50)
 def_op('GET_ANEXT', 51)
 def_op('BEFORE_ASYNC_WITH', 52)
-def_op('BEGIN_FINALLY', 53)
+
 def_op('END_ASYNC_FOR', 54)
 def_op('INPLACE_ADD', 55)
 def_op('INPLACE_SUBTRACT', 56)
@@ -115,14 +117,13 @@ def jabs_op(name, op):
 def_op('INPLACE_AND', 77)
 def_op('INPLACE_XOR', 78)
 def_op('INPLACE_OR', 79)
-def_op('WITH_CLEANUP_START', 81)
-def_op('WITH_CLEANUP_FINISH', 82)
+
 def_op('RETURN_VALUE', 83)
 def_op('IMPORT_STAR', 84)
 def_op('SETUP_ANNOTATIONS', 85)
 def_op('YIELD_VALUE', 86)
 def_op('POP_BLOCK', 87)
-def_op('END_FINALLY', 88)
+
 def_op('POP_EXCEPT', 89)
 
 HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
@@ -210,7 +211,5 @@ def jabs_op(name, op):
 
 name_op('LOAD_METHOD', 160)
 def_op('CALL_METHOD', 161)
-jrel_op('CALL_FINALLY', 162)
-def_op('POP_FINALLY', 163)
 
 del def_op, name_op, jrel_op, jabs_op