diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst new file mode 100644 index 00000000000000..bbc945016e2880 --- /dev/null +++ b/Doc/library/_interpreters.rst @@ -0,0 +1,90 @@ +:mod:`_interpreters` --- Low-level interpreters API +=================================================== + +.. module:: _interpreters + :synopsis: Low-level interpreters API. + +.. versionadded:: 3,7 + +-------------- + +This module provides low-level primitives for working with multiple +Python interpreters in the same runtime in the current process. + +More information about (sub)interpreters is found at +:ref:`sub-interpreter-support`, including what data is shared between +interpreters and what is unique. Note particularly that interpreters +aren't inherently threaded, even though they track and manage Python +threads. To run code in an interpreter in a different OS thread, call +:func:`run_string` in a function that you run in a new Python thread. +For example:: + + id = _interpreters.create() + def f(): + _interpreters.run_string(id, 'print("in a thread")') + + t = threading.Thread(target=f) + t.start() + +This module is optional. It is provided by Python implementations which +support multiple interpreters. + +It defines the following functions: + +.. function:: enumerate() + + Return a list of the IDs of every existing interpreter. + + +.. function:: get_current() + + Return the ID of the currently running interpreter. + + +.. function:: get_main() + + Return the ID of the main interpreter. + + +.. function:: is_running(id) + + Return whether or not the identified interpreter is currently + running any code. + + +.. function:: create() + + Initialize a new Python interpreter and return its identifier. The + interpreter will be created in the current thread and will remain + idle until something is run in it. + + +.. function:: destroy(id) + + Finalize and destroy the identified interpreter. + + +.. function:: run_string(id, command) + + A wrapper around :c:func:`PyRun_SimpleString` which runs the provided + Python program in the main thread of the identified interpreter. + Providing an invalid or unknown ID results in a RuntimeError, + likewise if the main interpreter or any other running interpreter + is used. + + Any value returned from the code is thrown away, similar to what + threads do. If the code results in an exception then that exception + is raised in the thread in which run_string() was called, similar to + how :func:`exec` works. This aligns with how interpreters are not + inherently threaded. Note that SystemExit (as raised by sys.exit()) + is not treated any differently and will result in the process ending + if not caught explicitly. + + +.. function:: run_string_unrestricted(id, command, ns=None) + + Like :c:func:`run_string` but returns the dict in which the code + was executed. It also supports providing a namespace that gets + merged into the execution namespace before execution. Note that + this allows objects to leak between interpreters, which may not + be desirable. diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 0de281bd149531..76b9c52f593555 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -29,3 +29,4 @@ The following are support modules for some of the above services: dummy_threading.rst _thread.rst _dummy_thread.rst + _interpreters.rst diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst new file mode 100644 index 00000000000000..bef62d30a673fe --- /dev/null +++ b/Doc/library/interpreters.rst @@ -0,0 +1,64 @@ +:mod:`interpreters` --- high-level interpreters API +=================================================== + +.. module:: interpreters + :synopsis: High-level interpreters API. + +**Source code:** :source:`Lib/interpreters.py` + +.. versionadded:: 3,7 + +-------------- + +This module provides high-level interfaces for interacting with +the Python interpreters. It is built on top of the lower- level +:mod:`_interpreters` module. + +.. XXX Summarize :ref:`_sub-interpreter-support` here. + +This module defines the following functions: + +.. function:: enumerate() + + Return a list of all existing interpreters. + + +.. function:: get_current() + + Return the currently running interpreter. + + +.. function:: get_main() + + Return the main interpreter. + + +.. function:: create() + + Initialize a new Python interpreter and return it. The + interpreter will be created in the current thread and will remain + idle until something is run in it. + + +This module also defines the following class: + +.. class:: Interpreter(id) + + ``id`` is the interpreter's ID. + + .. property:: id + + The interpreter's ID. + + .. method:: is_running() + + Return whether or not the interpreter is currently running. + + .. method:: destroy() + + Finalize and destroy the interpreter. + + .. method:: run(code) + + Run the provided Python code in the interpreter, in the current + OS thread. Supported code: source text. diff --git a/Lib/interpreters.py b/Lib/interpreters.py new file mode 100644 index 00000000000000..3bfc567b6a79b0 --- /dev/null +++ b/Lib/interpreters.py @@ -0,0 +1,79 @@ +"""Interpreters module providing a high-level API to Python interpreters.""" + +import _interpreters + + +__all__ = ['enumerate', 'current', 'main', 'create', 'Interpreter'] + + +def enumerate(): + """Return a list of all existing interpreters.""" + return [Interpreter(id) + for id in _interpreters.enumerate()] + + +def current(): + """Return the currently running interpreter.""" + id = _interpreters.get_current() + return Interpreter(id) + + +def main(): + """Return the main interpreter.""" + id = _interpreters.get_main() + return Interpreter(id) + + +def create(): + """Return a new Interpreter. + + The interpreter is created in the current OS thread. It will remain + idle until its run() method is called. + """ + id = _interpreters.create() + return Interpreter(id) + + +class Interpreter: + """A single Python interpreter in the current runtime.""" + + def __init__(self, id): + self._id = id + + def __repr__(self): + return '{}(id={!r})'.format(type(self).__name__, self._id) + + def __eq__(self, other): + try: + other_id = other.id + except AttributeError: + return False + return self._id == other_id + + @property + def id(self): + """The interpreter's ID.""" + return self._id + + def is_running(self): + """Return whether or not the interpreter is currently running.""" + return _interpreters.is_running(self._id) + + def destroy(self): + """Finalize and destroy the interpreter. + + After calling destroy(), all operations on this interpreter + will fail. + """ + _interpreters.destroy(self._id) + + def run(self, code): + """Run the provided Python code in this interpreter. + + If the code is a string then it will be run as it would by + calling exec(). Other kinds of code are not supported. + """ + if isinstance(code, str): + _interpreters.run_string(self._id, code) + else: + raise NotImplementedError diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py new file mode 100644 index 00000000000000..29d04aa4d00cfb --- /dev/null +++ b/Lib/test/test__interpreters.py @@ -0,0 +1,582 @@ +import contextlib +import os +import os.path +import shutil +import tempfile +from textwrap import dedent, indent +import threading +import unittest + +from test import support +from test.support import script_helper + +interpreters = support.import_module('_interpreters') + + +SCRIPT_THREADED_INTERP = """\ +from textwrap import dedent +import threading +import _interpreters +def f(): + _interpreters.run_string(id, dedent(''' + {} + ''')) + +t = threading.Thread(target=f) +t.start() +""" + + +@contextlib.contextmanager +def _blocked(dirname): + filename = os.path.join(dirname, '.lock') + wait_script = dedent(""" + import os.path + import time + while not os.path.exists('{}'): + time.sleep(0.1) + """).format(filename) + try: + yield wait_script + finally: + support.create_empty_file(filename) + + +class InterpreterTests(unittest.TestCase): + + def setUp(self): + self.dirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.dirname) + + def test_still_running_at_exit(self): + subscript = dedent(""" + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + """) + script = SCRIPT_THREADED_INTERP.format(indent(subscript, ' ')) + filename = script_helper.make_script(self.dirname, 'interp', script) + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + +class TestBase(unittest.TestCase): + + def tearDown(self): + for id in interpreters.enumerate(): + if id == 0: # main + continue + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + +class EnumerateTests(TestBase): + + def test_multiple(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + ids = interpreters.enumerate() + + self.assertEqual(set(ids), {main, id1, id2}) + + def test_main_only(self): + main, = interpreters.enumerate() + + self.assertEqual(main, 0) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main, = interpreters.enumerate() + id = interpreters.get_current() + + self.assertEqual(id, main) + + def test_sub(self): + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" + import _interpreters + id = _interpreters.get_current() + """)) + id2 = ns['id'] + + self.assertEqual(id2, id1) + + +class GetMainTests(TestBase): + + def test_main(self): + expected, = interpreters.enumerate() + main = interpreters.get_main() + + self.assertEqual(main, 0) + self.assertEqual(main, expected) + + +class IsRunningTests(TestBase): + + def test_main_running(self): + main, = interpreters.enumerate() + sub = interpreters.create() + main_running = interpreters.is_running(main) + sub_running = interpreters.is_running(sub) + + self.assertTrue(main_running) + self.assertFalse(sub_running) + + def test_sub_running(self): + main, = interpreters.enumerate() + sub1 = interpreters.create() + sub2 = interpreters.create() + ns = interpreters.run_string_unrestricted(sub1, dedent(f""" + import _interpreters + main = _interpreters.is_running({main}) + sub1 = _interpreters.is_running({sub1}) + sub2 = _interpreters.is_running({sub2}) + """)) + main_running = ns['main'] + sub1_running = ns['sub1'] + sub2_running = ns['sub2'] + + self.assertTrue(main_running) + self.assertTrue(sub1_running) + self.assertFalse(sub2_running) + + +class CreateTests(TestBase): + + def test_in_main(self): + id = interpreters.create() + + self.assertIn(id, interpreters.enumerate()) + + @unittest.skip('enable this test when working on pystate.c') + def test_unique_id(self): + seen = set() + for _ in range(100): + id = interpreters.create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_in_thread(self): + lock = threading.Lock() + id = None + def f(): + nonlocal id + id = interpreters.create() + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(id, interpreters.enumerate()) + + def test_in_subinterpreter(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" + import _interpreters + id = _interpreters.create() + """)) + id2 = ns['id'] + + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + ns = None + script = dedent(""" + import _interpreters + id = _interpreters.create() + """) + def f(): + nonlocal ns + ns = interpreters.run_string_unrestricted(id1, script) + + t = threading.Thread(target=f) + t.start() + t.join() + id2 = ns['id'] + + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) + + + def test_after_destroy_all(self): + before = set(interpreters.enumerate()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.enumerate()), before | {id}) + + def test_after_destroy_some(self): + before = set(interpreters.enumerate()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.enumerate()), before | {id, id2}) + + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + self.assertIn(id2, interpreters.enumerate()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters.enumerate()) + self.assertIn(id1, interpreters.enumerate()) + self.assertIn(id3, interpreters.enumerate()) + + def test_all(self): + before = set(interpreters.enumerate()) + ids = set() + for _ in range(3): + id = interpreters.create() + ids.add(id) + self.assertEqual(set(interpreters.enumerate()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters.enumerate()), before) + + def test_main(self): + main, = interpreters.enumerate() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + def test_from_current(self): + main, = interpreters.enumerate() + id = interpreters.create() + script = dedent(""" + import _interpreters + _interpreters.destroy({}) + """).format(id) + + with self.assertRaises(RuntimeError): + interpreters.run_string(id, script) + self.assertEqual(set(interpreters.enumerate()), {main, id}) + + def test_from_sibling(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + script = dedent(""" + import _interpreters + _interpreters.destroy({}) + """).format(id2) + interpreters.run_string(id1, script) + + self.assertEqual(set(interpreters.enumerate()), {main, id1}) + + def test_from_other_thread(self): + id = interpreters.create() + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_still_running(self): + # XXX Rewrite this test without files by using + # run_string_unrestricted(). + main, = interpreters.enumerate() + id = interpreters.create() + def f(): + interpreters.run_string(id, wait_script) + + dirname = tempfile.mkdtemp() + t = threading.Thread(target=f) + with _blocked(dirname) as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + t.join() + self.assertEqual(set(interpreters.enumerate()), {main, id}) + + +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + self.id = interpreters.create() + self.dirname = None + self.filename = None + + def tearDown(self): + if self.dirname is not None: + try: + shutil.rmtree(self.dirname) + except FileNotFoundError: + pass # already deleted + super().tearDown() + + def _resolve_filename(self, name=None): + if name is None: + name = self.FILENAME + if self.dirname is None: + self.dirname = tempfile.mkdtemp() + return os.path.join(self.dirname, name) + + def _empty_file(self): + self.filename = self._resolve_filename() + support.create_empty_file(self.filename) + return self.filename + + def assert_file_contains(self, expected, filename=None): + if filename is None: + filename = self.filename + self.assertIsNotNone(filename) + with open(filename) as out: + content = out.read() + self.assertEqual(content, expected) + + def test_success(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + def test_in_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assert_file_contains(expected) + + def test_create_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import threading + def f(): + with open('{}', 'w') as out: + out.write('{}') + + t = threading.Thread(target=f) + t.start() + t.join() + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import os + r, w = os.pipe() + pid = os.fork() + if pid == 0: # child + import sys + filename = '{}' + with open(filename, 'w') as out: + out.write('{}') + os.write(w, b'done!') + + # Kill the unittest runner in the child process. + os._exit(1) + else: + import select + try: + select.select([r], [], []) + finally: + os.close(r) + os.close(w) + """).format(filename, expected) + interpreters.run_string(self.id, script) + self.assert_file_contains(expected) + + def test_already_running(self): + def f(): + interpreters.run_string(self.id, wait_script) + + t = threading.Thread(target=f) + dirname = tempfile.mkdtemp() + with _blocked(dirname) as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + t.join() + + def test_does_not_exist(self): + id = 0 + while id in interpreters.enumerate(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + def test_invalid_syntax(self): + with self.assertRaises(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assertRaises(Exception) as caught: + interpreters.run_string(self.id, 'raise Exception("spam")') + self.assertEqual(str(caught.exception), 'spam') + + def test_sys_exit(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + self.assertIsNone(cm.exception.code) + + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + self.assertEqual(cm.exception.code, 42) + + def test_SystemError(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, 'raise SystemExit(42)') + self.assertEqual(cm.exception.code, 42) + + +class RunStringUnrestrictedTests(TestBase): + + def setUp(self): + self.id = interpreters.create() + + def test_without_ns(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + self.assertEqual(ns['spam'], 42) + + def test_with_ns(self): + updates = {'spam': 'ham', 'eggs': -1} + script = dedent(""" + spam = 42 + result = spam + eggs + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + + def test_ns_does_not_overwrite(self): + updates = {'__name__': 'not __main__'} + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['__name__'], '__main__') + + def test_main_not_shared(self): + ns1 = interpreters.run_string_unrestricted(self.id, 'spam = True') + ns2 = interpreters.run_string_unrestricted(self.id, 'eggs = False') + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertNotIn('spam', ns2) + + def test_return_execution_namespace(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py new file mode 100644 index 00000000000000..3e020139e0bdae --- /dev/null +++ b/Lib/test/test_interpreters.py @@ -0,0 +1,167 @@ +import ast +import os.path +import shutil +import tempfile +from textwrap import dedent +import threading +import unittest + +from test import support + +_interpreters = support.import_module('_interpreters') +interpreters = support.import_module('interpreters') + + +class TestBase(unittest.TestCase): + + def setUp(self): + self.dirname = None + + def tearDown(self): + if self.dirname is not None: + try: + shutil.rmtree(self.dirname) + except FileNotFoundError: + pass # already deleted + + for id in _interpreters.enumerate(): + if id == 0: # main + continue + try: + _interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + def _resolve_filename(self, name): + if self.dirname is None: + self.dirname = tempfile.mkdtemp() + return os.path.join(self.dirname, name) + + def _empty_file(self, name): + filename = self._resolve_filename(name) + support.create_empty_file(filename) + return filename + + def assert_file_contains(self, filename, expected): + with open(filename) as out: + content = out.read() + self.assertEqual(content, expected) + + +class ModuleTests(TestBase): + + def test_enumerate(self): + main_id = _interpreters.get_main() + got = interpreters.enumerate() + self.assertEqual(set(i.id for i in got), {main_id}) + + id1 = _interpreters.create() + id2 = _interpreters.create() + got = interpreters.enumerate() + self.assertEqual(set(i.id for i in got), {main_id, id1, id2}) + + def test_current(self): + main_id = _interpreters.get_main() + current = interpreters.current() + self.assertEqual(current.id, main_id) + + id = _interpreters.create() + script = dedent(""" + import interpreters + interp = interpreters.current() + """) + ns = _interpreters.run_string_unrestricted(id, script) + current = ns['interp'] + self.assertEqual(current.id, id) + + def test_main(self): + expected = _interpreters.get_main() + main = interpreters.main() + + self.assertEqual(main.id, expected) + + def test_create(self): + main_id = _interpreters.get_main() + interp = interpreters.create() + + self.assertIsInstance(interp, interpreters.Interpreter) + self.assertGreater(interp.id, main_id) + + +class InterpreterTests(TestBase): + + def test_repr(self): + interp = interpreters.Interpreter(10) + result = repr(interp) + + self.assertEqual(result, 'Interpreter(id=10)') + + def test_equality(self): + interp1 = interpreters.Interpreter(0) + interp2 = interpreters.Interpreter(10) + interp3 = interpreters.Interpreter(10) + different = (interp1 == interp2) + same = (interp2 == interp3) + identity = (interp1 == interp1) + + self.assertFalse(different) + self.assertTrue(same) + self.assertTrue(identity) + + def test_id(self): + interp = interpreters.Interpreter(10) + id = interp.id + + self.assertEqual(id, 10) + + def test_is_running(self): + interp = interpreters.create() + is_running = interp.is_running() + self.assertFalse(is_running) + + script = 'is_running = interp.is_running()' + ns = _interpreters.run_string_unrestricted(interp.id, script, { + 'interp': interp, + }) + is_running = ns['is_running'] + self.assertTrue(is_running) + + def test_destroy(self): + before = set(_interpreters.enumerate()) + interp = interpreters.create() + self.assertEqual(set(_interpreters.enumerate()), before | {interp.id}) + + interp.destroy() + after = set(_interpreters.enumerate()) + + self.assertEqual(before, after) + + def test_run_string(self): + filename = self._empty_file('spam') + data = 'success!' + script = dedent(f""" + with open('{filename}', 'w') as out: + out.write('{data}') + """) + interp = interpreters.create() + + interp.run(script) + self.assert_file_contains(filename, data) + + def test_run_unsupported(self): + script = 'raise Exception' + interp = interpreters.create() + + with self.assertRaises(NotImplementedError): + interp.run(None) + with self.assertRaises(NotImplementedError): + interp.run(10) + node = compile(script, '