Closed
Description
I want to capture stdout and stderr while running pytest programmatically; however, I get some strange behavior when I import Models before running pytest.main in combination with a user-defined ResultsCollector(). Here is a Colab reproducer that shows how it can run without issue prior to the import. Since I was only using Models in the context of accessing the name attribute from different models, it was easy to circumvent by hard-coding the names. This took a while to debug, and it was pretty important to the step I mentioned in #1479 (comment), so I wanted to open a separate issue.
Possibly related pytest issues include:
- Ensure stdin and stdout are in blocking mode before writing test summary pytest-dev/pytest#2251
- Occasional "invalid handle" error on Windows when using pytest within pytest pytest-dev/pytest#8257
However, I really didn't get much mileage out of them. It was only in trying to make a minimal reproducer that I eventually determined it was from the Models import.
For provenance, I'm including a fixed copy of the main code from the colab reproducer linked above.
from ax.modelbridge.factory import Models
import pytest
import time
class ResultsCollector:
def __init__(self):
self.reports = []
self.collected = 0
self.exitcode = 0
self.passed = 0
self.failed = 0
self.xfailed = 0
self.skipped = 0
self.total_duration = 0
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(self, item, call):
outcome = yield
report = outcome.get_result()
if report.when == "call":
self.reports.append(report)
def pytest_collection_modifyitems(self, items):
self.collected = len(items)
def pytest_terminal_summary(self, terminalreporter, exitstatus):
print(exitstatus, dir(exitstatus))
self.exitcode = exitstatus
self.passed = terminalreporter.stats.get("passed", [])
self.failed = terminalreporter.stats.get("failed", [])
self.xfailed = terminalreporter.stats.get("xfailed", [])
self.skipped = terminalreporter.stats.get("skipped", [])
self.num_passed = len(self.passed)
self.num_failed = len(self.failed)
self.num_xfailed = len(self.xfailed)
self.num_skipped = len(self.skipped)
self.total_duration = time.time() - terminalreporter._sessionstarttime
collector = ResultsCollector()
retcode = pytest.main(["-v", "tests/"], plugins=[collector])tests/test_ax.py
def test_simple():
from ax.service.ax_client import AxClient
from ax.utils.measurement.synthetic_functions import branin
ax_client = AxClient()
ax_client.create_experiment(
parameters=[
{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "x2", "type": "range", "bounds": [0.0, 10.0]},
],
objective_name="branin",
minimize=True,
)
for _ in range(15):
parameters, trial_index = ax_client.get_next_trial()
results = branin(parameters["x1"], parameters["x2"])
ax_client.complete_trial(trial_index=trial_index, raw_data=results)
best_parameters, metrics = ax_client.get_best_parameters()This is what some of the error strings inside of capstderr look like:
Details
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib/python3.10/logging/__init__.py", line 1103, in emit
stream.write(msg + self.terminator)
ValueError: I/O operation on closed file.
Call stack:
File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.10/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.10/dist-packages/traitlets/config/application.py", line 992, in launch_instance
app.start()
File "/usr/local/lib/python3.10/dist-packages/ipykernel/kernelapp.py", line 619, in start
self.io_loop.start()
File "/usr/local/lib/python3.10/dist-packages/tornado/platform/asyncio.py", line 195, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.10/asyncio/base_events.py", line 603, in run_forever
self._run_once()
File "/usr/lib/python3.10/asyncio/base_events.py", line 1909, in _run_once
handle._run()
File "/usr/lib/python3.10/asyncio/events.py", line 80, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.10/dist-packages/tornado/ioloop.py", line 685, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "/usr/local/lib/python3.10/dist-packages/tornado/ioloop.py", line 738, in _run_callback
ret = callback()
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 825, in inner
self.ctx_run(self.run)
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 786, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py", line 377, in dispatch_queue
yield self.process_one()
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 250, in wrapper
runner = Runner(ctx_run, result, future, yielded)
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 748, in __init__
self.ctx_run(self.run)
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 786, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py", line 361, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 234, in wrapper
yielded = ctx_run(next, result)
File "/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py", line 261, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 234, in wrapper
yielded = ctx_run(next, result)
File "/usr/local/lib/python3.10/dist-packages/ipykernel/kernelbase.py", line 539, in execute_request
self.do_execute(
File "/usr/local/lib/python3.10/dist-packages/tornado/gen.py", line 234, in wrapper
yielded = ctx_run(next, result)
File "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py", line 302, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.10/dist-packages/ipykernel/zmqshell.py", line 539, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py", line 2975, in run_cell
result = self._run_cell(
File "/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py", line 3030, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.10/dist-packages/IPython/core/async_helpers.py", line 78, in _pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py", line 3257, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
File "/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py", line 3473, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py", line 3553, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-df5f50d01ab8>", line 4, in <cell line: 4>
retcode = pytest.main(["-v", "tests/"], plugins=[collector])
File "/usr/local/lib/python3.10/dist-packages/_pytest/config/__init__.py", line 166, in main
ret: Union[ExitCode, int] = config.hook.pytest_cmdline_main(
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 433, in __call__
return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 112, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 80, in _multicall
res = hook_impl.function(*args)
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 317, in pytest_cmdline_main
return wrap_session(config, _main)
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 270, in wrap_session
session.exitstatus = doit(config, session) or 0
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 324, in _main
config.hook.pytest_runtestloop(session=session)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 433, in __call__
return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 112, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 80, in _multicall
res = hook_impl.function(*args)
File "/usr/local/lib/python3.10/dist-packages/_pytest/main.py", line 349, in pytest_runtestloop
item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 433, in __call__
return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 112, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 80, in _multicall
res = hook_impl.function(*args)
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 114, in pytest_runtest_protocol
runtestprotocol(item, nextitem=nextitem)
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 133, in runtestprotocol
reports.append(call_and_report(item, "call", log))
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 222, in call_and_report
call = call_runtest_hook(item, when, **kwds)
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 261, in call_runtest_hook
return CallInfo.from_call(
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 341, in from_call
result: Optional[TResult] = func()
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 262, in <lambda>
lambda: ihook(item=item, **kwds), when=when, reraise=reraise
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 433, in __call__
return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 112, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 80, in _multicall
res = hook_impl.function(*args)
File "/usr/local/lib/python3.10/dist-packages/_pytest/runner.py", line 169, in pytest_runtest_call
item.runtest()
File "/usr/local/lib/python3.10/dist-packages/_pytest/python.py", line 1788, in runtest
self.ihook.pytest_pyfunc_call(pyfuncitem=self)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 433, in __call__
return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 112, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 80, in _multicall
res = hook_impl.function(*args)
File "/usr/local/lib/python3.10/dist-packages/_pytest/python.py", line 194, in pytest_pyfunc_call
result = testfunction(**testargs)
File "/content/tests/test_ax.py", line 7, in test_simple
ax_client.create_experiment(
File "/usr/local/lib/python3.10/dist-packages/ax/service/ax_client.py", line 373, in create_experiment
self._set_generation_strategy(
File "/usr/local/lib/python3.10/dist-packages/ax/service/ax_client.py", line 1684, in _set_generation_strategy
self._generation_strategy = choose_generation_strategy(
File "/usr/local/lib/python3.10/dist-packages/ax/modelbridge/dispatch_utils.py", line 461, in choose_generation_strategy
logger.info(
Message: 'Calculating the number of remaining initialization trials based on num_initialization_trials=None max_initialization_trials=None num_tunable_parameters=2 num_trials=None use_batch_trials=False'
Arguments: ()On Windows, the error message is OSError: [WinError 6] The handle is invalid rather than ValueError: I/O operation on closed file.