From 288a23a6267188b2657b38a218be5a2ab67e2e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Thu, 10 Nov 2022 13:34:16 +0000 Subject: [PATCH] tests: garbage collect sqlite connections on Windows + Python 3.11 There seems to be a regression in Python 3.11, where the sqlite connections are not deallocated, due to some internal changes in Python 3.11, where they are now using LRU cache. They are not deallocated until `gc.collect()` is not called. See https://github.com/python/cpython/issues/97641. This affects only Windows, because when we try to remove the tempdir for the exp run, the sqlite connection is open which prevents us from deleting that folder. Although this may happen in real scenario in `exp run`, I am only fixing the tests by mocking `dvc.close()` and extending it by calling `gc.collect()` after it. We could also mock `State.close()` but didnot want to mock something that is not in dvc itself. The `diskcache` uses threadlocal for connections, so they are expected to be garbage collected, and therefore does not provide a good way to close the connections. The only API it offers is `self.close()` and that only closes main thread's connection. If we had access to connection, an easier way would have been to explicitly call `conn.close()`. But we don''t have such option at the moment. Related: https://github.com/iterative/dvc/pull/8404#issuecomment-1310024661 GHA Failure: https://github.com/iterative/dvc/actions/runs/3437324559/jobs/5731929385#step:5:57 --- tests/conftest.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 8846ece544..72835f6f67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import gc import json import os import sys @@ -259,3 +260,19 @@ def mock_hydra_conf(mocker): # NOTE: using sentinel here so that any imports from `hydra.conf` # return a mock. sys.modules["hydra.conf"] = mocker.sentinel + + +@pytest.fixture(autouse=True) +def gc_collect_on_dvc_close_on_win_311(mocker): + if sys.version_info < (3, 11) and os.name != "nt": + return + + from dvc.repo import Repo + + close = Repo.close + + def wrapped(repo): + close(repo) + gc.collect() + + mocker.patch("dvc.repo.Repo.close", wrapped)