From 761f178651b032de505ca4ccc8d012c38beeab25 Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Mon, 16 Jun 2025 22:39:22 +0800 Subject: [PATCH 01/12] Fix: add immutable=1 flag for read-only SQLite access to avoid WAL/SHM errors on readonly DB --- Lib/dbm/sqlite3.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index b296a1bcd1bbfa..7ee35d04399420 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -59,19 +59,24 @@ def __init__(self, path, /, *, flag, mode): # We use the URI format when opening the database. uri = _normalize_uri(path) - uri = f"{uri}?mode={flag}" + if flag == "ro": + # Add immutable=1 to allow read-only SQLite access even if wal/shm missing + uri = f"{uri}?mode={flag}&immutable=1" + else: + uri = f"{uri}?mode={flag}" try: self._cx = sqlite3.connect(uri, autocommit=True, uri=True) except sqlite3.Error as exc: raise error(str(exc)) - + self._readonly = (flag == "ro") # This is an optimization only; it's ok if it fails. - with suppress(sqlite3.OperationalError): - self._cx.execute("PRAGMA journal_mode = wal") + if not self._readonly: + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = OFF") - if flag == "rwc": - self._execute(BUILD_TABLE) + if flag == "rwc": + self._execute(BUILD_TABLE) def _execute(self, *args, **kwargs): if not self._cx: From 3e3a3ba6c6f1a8ea747e08b3d1ad6b098366d4b6 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 15:00:14 +0000 Subject: [PATCH 02/12] gh-135386: Fix dbm.sqlite3 readonly open error by using immutable=1 --- Lib/dbm/sqlite3.py | 1 + Lib/test/test_dbm_sqlite3.py | 17 +++++++++++++++++ ...25-06-16-15-00-13.gh-issue-135386.lNrxLc.rst | 1 + 3 files changed, 19 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-06-16-15-00-13.gh-issue-135386.lNrxLc.rst diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 7ee35d04399420..9ca9f8f999e207 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -69,6 +69,7 @@ def __init__(self, path, /, *, flag, mode): self._cx = sqlite3.connect(uri, autocommit=True, uri=True) except sqlite3.Error as exc: raise error(str(exc)) + self._readonly = (flag == "ro") # This is an optimization only; it's ok if it fails. if not self._readonly: diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 9216da8a63f957..1553e28cfee197 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -1,3 +1,4 @@ +import os import sys import unittest from contextlib import closing @@ -89,6 +90,22 @@ def test_readonly_keys(self): def test_readonly_iter(self): self.assertEqual([k for k in self.db], [b"key1", b"key2"]) + def test_readonly_open_without_wal_shm(self): + wal_path = self.filename + "-wal" + shm_path = self.filename + "-shm" + + for suffix in wal_path, shm_path: + try: + os.remove(suffix) + except FileNotFoundError: + pass + + os.chmod(self.filename, 0o444) + + with dbm_sqlite3.open(self.filename, "r") as db: + self.assertEqual(db[b"key1"], b"value1") + self.assertEqual(db[b"key2"], b"value2") + class ReadWrite(_SQLiteDbmTests): diff --git a/Misc/NEWS.d/next/Library/2025-06-16-15-00-13.gh-issue-135386.lNrxLc.rst b/Misc/NEWS.d/next/Library/2025-06-16-15-00-13.gh-issue-135386.lNrxLc.rst new file mode 100644 index 00000000000000..d3f81cb9201aaf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-16-15-00-13.gh-issue-135386.lNrxLc.rst @@ -0,0 +1 @@ +Fix :exc:`sqlite3.OperationalError` error when using :func:`dbm.open` with a read-only file object. From 7f988ddabc6df30213af62294b26fbbe57c9c0a7 Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Tue, 17 Jun 2025 21:56:37 +0800 Subject: [PATCH 03/12] Add test: readonly sqlite db without wal/shm (skip on Windows) --- Lib/test/test_dbm_sqlite3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 1553e28cfee197..467ada060cfe77 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -90,6 +90,7 @@ def test_readonly_keys(self): def test_readonly_iter(self): self.assertEqual([k for k in self.db], [b"key1", b"key2"]) + @unittest.skipIf(sys.platform.startswith("win"), "incompatible with Windows file locking") def test_readonly_open_without_wal_shm(self): wal_path = self.filename + "-wal" shm_path = self.filename + "-shm" From 775683e378d21dc8e5d53a18b68693029c4ba9eb Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Mon, 23 Jun 2025 21:38:57 +0800 Subject: [PATCH 04/12] optimize(code): Update Lib/dbm/sqlite3.py Co-authored-by: Tomas R. --- Lib/dbm/sqlite3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 9ca9f8f999e207..7625b73e8931a5 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -72,7 +72,7 @@ def __init__(self, path, /, *, flag, mode): self._readonly = (flag == "ro") # This is an optimization only; it's ok if it fails. - if not self._readonly: + if flag != "ro": with suppress(sqlite3.OperationalError): self._cx.execute("PRAGMA journal_mode = OFF") From 66899ff366c5fcb6e0f7730c1670e3d42b451429 Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Mon, 23 Jun 2025 21:39:12 +0800 Subject: [PATCH 05/12] optimize(code): update Lib/dbm/sqlite3.py Co-authored-by: Tomas R. --- Lib/dbm/sqlite3.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 7625b73e8931a5..3d87eeebcffbf9 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -59,11 +59,10 @@ def __init__(self, path, /, *, flag, mode): # We use the URI format when opening the database. uri = _normalize_uri(path) + uri = f"{uri}?mode={flag}" if flag == "ro": # Add immutable=1 to allow read-only SQLite access even if wal/shm missing - uri = f"{uri}?mode={flag}&immutable=1" - else: - uri = f"{uri}?mode={flag}" + uri += "&immutable=1" try: self._cx = sqlite3.connect(uri, autocommit=True, uri=True) From 4561075b3acab2506d34b7c769336a6fb513a5d7 Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Tue, 24 Jun 2025 01:00:00 +0800 Subject: [PATCH 06/12] Test: make read-only dbm.sqlite3 test compatible with Windows --- Lib/test/test_dbm_sqlite3.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 467ada060cfe77..a9aa270e9a35e4 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -1,3 +1,4 @@ +import stat import os import sys import unittest @@ -90,22 +91,28 @@ def test_readonly_keys(self): def test_readonly_iter(self): self.assertEqual([k for k in self.db], [b"key1", b"key2"]) - @unittest.skipIf(sys.platform.startswith("win"), "incompatible with Windows file locking") def test_readonly_open_without_wal_shm(self): wal_path = self.filename + "-wal" shm_path = self.filename + "-shm" for suffix in wal_path, shm_path: - try: - os.remove(suffix) - except FileNotFoundError: - pass + os_helper.unlink(suffix) - os.chmod(self.filename, 0o444) + try: + self.db.close() + except Exception: + pass - with dbm_sqlite3.open(self.filename, "r") as db: + os.chmod(self.filename, stat.S_IREAD) + + db = dbm_sqlite3.open(self.filename, "r") + try: self.assertEqual(db[b"key1"], b"value1") self.assertEqual(db[b"key2"], b"value2") + finally: + db.close() + + os.chmod(self.filename, stat.S_IWRITE) class ReadWrite(_SQLiteDbmTests): From 7c40be39022e08aa7305a6483b4a1d6bf0a9bfad Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Tue, 24 Jun 2025 23:08:12 +0800 Subject: [PATCH 07/12] Refactor: remove unused _readonly flag, simplify readonly WAL/SHM test --- Lib/dbm/sqlite3.py | 1 - Lib/test/test_dbm_sqlite3.py | 35 ++++++++++++++++++----------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 3d87eeebcffbf9..42c04878d2f85c 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -69,7 +69,6 @@ def __init__(self, path, /, *, flag, mode): except sqlite3.Error as exc: raise error(str(exc)) - self._readonly = (flag == "ro") # This is an optimization only; it's ok if it fails. if flag != "ro": with suppress(sqlite3.OperationalError): diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index a9aa270e9a35e4..60e1bc50bf6c24 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -91,28 +91,29 @@ def test_readonly_keys(self): def test_readonly_iter(self): self.assertEqual([k for k in self.db], [b"key1", b"key2"]) - def test_readonly_open_without_wal_shm(self): - wal_path = self.filename + "-wal" - shm_path = self.filename + "-shm" +class Immutable(unittest.TestCase): + def setUp(self): + self.filename = os_helper.TESTFN - for suffix in wal_path, shm_path: - os_helper.unlink(suffix) + db = dbm_sqlite3.open(self.filename, "c") + db[b"key"] = b"value" + db.close() - try: - self.db.close() - except Exception: - pass + self.db = dbm_sqlite3.open(self.filename, "r") - os.chmod(self.filename, stat.S_IREAD) + def tearDown(self): + self.db.close() + for suffix in "", "-wal", "-shm": + os_helper.unlink(self.filename + suffix) - db = dbm_sqlite3.open(self.filename, "r") - try: - self.assertEqual(db[b"key1"], b"value1") - self.assertEqual(db[b"key2"], b"value2") - finally: - db.close() + def test_readonly_open_without_wal_shm(self): + wal_path = self.filename + "-wal" + shm_path = self.filename + "-shm" - os.chmod(self.filename, stat.S_IWRITE) + self.assertFalse(os.path.exists(wal_path)) + self.assertFalse(os.path.exists(shm_path)) + + self.assertEqual(self.db[b"key"], b"value") class ReadWrite(_SQLiteDbmTests): From 759f2186f1634fbec4aba3afe620d45e67c465a4 Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Tue, 24 Jun 2025 23:32:20 +0800 Subject: [PATCH 08/12] Remove unused import --- Lib/test/test_dbm_sqlite3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 60e1bc50bf6c24..686ce9f1e015b5 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -1,4 +1,3 @@ -import stat import os import sys import unittest @@ -112,7 +111,7 @@ def test_readonly_open_without_wal_shm(self): self.assertFalse(os.path.exists(wal_path)) self.assertFalse(os.path.exists(shm_path)) - + self.assertEqual(self.db[b"key"], b"value") From 73d9e24b81a643ccc671b6fbc8de6da33b9347c5 Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Wed, 9 Jul 2025 11:53:32 +0800 Subject: [PATCH 09/12] Test: add read-only directory and file tests for dbm.sqlite3 --- Lib/test/test_dbm_sqlite3.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 686ce9f1e015b5..26bebfd42aedb9 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -1,4 +1,5 @@ import os +import stat import sys import unittest from contextlib import closing @@ -115,6 +116,40 @@ def test_readonly_open_without_wal_shm(self): self.assertEqual(self.db[b"key"], b"value") +class ReadOnlyFilesystem(unittest.TestCase): + + def setUp(self): + self.test_dir = os_helper.TESTFN + os.mkdir(self.test_dir) + self.db_path = os.path.join(self.test_dir, "test.db") + + db = dbm_sqlite3.open(self.db_path, "c") + db[b"key"] = b"value" + db.close() + + def tearDown(self): + os.chmod(self.db_path, stat.S_IWRITE) + os.chmod(self.test_dir, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) + os_helper.rmtree(self.test_dir) + + def test_open_readonly_dir_success_ro(self): + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_open_readonly_file_success(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_open_readonly_file_fail_rw(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "w") as db: + with self.assertRaises(OSError): + db[b"newkey"] = b"newvalue" + + + class ReadWrite(_SQLiteDbmTests): def setUp(self): From c4c8065f79948949f1e4d1b69584c666fb857719 Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Wed, 9 Jul 2025 13:11:41 +0800 Subject: [PATCH 10/12] Test: improve read-only filesystem tests for dbm.sqlite3 per review feedback --- Lib/test/test_dbm_sqlite3.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 26bebfd42aedb9..4826cc2b21ad76 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -133,6 +133,9 @@ def tearDown(self): os_helper.rmtree(self.test_dir) def test_open_readonly_dir_success_ro(self): + files = os.listdir(self.test_dir) + self.assertEqual(sorted(files), ["test.db"]) + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) with dbm_sqlite3.open(self.db_path, "r") as db: self.assertEqual(db[b"key"], b"value") @@ -148,6 +151,28 @@ def test_open_readonly_file_fail_rw(self): with self.assertRaises(OSError): db[b"newkey"] = b"newvalue" + def test_open_readonly_dir_fail_rw_missing_wal_shm(self): + for suffix in ("-wal", "-shm"): + os_helper.unlink(self.db_path + suffix) + + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + + with self.assertRaises(OSError): + db = dbm_sqlite3.open(self.db_path, "w") + db[b"newkey"] = b"newvalue" + db.close() + + def test_open_readonly_dir_fail_rw_with_writable_db(self): + os.chmod(self.db_path, stat.S_IREAD | stat.S_IWRITE) + for suffix in ("-wal", "-shm"): + os_helper.unlink(self.db_path + suffix) + + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + + with self.assertRaises(OSError): + db = dbm_sqlite3.open(self.db_path, "w") + db[b"newkey"] = b"newvalue" + db.close() class ReadWrite(_SQLiteDbmTests): From b586fc3b8e10e95642378ee108f6710284e13b9c Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Wed, 9 Jul 2025 13:33:33 +0800 Subject: [PATCH 11/12] Fix test and avoid unclosed DB connection. --- Lib/test/test_dbm_sqlite3.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 4826cc2b21ad76..efbd885f34defc 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -158,9 +158,9 @@ def test_open_readonly_dir_fail_rw_missing_wal_shm(self): os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) with self.assertRaises(OSError): - db = dbm_sqlite3.open(self.db_path, "w") - db[b"newkey"] = b"newvalue" - db.close() + with dbm_sqlite3.open(self.db_path, "w") as db: + db[b"newkey"] = b"newvalue" + def test_open_readonly_dir_fail_rw_with_writable_db(self): os.chmod(self.db_path, stat.S_IREAD | stat.S_IWRITE) @@ -170,9 +170,8 @@ def test_open_readonly_dir_fail_rw_with_writable_db(self): os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) with self.assertRaises(OSError): - db = dbm_sqlite3.open(self.db_path, "w") - db[b"newkey"] = b"newvalue" - db.close() + with dbm_sqlite3.open(self.db_path, "w") as db: + db[b"newkey"] = b"newvalue" class ReadWrite(_SQLiteDbmTests): From bd405f2407fd01c950edba8739c1e5c883b39c3e Mon Sep 17 00:00:00 2001 From: General_K1ng Date: Wed, 9 Jul 2025 15:36:14 +0800 Subject: [PATCH 12/12] Skip platform-dependent write failure tests in readonly directory --- Lib/test/test_dbm_sqlite3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index efbd885f34defc..321ad41c7f36bc 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -150,7 +150,7 @@ def test_open_readonly_file_fail_rw(self): with dbm_sqlite3.open(self.db_path, "w") as db: with self.assertRaises(OSError): db[b"newkey"] = b"newvalue" - + @unittest.skipUnless(sys.platform == "darwin", "SQLite fallback behavior differs on non-macOS") def test_open_readonly_dir_fail_rw_missing_wal_shm(self): for suffix in ("-wal", "-shm"): os_helper.unlink(self.db_path + suffix) @@ -161,7 +161,7 @@ def test_open_readonly_dir_fail_rw_missing_wal_shm(self): with dbm_sqlite3.open(self.db_path, "w") as db: db[b"newkey"] = b"newvalue" - + @unittest.skipUnless(sys.platform == "darwin", "SQLite fallback behavior differs on non-macOS") def test_open_readonly_dir_fail_rw_with_writable_db(self): os.chmod(self.db_path, stat.S_IREAD | stat.S_IWRITE) for suffix in ("-wal", "-shm"):