Skip to content

Commit 7e35986

Browse files
authored
Merge pull request #2767 from jku/bootstrap-root-metadata
Cache all root metadata versions
2 parents 72bb243 + 38e4eab commit 7e35986

7 files changed

+233
-89
lines changed

examples/client/client

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ import sys
1111
import traceback
1212
from hashlib import sha256
1313
from pathlib import Path
14-
from urllib import request
14+
15+
import urllib3
1516

1617
from tuf.api.exceptions import DownloadError, RepositoryError
1718
from tuf.ngclient import Updater
@@ -29,19 +30,30 @@ def build_metadata_dir(base_url: str) -> str:
2930

3031
def init_tofu(base_url: str) -> bool:
3132
"""Initialize local trusted metadata (Trust-On-First-Use) and create a
32-
directory for downloads"""
33+
directory for downloads
34+
35+
NOTE: This is unsafe and for demonstration only: the bootstrap root
36+
should be deployed alongside your updater application
37+
"""
38+
3339
metadata_dir = build_metadata_dir(base_url)
3440

3541
if not os.path.isdir(metadata_dir):
3642
os.makedirs(metadata_dir)
3743

38-
root_url = f"{base_url}/metadata/1.root.json"
39-
try:
40-
request.urlretrieve(root_url, f"{metadata_dir}/root.json")
41-
except OSError:
42-
print(f"Failed to download initial root from {root_url}")
44+
response = urllib3.request("GET", f"{base_url}/metadata/1.root.json")
45+
if response.status != 200:
46+
print(f"Failed to download initial root {base_url}/metadata/1.root.json")
4347
return False
4448

49+
Updater(
50+
metadata_dir=metadata_dir,
51+
metadata_base_url=f"{base_url}/metadata/",
52+
target_base_url=f"{base_url}/targets/",
53+
target_dir=DOWNLOAD_DIR,
54+
bootstrap=response.data,
55+
)
56+
4557
print(f"Trust-on-First-Use: Initialized new root in {metadata_dir}")
4658
return True
4759

@@ -73,6 +85,9 @@ def download(base_url: str, target: str) -> bool:
7385
os.mkdir(DOWNLOAD_DIR)
7486

7587
try:
88+
# NOTE: initial root should be provided with ``bootstrap`` argument:
89+
# This examples uses unsafe Trust-On-First-Use initialization so it is
90+
# not possible here.
7691
updater = Updater(
7792
metadata_dir=metadata_dir,
7893
metadata_base_url=f"{base_url}/metadata/",
@@ -104,7 +119,7 @@ def download(base_url: str, target: str) -> bool:
104119
return True
105120

106121

107-
def main() -> None:
122+
def main() -> str | None:
108123
"""Main TUF Client Example function"""
109124

110125
client_args = argparse.ArgumentParser(description="TUF Client Example")
@@ -169,6 +184,8 @@ def main() -> None:
169184
else:
170185
client_args.print_help()
171186

187+
return None
188+
172189

173190
if __name__ == "__main__":
174191
sys.exit(main())

tests/test_updater_consistent_snapshot.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def teardown_subtest(self) -> None:
6262
if self.dump_dir is not None:
6363
self.sim.write()
6464

65-
utils.cleanup_dir(self.metadata_dir)
65+
utils.cleanup_metadata_dir(self.metadata_dir)
6666

6767
def _init_repo(
6868
self, consistent_snapshot: bool, prefix_targets: bool = True

tests/test_updater_delegation_graphs.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def setup_subtest(self) -> None:
9292
self.sim.write()
9393

9494
def teardown_subtest(self) -> None:
95-
utils.cleanup_dir(self.metadata_dir)
95+
utils.cleanup_metadata_dir(self.metadata_dir)
9696

9797
def _init_repo(self, test_case: DelegationsTestCase) -> None:
9898
"""Create a new RepositorySimulator instance and

tests/test_updater_ng.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import sys
1212
import tempfile
1313
import unittest
14+
from collections.abc import Iterable
1415
from typing import TYPE_CHECKING, Callable, ClassVar
1516
from unittest.mock import MagicMock, patch
1617

tests/test_updater_top_level_update.py

Lines changed: 107 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,6 @@ def setUp(self) -> None:
6262

6363
self.sim = RepositorySimulator()
6464

65-
# boostrap client with initial root metadata
66-
with open(os.path.join(self.metadata_dir, "root.json"), "bw") as f:
67-
f.write(self.sim.signed_roots[0])
68-
6965
if self.dump_dir is not None:
7066
# create test specific dump directory
7167
name = self.id().split(".")[-1]
@@ -75,22 +71,13 @@ def setUp(self) -> None:
7571
def tearDown(self) -> None:
7672
self.temp_dir.cleanup()
7773

78-
def _run_refresh(self) -> Updater:
74+
def _run_refresh(self, skip_bootstrap: bool = False) -> Updater:
7975
"""Create a new Updater instance and refresh"""
80-
if self.dump_dir is not None:
81-
self.sim.write()
82-
83-
updater = Updater(
84-
self.metadata_dir,
85-
"https://example.com/metadata/",
86-
self.targets_dir,
87-
"https://example.com/targets/",
88-
self.sim,
89-
)
76+
updater = self._init_updater(skip_bootstrap)
9077
updater.refresh()
9178
return updater
9279

93-
def _init_updater(self) -> Updater:
80+
def _init_updater(self, skip_bootstrap: bool = False) -> Updater:
9481
"""Create a new Updater instance"""
9582
if self.dump_dir is not None:
9683
self.sim.write()
@@ -101,6 +88,7 @@ def _init_updater(self) -> Updater:
10188
self.targets_dir,
10289
"https://example.com/targets/",
10390
self.sim,
91+
bootstrap=None if skip_bootstrap else self.sim.signed_roots[0],
10492
)
10593

10694
def _assert_files_exist(self, roles: Iterable[str]) -> None:
@@ -126,9 +114,6 @@ def _assert_version_equals(self, role: str, expected_version: int) -> None:
126114
self.assertEqual(md.signed.version, expected_version)
127115

128116
def test_first_time_refresh(self) -> None:
129-
# Metadata dir contains only the mandatory initial root.json
130-
self._assert_files_exist([Root.type])
131-
132117
# Add one more root version to repository so that
133118
# refresh() updates from local trusted root (v1) to
134119
# remote root (v2)
@@ -142,10 +127,11 @@ def test_first_time_refresh(self) -> None:
142127
version = 2 if role == Root.type else None
143128
self._assert_content_equals(role, version)
144129

145-
def test_trusted_root_missing(self) -> None:
146-
os.remove(os.path.join(self.metadata_dir, "root.json"))
130+
def test_cached_root_missing_without_bootstrap(self) -> None:
131+
# Run update without a bootstrap, with empty cache: this fails since there is no
132+
# trusted root
147133
with self.assertRaises(OSError):
148-
self._run_refresh()
134+
self._run_refresh(skip_bootstrap=True)
149135

150136
# Metadata dir is empty
151137
self.assertFalse(os.listdir(self.metadata_dir))
@@ -178,15 +164,15 @@ def test_trusted_root_expired(self) -> None:
178164
self._assert_files_exist(TOP_LEVEL_ROLE_NAMES)
179165
self._assert_content_equals(Root.type, 3)
180166

181-
def test_trusted_root_unsigned(self) -> None:
182-
# Local trusted root is not signed
167+
def test_trusted_root_unsigned_without_bootstrap(self) -> None:
168+
# Cached root is not signed, bootstrap root is not used
183169
root_path = os.path.join(self.metadata_dir, "root.json")
184-
md_root = Metadata.from_file(root_path)
170+
md_root = Metadata.from_bytes(self.sim.signed_roots[0])
185171
md_root.signatures.clear()
186172
md_root.to_file(root_path)
187173

188174
with self.assertRaises(UnsignedMetadataError):
189-
self._run_refresh()
175+
self._run_refresh(skip_bootstrap=True)
190176

191177
# The update failed, no changes in metadata
192178
self._assert_files_exist([Root.type])
@@ -204,10 +190,7 @@ def test_max_root_rotations(self) -> None:
204190
self.sim.root.version += 1
205191
self.sim.publish_root()
206192

207-
md_root = Metadata.from_file(
208-
os.path.join(self.metadata_dir, "root.json")
209-
)
210-
initial_root_version = md_root.signed.version
193+
initial_root_version = 1
211194

212195
updater.refresh()
213196

@@ -712,26 +695,20 @@ def test_load_metadata_from_cache(self, wrapped_open: MagicMock) -> None:
712695
updater = self._run_refresh()
713696
updater.get_targetinfo("non_existent_target")
714697

715-
# Clean up calls to open during refresh()
698+
# Clear statistics for open() calls and metadata requests
716699
wrapped_open.reset_mock()
717-
# Clean up fetch tracker metadata
718700
self.sim.fetch_tracker.metadata.clear()
719701

720702
# Create a new updater and perform a second update while
721703
# the metadata is already stored in cache (metadata dir)
722-
updater = Updater(
723-
self.metadata_dir,
724-
"https://example.com/metadata/",
725-
self.targets_dir,
726-
"https://example.com/targets/",
727-
self.sim,
728-
)
704+
updater = self._init_updater()
729705
updater.get_targetinfo("non_existent_target")
730706

731707
# Test that metadata is loaded from cache and not downloaded
708+
root_dir = os.path.join(self.metadata_dir, "root_history")
732709
wrapped_open.assert_has_calls(
733710
[
734-
call(os.path.join(self.metadata_dir, "root.json"), "rb"),
711+
call(os.path.join(root_dir, "2.root.json"), "rb"),
735712
call(os.path.join(self.metadata_dir, "timestamp.json"), "rb"),
736713
call(os.path.join(self.metadata_dir, "snapshot.json"), "rb"),
737714
call(os.path.join(self.metadata_dir, "targets.json"), "rb"),
@@ -742,6 +719,96 @@ def test_load_metadata_from_cache(self, wrapped_open: MagicMock) -> None:
742719
expected_calls = [("root", 2), ("timestamp", None)]
743720
self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
744721

722+
@patch.object(builtins, "open", wraps=builtins.open)
723+
def test_intermediate_root_cache(self, wrapped_open: MagicMock) -> None:
724+
"""Test that refresh uses the intermediate roots from cache"""
725+
# Add root versions 2, 3
726+
self.sim.root.version += 1
727+
self.sim.publish_root()
728+
self.sim.root.version += 1
729+
self.sim.publish_root()
730+
731+
# Make a successful update of valid metadata which stores it in cache
732+
self._run_refresh()
733+
734+
# assert that cache lookups happened but data was downloaded from remote
735+
root_dir = os.path.join(self.metadata_dir, "root_history")
736+
wrapped_open.assert_has_calls(
737+
[
738+
call(os.path.join(root_dir, "2.root.json"), "rb"),
739+
call(os.path.join(root_dir, "3.root.json"), "rb"),
740+
call(os.path.join(root_dir, "4.root.json"), "rb"),
741+
call(os.path.join(self.metadata_dir, "timestamp.json"), "rb"),
742+
call(os.path.join(self.metadata_dir, "snapshot.json"), "rb"),
743+
call(os.path.join(self.metadata_dir, "targets.json"), "rb"),
744+
]
745+
)
746+
expected_calls = [
747+
("root", 2),
748+
("root", 3),
749+
("root", 4),
750+
("timestamp", None),
751+
("snapshot", 1),
752+
("targets", 1),
753+
]
754+
self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
755+
756+
# Clear statistics for open() calls and metadata requests
757+
wrapped_open.reset_mock()
758+
self.sim.fetch_tracker.metadata.clear()
759+
760+
# Run update again, assert that metadata from cache was used (including intermediate roots)
761+
self._run_refresh()
762+
wrapped_open.assert_has_calls(
763+
[
764+
call(os.path.join(root_dir, "2.root.json"), "rb"),
765+
call(os.path.join(root_dir, "3.root.json"), "rb"),
766+
call(os.path.join(root_dir, "4.root.json"), "rb"),
767+
call(os.path.join(self.metadata_dir, "timestamp.json"), "rb"),
768+
call(os.path.join(self.metadata_dir, "snapshot.json"), "rb"),
769+
call(os.path.join(self.metadata_dir, "targets.json"), "rb"),
770+
]
771+
)
772+
expected_calls = [("root", 4), ("timestamp", None)]
773+
self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
774+
775+
def test_intermediate_root_cache_poisoning(self) -> None:
776+
"""Test that refresh works as expected when intermediate roots in cache are poisoned"""
777+
# Add root versions 2, 3
778+
self.sim.root.version += 1
779+
self.sim.publish_root()
780+
self.sim.root.version += 1
781+
self.sim.publish_root()
782+
783+
# Make a successful update of valid metadata which stores it in cache
784+
self._run_refresh()
785+
786+
# Modify cached intermediate root v2 so that it's no longer signed correctly
787+
root_path = os.path.join(
788+
self.metadata_dir, "root_history", "2.root.json"
789+
)
790+
md = Metadata.from_file(root_path)
791+
md.signatures.clear()
792+
md.to_file(root_path)
793+
794+
# Clear statistics for metadata requests
795+
self.sim.fetch_tracker.metadata.clear()
796+
797+
# Update again, assert that intermediate root v2 was downloaded again
798+
self._run_refresh()
799+
800+
expected_calls = [("root", 2), ("root", 4), ("timestamp", None)]
801+
self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
802+
803+
# Clear statistics for metadata requests
804+
self.sim.fetch_tracker.metadata.clear()
805+
806+
# Update again, this time assert that intermediate root v2 was used from cache
807+
self._run_refresh()
808+
809+
expected_calls = [("root", 4), ("timestamp", None)]
810+
self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
811+
745812
def test_expired_metadata(self) -> None:
746813
"""Verifies that expired local timestamp/snapshot can be used for
747814
updating from remote.

tests/utils.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -155,12 +155,16 @@ def configure_test_logging(argv: list[str]) -> None:
155155
logging.basicConfig(level=loglevel)
156156

157157

158-
def cleanup_dir(path: str) -> None:
159-
"""Delete all files inside a directory"""
160-
for filepath in [
161-
os.path.join(path, filename) for filename in os.listdir(path)
162-
]:
163-
os.remove(filepath)
158+
def cleanup_metadata_dir(path: str) -> None:
159+
"""Delete the local metadata dir"""
160+
with os.scandir(path) as it:
161+
for entry in it:
162+
if entry.name == "root_history":
163+
cleanup_metadata_dir(entry.path)
164+
elif entry.name.endswith(".json"):
165+
os.remove(entry.path)
166+
else:
167+
raise ValueError(f"Unexpected local metadata file {entry.path}")
164168

165169

166170
class TestServerProcess:

0 commit comments

Comments
 (0)