Skip to content

Implemented fetch_tracker to RepositorySimulator #1705

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 8, 2021
Merged

Implemented fetch_tracker to RepositorySimulator #1705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion tests/repository_simulator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
import os
import tempfile
from collections import OrderedDict
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from typing import Dict, Iterator, List, Optional, Tuple
from urllib import parse
Expand Down Expand Up @@ -81,6 +81,14 @@
SPEC_VER = ".".join(SPECIFICATION_VERSION)


@dataclass
class FetchTracker:
"""Fetcher counter for metadata and targets."""

metadata: List[Tuple[str, Optional[int]]] = field(default_factory=list)
targets: List[Tuple[str, Optional[str]]] = field(default_factory=list)


@dataclass
class RepositoryTarget:
"""Contains actual target data and the related target metadata."""
Expand Down Expand Up @@ -116,6 +124,8 @@ def __init__(self) -> None:
self.dump_dir: Optional[str] = None
self.dump_version = 0

self.fetch_tracker = FetchTracker()

now = datetime.utcnow()
self.safe_expiry = now.replace(microsecond=0) + timedelta(days=30)

Expand Down Expand Up @@ -229,6 +239,8 @@ def _fetch_target(

If hash is None, then consistent_snapshot is not used.
"""
self.fetch_tracker.targets.append((target_path, target_hash))

repo_target = self.target_files.get(target_path)
if repo_target is None:
raise FetcherHTTPError(f"No target {target_path}", 404)
Expand All @@ -248,6 +260,8 @@ def _fetch_metadata(

If version is None, non-versioned metadata is being requested.
"""
self.fetch_tracker.metadata.append((role, version))

if role == Root.type:
# return a version previously serialized in publish_root()
if version is None or version > len(self.signed_roots):
Expand Down
87 changes: 42 additions & 45 deletions tests/test_updater_consistent_snapshot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import tempfile
import unittest
from typing import Any, Dict, Iterable, List, Optional
from unittest.mock import call, patch

from tests import utils
from tests.repository_simulator import RepositorySimulator
Expand Down Expand Up @@ -90,19 +89,19 @@ def _assert_targets_files_exist(self, filenames: Iterable[str]) -> None:
"consistent_snaphot disabled": {
"consistent_snapshot": False,
"calls": [
call("root", 3),
call("timestamp", None),
call("snapshot", None),
call("targets", None),
("root", 3),
("timestamp", None),
("snapshot", None),
("targets", None),
],
},
"consistent_snaphot enabled": {
"consistent_snapshot": True,
"calls": [
call("root", 3),
call("timestamp", None),
call("snapshot", 1),
call("targets", 1),
("root", 3),
("timestamp", None),
("snapshot", 1),
("targets", 1),
],
},
}
Expand All @@ -117,15 +116,14 @@ def test_top_level_roles_update(self, test_case_data: Dict[str, Any]):
sim = self._init_repo(consistent_snapshot)
updater = self._init_updater(sim)

with patch.object(
sim, "_fetch_metadata", wraps=sim._fetch_metadata
) as wrapped_fetch:
updater.refresh()
# cleanup fetch tracker metadata
sim.fetch_tracker.metadata.clear()
updater.refresh()

# metadata files are fetched with the expected version (or None)
self.assertListEqual(wrapped_fetch.call_args_list, expected_calls)
# metadata files are always persisted without a version prefix
self._assert_metadata_files_exist(TOP_LEVEL_ROLE_NAMES)
# metadata files are fetched with the expected version (or None)
self.assertListEqual(sim.fetch_tracker.metadata, expected_calls)
# metadata files are always persisted without a version prefix
self._assert_metadata_files_exist(TOP_LEVEL_ROLE_NAMES)

self._cleanup_dir(self.metadata_dir)

Expand All @@ -147,7 +145,7 @@ def test_delegated_roles_update(self, test_case_data: Dict[str, Any]):
consistent_snapshot: bool = test_case_data["consistent_snapshot"]
expected_version: Optional[int] = test_case_data["expected_version"]
rolenames = ["role1", "..", "."]
expected_calls = [call(role, expected_version) for role in rolenames]
expected_calls = [(role, expected_version) for role in rolenames]

sim = self._init_repo(consistent_snapshot)
# Add new delegated targets
Expand All @@ -159,15 +157,14 @@ def test_delegated_roles_update(self, test_case_data: Dict[str, Any]):
updater = self._init_updater(sim)
updater.refresh()

with patch.object(
sim, "_fetch_metadata", wraps=sim._fetch_metadata
) as wrapped_fetch:
# trigger updater to fetch the delegated metadata
updater.get_targetinfo("anything")
# metadata files are fetched with the expected version (or None)
self.assertListEqual(wrapped_fetch.call_args_list, expected_calls)
# metadata files are always persisted without a version prefix
self._assert_metadata_files_exist(rolenames)
# cleanup fetch tracker metadata
sim.fetch_tracker.metadata.clear()
# trigger updater to fetch the delegated metadata
updater.get_targetinfo("anything")
# metadata files are fetched with the expected version (or None)
self.assertListEqual(sim.fetch_tracker.metadata, expected_calls)
# metadata files are always persisted without a version prefix
self._assert_metadata_files_exist(rolenames)

self._cleanup_dir(self.metadata_dir)

Expand All @@ -176,16 +173,19 @@ def test_delegated_roles_update(self, test_case_data: Dict[str, Any]):
"consistent_snapshot": False,
"prefix_targets": True,
"hash_algo": None,
"targetpaths": ["file", "file.txt", "..file.ext", "f.le"],
},
"consistent_snaphot enabled without prefixed targets": {
"consistent_snapshot": True,
"prefix_targets": False,
"hash_algo": None,
"targetpaths": ["file", "file.txt", "..file.ext", "f.le"],
},
"consistent_snaphot enabled with prefixed targets": {
"consistent_snapshot": True,
"prefix_targets": True,
"hash_algo": "sha256",
"targetpaths": ["file", "file.txt", "..file.ext", "f.le"],
},
}

Expand All @@ -197,7 +197,7 @@ def test_download_targets(self, test_case_data: Dict[str, Any]):
consistent_snapshot: bool = test_case_data["consistent_snapshot"]
prefix_targets_with_hash: bool = test_case_data["prefix_targets"]
hash_algo: Optional[str] = test_case_data["hash_algo"]
targetpaths = ["file", "file.txt", "..file.ext", "f.le"]
targetpaths: List[str] = test_case_data["targetpaths"]

sim = self._init_repo(consistent_snapshot, prefix_targets_with_hash)
# Add targets to repository
Expand All @@ -210,23 +210,20 @@ def test_download_targets(self, test_case_data: Dict[str, Any]):
updater.config.prefix_targets_with_hash = prefix_targets_with_hash
updater.refresh()

with patch.object(
sim, "_fetch_target", wraps=sim._fetch_target
) as wrapped_fetch_target:

for targetpath in targetpaths:
info = updater.get_targetinfo(targetpath)
updater.download_target(info)
expected_prefix = (
None if not hash_algo else info.hashes[hash_algo]
)
# files are fetched with the expected hash prefix (or None)
wrapped_fetch_target.assert_called_once_with(
info.path, expected_prefix
)
# target files are always persisted without hash prefix
self._assert_targets_files_exist([info.path])
wrapped_fetch_target.reset_mock()
for targetpath in targetpaths:
info = updater.get_targetinfo(targetpath)
updater.download_target(info)

# target files are always persisted without hash prefix
self._assert_targets_files_exist([info.path])

# files are fetched with the expected hash prefix (or None)
expected_fetches = [
(targetpath, None if not hash_algo else info.hashes[hash_algo])
]

self.assertListEqual(sim.fetch_tracker.targets, expected_fetches)
sim.fetch_tracker.targets.clear()

self._cleanup_dir(self.targets_dir)

Expand Down