New metadata API: add support for ADR 0008

In order to support ADR 0008 we would want to accept unrecognized
fields in all metadata classes, including the classes that would be
added representing a subportion of a role like "meta", "delegations"
and "roles".

Also, we should test that we support unrecognized fields when adding
new classes or modifying existing ones to make sure we support
ADR 0008.

Signed-off-by: Martin Vrachev <[email protected]>

Author: MVrachev

tests/test_api.py

@@ -50,6 +50,9 @@
 from securesystemslib.signer import (
     SSlibSigner
 )
+from securesystemslib.storage import(
+    FilesystemBackend
+)
 
 logger = logging.getLogger(__name__)
 
@@ -330,6 +333,32 @@ def test_metadata_targets(self):
         # Verify that data is updated
         self.assertEqual(targets.signed.targets[filename], fileinfo)
 
+    def setup_json_dict_with_unrecognized_fields(self, file_path):
+        storage_backend = FilesystemBackend()
+        json_dict = {}
+        with storage_backend.get(file_path) as file_obj:
+            raw_data = file_obj.read()
+            json_dict = json.loads(raw_data.decode("utf-8"))
+        # We are changing the json dict without changing the signature.
+        # This could be a problem if we want to do verification on this dict.
+        json_dict["signed"]["foo"] = "bar"
+        return json_dict
+
+    def test_support_for_unrecognized_fields(self):
+        for metadata in ["root", "timestamp", "snapshot", "targets"]:
+            path = os.path.join(self.repo_dir, "metadata", metadata + ".json")
+            json_dict = self.setup_json_dict_with_unrecognized_fields(path)
+            # Test that the metadata classes store unrecognized fields when
+            # initializing and passes them when casting the instance to a dict.
+
+            # TODO: Remove the deepcopy when Metadata.from_dict() doesn't have
+            # the side effect to destroy the passed dictionary.
+            temp_copy = copy.deepcopy(json_dict)
+            metadata_obj = Metadata.from_dict(temp_copy)
+
+            self.assertEqual(json_dict["signed"], metadata_obj.signed.to_dict())
+
+
 # Run unit test.
 if __name__ == '__main__':
     utils.configure_test_logging(sys.argv)

tuf/api/metadata.py

@@ -361,6 +361,20 @@ def bump_version(self) -> None:
         self.version += 1
 
 
+def _get_unrecognized_fields(
+    all_fields: Mapping[str, Any], recognized_fields: list
+) -> Mapping[str, Any]:
+    """Utility function to get the additional unrecognized fields from a dict.
+    Read ADR 0008."""
+
+    unrecognized_fields = {}
+    for key, value in all_fields.items():
+        if key not in recognized_fields:
+            unrecognized_fields[key] = value
+
+    return unrecognized_fields
+
+
 class Root(Signed):
     """A container for the signed part of root metadata.
 
@@ -393,6 +407,9 @@ class Root(Signed):
                 },
                 ...
             }
+        unrecognized_fields: An optional dictionary used to store any
+            unrecognized fields. Needed for backward compatibility.
+            Read ADR 0008.
 
     """
 
@@ -409,12 +426,14 @@ def __init__(
         consistent_snapshot: bool,
         keys: Mapping[str, Any],
         roles: Mapping[str, Any],
+        unrecognized_fields: Optional[Mapping[str, Any]] = None,
     ) -> None:
         super().__init__(_type, version, spec_version, expires)
         # TODO: Add classes for keys and roles
         self.consistent_snapshot = consistent_snapshot
         self.keys = keys
         self.roles = roles
+        self.unrecognized_fields = unrecognized_fields
 
     @classmethod
     def from_dict(cls, root_dict: Mapping[str, Any]) -> "Root":
@@ -423,7 +442,22 @@ def from_dict(cls, root_dict: Mapping[str, Any]) -> "Root":
         consistent_snapshot = root_dict.pop("consistent_snapshot")
         keys = root_dict.pop("keys")
         roles = root_dict.pop("roles")
-        return cls(*common_args, consistent_snapshot, keys, roles)
+        # Store unregonized fields. Read ADR 0008.
+        expected_fields = [
+            "_type",
+            "version",
+            "spec_version",
+            "expires",
+            "consistent_snapshot",
+            "keys",
+            "roles",
+        ]
+        unrecognized_fields = _get_unrecognized_fields(
+            root_dict, expected_fields
+        )
+        return cls(
+            *common_args, consistent_snapshot, keys, roles, unrecognized_fields
+        )
 
     def to_dict(self) -> Dict[str, Any]:
         """Returns the dict representation of self. """
@@ -433,6 +467,7 @@ def to_dict(self) -> Dict[str, Any]:
                 "consistent_snapshot": self.consistent_snapshot,
                 "keys": self.keys,
                 "roles": self.roles,
+                **self.unrecognized_fields,
             }
         )
         return root_dict
@@ -475,6 +510,9 @@ class Timestamp(Signed):
                     }
                 }
             }
+        unrecognized_fields: An optional dictionary used to store any
+            unrecognized fields. Needed for backward compatibility.
+            Read ADR 0008.
 
     """
 
@@ -485,22 +523,35 @@ def __init__(
         spec_version: str,
         expires: datetime,
         meta: Mapping[str, Any],
+        unrecognized_fields: Optional[Mapping[str, Any]] = None,
     ) -> None:
         super().__init__(_type, version, spec_version, expires)
         # TODO: Add class for meta
         self.meta = meta
+        self.unrecognized_fields = unrecognized_fields
 
     @classmethod
     def from_dict(cls, timestamp_dict: Mapping[str, Any]) -> "Timestamp":
         """Creates Timestamp object from its dict representation. """
         common_args = cls._common_fields_from_dict(timestamp_dict)
         meta = timestamp_dict.pop("meta")
-        return cls(*common_args, meta)
+        # Store unregonized fields. Read ADR 0008.
+        expected_fields = [
+            "_type",
+            "version",
+            "spec_version",
+            "expires",
+            "meta",
+        ]
+        unrecognized_fields = _get_unrecognized_fields(
+            timestamp_dict, expected_fields
+        )
+        return cls(*common_args, meta, unrecognized_fields)
 
     def to_dict(self) -> Dict[str, Any]:
         """Returns the dict representation of self. """
         timestamp_dict = self._common_fields_to_dict()
-        timestamp_dict.update({"meta": self.meta})
+        timestamp_dict.update({"meta": self.meta, **self.unrecognized_fields})
         return timestamp_dict
 
     # Modification.
@@ -539,6 +590,9 @@ class Snapshot(Signed):
                 },
                 ...
             }
+        unrecognized_fields: An optional dictionary used to store any
+            unrecognized fields. Needed for backward compatibility.
+            Read ADR 0008.
 
     """
 
@@ -549,22 +603,35 @@ def __init__(
         spec_version: str,
         expires: datetime,
         meta: Mapping[str, Any],
+        unrecognized_fields: Optional[Mapping[str, Any]] = None,
     ) -> None:
         super().__init__(_type, version, spec_version, expires)
         # TODO: Add class for meta
         self.meta = meta
+        self.unrecognized_fields = unrecognized_fields
 
     @classmethod
     def from_dict(cls, snapshot_dict: Mapping[str, Any]) -> "Snapshot":
         """Creates Snapshot object from its dict representation. """
         common_args = cls._common_fields_from_dict(snapshot_dict)
         meta = snapshot_dict.pop("meta")
-        return cls(*common_args, meta)
+        # Store unregonized fields. Read ADR 0008.
+        expected_fields = [
+            "_type",
+            "version",
+            "spec_version",
+            "expires",
+            "meta",
+        ]
+        unrecognized_fields = _get_unrecognized_fields(
+            snapshot_dict, expected_fields
+        )
+        return cls(*common_args, meta, unrecognized_fields)
 
     def to_dict(self) -> Dict[str, Any]:
         """Returns the dict representation of self. """
         snapshot_dict = self._common_fields_to_dict()
-        snapshot_dict.update({"meta": self.meta})
+        snapshot_dict.update({"meta": self.meta, **self.unrecognized_fields})
         return snapshot_dict
 
     # Modification.
@@ -638,6 +705,10 @@ class Targets(Signed):
                 ]
             }
 
+        unrecognized_fields: An optional dictionary used to store any
+            unrecognized fields. Needed for backward compatibility.
+            Read ADR 0008.
+
     """
 
     # TODO: determine an appropriate value for max-args and fix places where
@@ -652,19 +723,33 @@ def __init__(
         expires: datetime,
         targets: Mapping[str, Any],
         delegations: Mapping[str, Any],
+        unrecognized_fields: Optional[Mapping[str, Any]] = None,
     ) -> None:
         super().__init__(_type, version, spec_version, expires)
         # TODO: Add class for meta
         self.targets = targets
         self.delegations = delegations
+        self.unrecognized_fields = unrecognized_fields
 
     @classmethod
     def from_dict(cls, targets_dict: Mapping[str, Any]) -> "Targets":
         """Creates Targets object from its dict representation. """
         common_args = cls._common_fields_from_dict(targets_dict)
         targets = targets_dict.pop("targets")
         delegations = targets_dict.pop("delegations")
-        return cls(*common_args, targets, delegations)
+        # Store unregonized fields. Read ADR 0008.
+        expected_fields = [
+            "_type",
+            "version",
+            "spec_version",
+            "expires",
+            "meta",
+            ["targets", "delegations"],
+        ]
+        unrecognized_fields = _get_unrecognized_fields(
+            targets_dict, expected_fields
+        )
+        return cls(*common_args, targets, delegations, unrecognized_fields)
 
     def to_dict(self) -> Dict[str, Any]:
         """Returns the dict representation of self. """
@@ -673,6 +758,7 @@ def to_dict(self) -> Dict[str, Any]:
             {
                 "targets": self.targets,
                 "delegations": self.delegations,
+                **self.unrecognized_fields,
             }
         )
         return targets_dict