Using decorators for comprehensive testing

Jussi in his comment here:
#1391 (comment)
proposed using decorators when creating comprehensive testing
for metadata serialization.
The main problems he pointed out is that:
1) there is a lot of code needed to generate the data for each case
2) the test implementation scales badly when you want to add new
cases for your tests, then you would have to add code as well
3) the dictionary format is not visible - we are loading external files
and assuming they are not changed and valid

In this change, I am using a decorator with an argument that complicates
the implementation of the decorator and requires three nested functions,
but the advantages are that we are resolving the above three problems:
1) we don't need new code when adding a new test case
2) a small amount of hardcoded data is required for each new test
3) the dictionaries are all in the test module without the need of
creating new directories and copying data.

Signed-off-by: Martin Vrachev <[email protected]>

Author: MVrachev

tests/test_api.py

@@ -333,7 +333,7 @@ def test_key_class(self):
                 test_key_dict = key_dict.copy()
                 del test_key_dict[key]
                 with self.assertRaises(KeyError):
-                    Key.from_dict(test_key_dict)
+                    Key.from_dict("id", test_key_dict)
 
 
     def test_role_class(self):
@@ -357,7 +357,7 @@ def test_role_class(self):
                 test_role_dict = role_dict.copy()
                 del test_role_dict[role_attr]
                 with self.assertRaises(KeyError):
-                    Key.from_dict(test_role_dict)
+                    Role.from_dict(test_role_dict)
 
 
     def test_metadata_root(self):

tests/test_metadata_serialization.py

@@ -0,0 +1,206 @@
+# Copyright New York University and the TUF contributors
+# SPDX-License-Identifier: MIT OR Apache-2.0
+
+""" Unit tests testing tuf/api/metadata.py classes
+serialization and deserialization.
+
+"""
+
+import json
+import sys
+import logging
+import unittest
+import copy
+
+from typing import Any, Dict, Callable, Type
+
+from tests import utils
+
+from tuf.api.metadata import (
+    Root,
+    Snapshot,
+    Timestamp,
+    Targets,
+    Key,
+    Role,
+    MetaFile,
+    TargetFile,
+    Delegations,
+    DelegatedRole,
+)
+
+logger = logging.getLogger(__name__)
+
+# DataSet is only here so type hints can be used:
+# It is a dict of name to test dict
+DataSet = Dict[str, Dict[str, Any]]
+
+# Test runner decorator: Runs the test as a set of N SubTests,
+# (where N is number of items in dataset), feeding the actual test
+# function one data item at a time
+def run_sub_tests_with_dataset(dataset: Type[DataSet]):
+    def real_decorator(function: Callable[["TestSerialization", DataSet], None]):
+        def wrapper(test_cls: "TestSerialization"):
+            for case, data in dataset.items():
+                with test_cls.subTest(case=case):
+                    function(test_cls, data)
+        return wrapper
+    return real_decorator
+
+
+class TestSerialization(unittest.TestCase):
+
+    KEY = '{"keytype": "rsa", "scheme": "rsassa-pss-sha256", \
+        "keyval": {"public": "foo"}}'
+    SIGNED_COMMON = '"spec_version": "1.0.0", "version": 1, \
+        "expires": "2030-01-01T00:00:00Z"'
+
+    valid_keys: DataSet = {
+        "all": '{"keytype": "rsa", "scheme": "rsassa-pss-sha256", \
+            "keyval": {"public": "foo"}}',
+    }
+
+    @run_sub_tests_with_dataset(valid_keys)
+    def test_key_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        key = Key.from_dict("id", copy.copy(case_dict))
+        self.assertDictEqual(case_dict, key.to_dict())
+
+
+    valid_roles: DataSet = {
+        "all": '{"keyids": ["keyid"], "threshold": 3}'
+    }
+
+    @run_sub_tests_with_dataset(valid_roles)
+    def test_role_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        role = Role.from_dict(copy.deepcopy(case_dict))
+        self.assertDictEqual(case_dict, role.to_dict())
+
+
+    valid_roots: DataSet = {
+        "all": f'{{ "_type": "root", {SIGNED_COMMON}, \
+            "consistent_snapshot": false, "keys": {{"keyid" : {KEY} }}, \
+            "roles": {{ "targets": {{"keyids": ["keyid"], "threshold": 3}} }} \
+            }}',
+        "no consistent_snapshot": f'{{ "_type": "root", {SIGNED_COMMON}, \
+            "keys": {{"keyid" : {KEY}}}, \
+            "roles": {{ "targets": {{"keyids": ["keyid"], "threshold": 3}} }} \
+            }}',
+    }
+
+    @run_sub_tests_with_dataset(valid_roots)
+    def test_root_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        root = Root.from_dict(copy.deepcopy(case_dict))
+        self.assertDictEqual(case_dict, root.to_dict())
+
+    valid_metafiles: DataSet = {
+        "all": '{"hashes": {"sha256" : "abc"}, "length": 12, "version": 1}',
+        "no length": '{"hashes": {"sha256" : "abc"}, "version": 1 }',
+        "no hashes": '{"length": 12, "version": 1}'
+    }
+
+    @run_sub_tests_with_dataset(valid_metafiles)
+    def test_metafile_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        metafile = MetaFile.from_dict(copy.copy(case_dict))
+        self.assertDictEqual(case_dict, metafile.to_dict())
+
+
+    valid_timestamps: DataSet = {
+        "all": f'{{ "_type": "timestamp", {SIGNED_COMMON}, \
+            "meta": {{ "snapshot.json": {{ "hashes": {{"sha256" : "abc"}}, "version": 1 }} }} \
+            }}'
+    }
+
+    @run_sub_tests_with_dataset(valid_timestamps)
+    def test_timestamp_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        timestamp = Timestamp.from_dict(copy.deepcopy(case_dict))
+        self.assertDictEqual(case_dict, timestamp.to_dict())
+
+
+    valid_snapshots: DataSet = {
+        "all": f'{{ "_type": "snapshot", {SIGNED_COMMON}, \
+            "meta": {{ "file.txt": \
+            {{ "hashes": {{"sha256" : "abc"}}, "version": 1 }} }} }}'
+    }
+
+    @run_sub_tests_with_dataset(valid_snapshots)
+    def test_snapshot_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        snapshot = Snapshot.from_dict(copy.deepcopy(case_dict))
+        self.assertDictEqual(case_dict, snapshot.to_dict())
+
+
+    valid_delegated_roles: DataSet = {
+        "no hash prefix attribute":
+            '{"keyids": ["keyid"], "name": "a", "paths": ["fn1", "fn2"], \
+            "terminating": false, "threshold": 1}',
+        "no path attribute":
+            '{"keyids": ["keyid"], "name": "a", "terminating": false, \
+            "path_hash_prefixes": ["h1", "h2"], "threshold": 99}',
+        "no hash or path prefix":
+            '{"keyids": ["keyid"], "name": "a", "terminating": true, "threshold": 3}',
+    }
+
+    @run_sub_tests_with_dataset(valid_delegated_roles)
+    def test_delegated_role_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        deserialized_role = DelegatedRole.from_dict(copy.copy(case_dict))
+        self.assertDictEqual(case_dict, deserialized_role.to_dict())
+
+
+    valid_delegations: DataSet = {
+        "all": f'{{"keys": {{"keyid" : {KEY}}}, "roles": [ {{"keyids": ["keyid"], \
+            "name": "a", "terminating": true, "threshold": 3}} ]}}'
+    }
+
+    @run_sub_tests_with_dataset(valid_delegations)
+    def test_delegation_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        delegation = Delegations.from_dict(copy.deepcopy(case_dict))
+        self.assertDictEqual(case_dict, delegation.to_dict())
+
+
+    valid_targetfiles: DataSet = {
+        "all": '{"length": 12, "hashes": {"sha256" : "abc"}, \
+            "custom" : {"foo": "bar"} }',
+        "no custom": '{"length": 12, "hashes": {"sha256" : "abc"}}'
+    }
+
+    @run_sub_tests_with_dataset(valid_targetfiles)
+    def test_targetfile_serialization(self, test_case_data: str):
+        case_dict = json.loads(test_case_data)
+        target_file = TargetFile.from_dict(copy.copy(case_dict))
+        self.assertDictEqual(case_dict, target_file.to_dict())
+
+
+    valid_targets: DataSet = {
+        "all attributes": f'{{"_type": "targets", {SIGNED_COMMON}, \
+            "targets": {{ "file.txt": {{"length": 12, "hashes": {{"sha256" : "abc"}} }} }}, \
+            "delegations": {{"keys": {{"keyid" : {KEY}}}, \
+                "roles": [ {{"keyids": ["keyid"], "name": "a", "terminating": true, "threshold": 3}} ]}} \
+            }}',
+        "empty targets": f'{{"_type": "targets", {SIGNED_COMMON}, \
+            "targets": {{}}, \
+            "delegations": {{"keys": {{"keyid" : {KEY}}}, \
+                "roles": [ {{"keyids": ["keyid"], "name": "a", "terminating": true, "threshold": 3}} ] \
+            }} }}',
+        "no delegations": f'{{"_type": "targets", {SIGNED_COMMON}, \
+            "targets":  {{ "file.txt": {{"length": 12, "hashes": {{"sha256" : "abc"}} }} }} \
+            }}'
+    }
+
+    @run_sub_tests_with_dataset(valid_targets)
+    def test_targets_serialization(self, test_case_data):
+        case_dict = json.loads(test_case_data)
+        targets = Targets.from_dict(copy.deepcopy(case_dict))
+        self.assertDictEqual(case_dict, targets.to_dict())
+
+
+# Run unit test.
+if __name__ == '__main__':
+    utils.configure_test_logging(sys.argv)
+    unittest.main()