Merge pull request #172 from simleo/cli_add_properties

CLI: Support for adding arbitrary properties

Author: simleo

README.md

@@ -310,7 +310,7 @@ The command acts on the current directory, unless the `-c` option is specified.
 
 ### Adding items to the crate
 
-The `rocrate add` command allows to add workflows and other entity types (currently [testing-related metadata](https://crs4.github.io/life_monitor/workflow_testing_ro_crate)) to an RO-Crate:
+The `rocrate add` command allows to add file, datasets (directories), workflows and other entity types (currently [testing-related metadata](https://crs4.github.io/life_monitor/workflow_testing_ro_crate)) to an RO-Crate:
 
 ```console
 $ rocrate add --help
@@ -320,6 +320,8 @@ Options:
   --help  Show this message and exit.
 
 Commands:
+  dataset
+  file
   test-definition
   test-instance
   test-suite
@@ -372,6 +374,29 @@ rocrate add test-instance test1 http://example.com -r jobs -i test1_1
 rocrate add test-definition test1 test/test1/sort-and-change-case-test.yml -e planemo -v '>=0.70'
 ```
 
+To add files or directories after crate initialization:
+
+```bash
+cp ../sample_file.txt .
+rocrate add file sample_file.txt -P name=sample -P description="Sample file"
+cp -r ../test_add_dir .
+rocrate add dataset test_add_dir
+```
+
+The above example also shows how to set arbitrary properties for the entity with `-P`. This is supported by most `rocrate add` subcommands.
+
+```console
+$ rocrate add workflow --help
+Usage: rocrate add workflow [OPTIONS] PATH
+
+Options:
+  -l, --language [cwl|galaxy|knime|nextflow|snakemake|compss|autosubmit]
+  -c, --crate-dir PATH
+  -P, --property KEY=VALUE
+  --help                          Show this message and exit.
+```
+
+
 ## License
 
  * Copyright 2019-2024 The University of Manchester, UK

rocrate/cli.py

@@ -44,8 +44,20 @@ def convert(self, value, param, ctx):
             self.fail(f"{value!r} is not splittable", param, ctx)
 
 
+class KeyValueParamType(click.ParamType):
+    name = "key_value"
+
+    def convert(self, value, param, ctx):
+        try:
+            return tuple(value.split("=", 1)) if value else ()
+        except AttributeError:
+            self.fail(f"{value!r} is not splittable", param, ctx)
+
+
 CSV = CSVParamType()
+KeyValue = KeyValueParamType()
 OPTION_CRATE_PATH = click.option('-c', '--crate-dir', type=click.Path(), default=os.getcwd)
+OPTION_PROPS = click.option('-P', '--property', type=KeyValue, multiple=True, metavar="KEY=VALUE")
 
 
 @click.group()
@@ -72,38 +84,41 @@ def add():
 @add.command()
 @click.argument('path', type=click.Path(exists=True, dir_okay=False))
 @OPTION_CRATE_PATH
-def file(crate_dir, path):
+@OPTION_PROPS
+def file(crate_dir, path, property):
     crate = ROCrate(crate_dir, init=False, gen_preview=False)
     source = Path(path).resolve(strict=True)
     try:
         dest_path = source.relative_to(crate_dir)
     except ValueError:
         # For now, only support adding an existing file to the metadata
         raise ValueError(f"{source} is not in the crate dir {crate_dir}")
-    crate.add_file(source, dest_path)
+    crate.add_file(source, dest_path, properties=dict(property))
     crate.metadata.write(crate_dir)
 
 
 @add.command()
 @click.argument('path', type=click.Path(exists=True, file_okay=False))
 @OPTION_CRATE_PATH
-def dataset(crate_dir, path):
+@OPTION_PROPS
+def dataset(crate_dir, path, property):
     crate = ROCrate(crate_dir, init=False, gen_preview=False)
     source = Path(path).resolve(strict=True)
     try:
         dest_path = source.relative_to(crate_dir)
     except ValueError:
         # For now, only support adding an existing directory to the metadata
         raise ValueError(f"{source} is not in the crate dir {crate_dir}")
-    crate.add_dataset(source, dest_path)
+    crate.add_dataset(source, dest_path, properties=dict(property))
     crate.metadata.write(crate_dir)
 
 
 @add.command()
 @click.argument('path', type=click.Path(exists=True))
 @click.option('-l', '--language', type=click.Choice(LANG_CHOICES), default="cwl")
 @OPTION_CRATE_PATH
-def workflow(crate_dir, path, language):
+@OPTION_PROPS
+def workflow(crate_dir, path, language, property):
     crate = ROCrate(crate_dir, init=False, gen_preview=False)
     source = Path(path).resolve(strict=True)
     try:
@@ -112,7 +127,7 @@ def workflow(crate_dir, path, language):
         # For now, only support marking an existing file as a workflow
         raise ValueError(f"{source} is not in the crate dir {crate_dir}")
     # TODO: add command options for main and gen_cwl
-    crate.add_workflow(source, dest_path, main=True, lang=language, gen_cwl=False)
+    crate.add_workflow(source, dest_path, main=True, lang=language, gen_cwl=False, properties=dict(property))
     crate.metadata.write(crate_dir)
 
 
@@ -121,9 +136,13 @@ def workflow(crate_dir, path, language):
 @click.option('-n', '--name')
 @click.option('-m', '--main-entity')
 @OPTION_CRATE_PATH
-def suite(crate_dir, identifier, name, main_entity):
+@OPTION_PROPS
+def suite(crate_dir, identifier, name, main_entity, property):
     crate = ROCrate(crate_dir, init=False, gen_preview=False)
-    suite = crate.add_test_suite(identifier=add_hash(identifier), name=name, main_entity=main_entity)
+    suite = crate.add_test_suite(
+        identifier=add_hash(identifier), name=name, main_entity=main_entity,
+        properties=dict(property)
+    )
     crate.metadata.write(crate_dir)
     print(suite.id)
 
@@ -136,11 +155,12 @@ def suite(crate_dir, identifier, name, main_entity):
 @click.option('-i', '--identifier')
 @click.option('-n', '--name')
 @OPTION_CRATE_PATH
-def instance(crate_dir, suite, url, resource, service, identifier, name):
+@OPTION_PROPS
+def instance(crate_dir, suite, url, resource, service, identifier, name, property):
     crate = ROCrate(crate_dir, init=False, gen_preview=False)
     instance_ = crate.add_test_instance(
         add_hash(suite), url, resource=resource, service=service,
-        identifier=add_hash(identifier), name=name
+        identifier=add_hash(identifier), name=name, properties=dict(property)
     )
     crate.metadata.write(crate_dir)
     print(instance_.id)
@@ -152,7 +172,8 @@ def instance(crate_dir, suite, url, resource, service, identifier, name):
 @click.option('-e', '--engine', type=click.Choice(ENGINE_CHOICES), default="planemo")
 @click.option('-v', '--engine-version')
 @OPTION_CRATE_PATH
-def definition(crate_dir, suite, path, engine, engine_version):
+@OPTION_PROPS
+def definition(crate_dir, suite, path, engine, engine_version, property):
     crate = ROCrate(crate_dir, init=False, gen_preview=False)
     source = Path(path).resolve(strict=True)
     try:
@@ -162,7 +183,7 @@ def definition(crate_dir, suite, path, engine, engine_version):
         raise ValueError(f"{source} is not in the crate dir {crate_dir}")
     crate.add_test_definition(
         add_hash(suite), source=source, dest_path=dest_path, engine=engine,
-        engine_version=engine_version
+        engine_version=engine_version, properties=dict(property)
     )
     crate.metadata.write(crate_dir)
 

rocrate/rocrate.py

@@ -497,23 +497,24 @@ def add_workflow(
             workflow.subjectOf = cwl_workflow
         return workflow
 
-    def add_test_suite(self, identifier=None, name=None, main_entity=None):
+    def add_test_suite(self, identifier=None, name=None, main_entity=None, properties=None):
         test_ref_prop = "mentions"
         if not main_entity:
             main_entity = self.mainEntity
             if not main_entity:
                 test_ref_prop = "about"
-        suite = self.add(TestSuite(self, identifier))
-        suite.name = name or suite.id.lstrip("#")
+        suite = self.add(TestSuite(self, identifier, properties=properties))
+        if not properties or "name" not in properties:
+            suite.name = name or suite.id.lstrip("#")
         if main_entity:
             suite["mainEntity"] = main_entity
         self.root_dataset.append_to(test_ref_prop, suite)
         self.metadata.extra_terms.update(TESTING_EXTRA_TERMS)
         return suite
 
-    def add_test_instance(self, suite, url, resource="", service="jenkins", identifier=None, name=None):
+    def add_test_instance(self, suite, url, resource="", service="jenkins", identifier=None, name=None, properties=None):
         suite = self.__validate_suite(suite)
-        instance = self.add(TestInstance(self, identifier))
+        instance = self.add(TestInstance(self, identifier, properties=properties))
         instance.url = url
         instance.resource = resource
         if isinstance(service, TestService):
@@ -522,7 +523,8 @@ def add_test_instance(self, suite, url, resource="", service="jenkins", identifi
             service = get_service(self, service)
             self.add(service)
         instance.service = service
-        instance.name = name or instance.id.lstrip("#")
+        if not properties or "name" not in properties:
+            instance.name = name or instance.id.lstrip("#")
         suite.append_to("instance", instance)
         self.metadata.extra_terms.update(TESTING_EXTRA_TERMS)
         return instance

test/test_cli.py

@@ -145,18 +145,19 @@ def test_cli_add_file(tmpdir, test_data_dir, helpers, monkeypatch, cwd):
     # add
     shutil.copy(test_data_dir / "sample_file.txt", crate_dir)
     file_path = crate_dir / "sample_file.txt"
-    args = ["add", "file"]
+    args = ["add", "file", str(file_path), "-P", "name=foo", "-P", "description=foo bar"]
     if cwd:
         monkeypatch.chdir(str(crate_dir))
         file_path = file_path.relative_to(crate_dir)
     else:
         args.extend(["-c", str(crate_dir)])
-    args.append(str(file_path))
     result = runner.invoke(cli, args)
     assert result.exit_code == 0
     json_entities = helpers.read_json_entities(crate_dir)
     assert "sample_file.txt" in json_entities
     assert json_entities["sample_file.txt"]["@type"] == "File"
+    assert json_entities["sample_file.txt"]["name"] == "foo"
+    assert json_entities["sample_file.txt"]["description"] == "foo bar"
 
 
 @pytest.mark.parametrize("cwd", [False, True])
@@ -171,18 +172,19 @@ def test_cli_add_dataset(tmpdir, test_data_dir, helpers, monkeypatch, cwd):
     # add
     dataset_path = crate_dir / "test_add_dir"
     shutil.copytree(test_data_dir / "test_add_dir", dataset_path)
-    args = ["add", "dataset"]
+    args = ["add", "dataset", str(dataset_path), "-P", "name=foo", "-P", "description=foo bar"]
     if cwd:
         monkeypatch.chdir(str(crate_dir))
         dataset_path = dataset_path.relative_to(crate_dir)
     else:
         args.extend(["-c", str(crate_dir)])
-    args.append(str(dataset_path))
     result = runner.invoke(cli, args)
     assert result.exit_code == 0
     json_entities = helpers.read_json_entities(crate_dir)
     assert "test_add_dir/" in json_entities
     assert json_entities["test_add_dir/"]["@type"] == "Dataset"
+    assert json_entities["test_add_dir/"]["name"] == "foo"
+    assert json_entities["test_add_dir/"]["description"] == "foo bar"
 
 
 @pytest.mark.parametrize("cwd", [False, True])
@@ -196,7 +198,7 @@ def test_cli_add_workflow(test_data_dir, helpers, monkeypatch, cwd):
     assert json_entities["sort-and-change-case.ga"]["@type"] == "File"
     # add
     wf_path = crate_dir / "sort-and-change-case.ga"
-    args = ["add", "workflow"]
+    args = ["add", "workflow", "-P", "name=foo", "-P", "description=foo bar"]
     if cwd:
         monkeypatch.chdir(str(crate_dir))
         wf_path = wf_path.relative_to(crate_dir)
@@ -212,6 +214,8 @@ def test_cli_add_workflow(test_data_dir, helpers, monkeypatch, cwd):
         lang_id = f"https://w3id.org/workflowhub/workflow-ro-crate#{lang}"
         assert lang_id in json_entities
         assert json_entities["sort-and-change-case.ga"]["programmingLanguage"]["@id"] == lang_id
+        assert json_entities["sort-and-change-case.ga"]["name"] == "foo"
+        assert json_entities["sort-and-change-case.ga"]["description"] == "foo bar"
 
 
 @pytest.mark.parametrize("cwd", [False, True])
@@ -228,20 +232,27 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd):
     wf_path = crate_dir / "sort-and-change-case.ga"
     assert runner.invoke(cli, ["add", "workflow", "-c", str(crate_dir), "-l", "galaxy", str(wf_path)]).exit_code == 0
     # add test suite
-    result = runner.invoke(cli, ["add", "test-suite", "-c", str(crate_dir)])
+    result = runner.invoke(cli, ["add", "test-suite", "-c", str(crate_dir),
+                                 "-P", "name=foo", "-P", "description=foo bar"])
     assert result.exit_code == 0
     suite_id = result.output.strip()
     json_entities = helpers.read_json_entities(crate_dir)
     assert suite_id in json_entities
+    assert json_entities[suite_id]["name"] == "foo"
+    assert json_entities[suite_id]["description"] == "foo bar"
     # add test instance
-    result = runner.invoke(cli, ["add", "test-instance", "-c", str(crate_dir), suite_id, "http://example.com", "-r", "jobs"])
+    result = runner.invoke(cli, ["add", "test-instance", "-c", str(crate_dir),
+                                 suite_id, "http://example.com", "-r", "jobs",
+                                 "-P", "name=foo", "-P", "description=foo bar"])
     assert result.exit_code == 0
     instance_id = result.output.strip()
     json_entities = helpers.read_json_entities(crate_dir)
     assert instance_id in json_entities
+    assert json_entities[instance_id]["name"] == "foo"
+    assert json_entities[instance_id]["description"] == "foo bar"
     # add test definition
     def_path = crate_dir / def_id
-    args = ["add", "test-definition"]
+    args = ["add", "test-definition", "-P", "name=foo", "-P", "description=foo bar"]
     if cwd:
         monkeypatch.chdir(str(crate_dir))
         def_path = def_path.relative_to(crate_dir)
@@ -253,6 +264,8 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd):
     json_entities = helpers.read_json_entities(crate_dir)
     assert def_id in json_entities
     assert set(json_entities[def_id]["@type"]) == {"File", "TestDefinition"}
+    assert json_entities[def_id]["name"] == "foo"
+    assert json_entities[def_id]["description"] == "foo bar"
     # check extra terms
     metadata_path = crate_dir / helpers.METADATA_FILE_NAME
     with open(metadata_path, "rt") as f: