Merge branch feat-persist-custom-schema-annotations into oold-and-opensemantic-package-integration

Author: simontaurus

src/osw/core.py

@@ -33,6 +33,7 @@
 import osw.model.entity as model
 from osw.defaults import params as default_params
 from osw.model.static import OswBaseModel
+from osw.utils.codegen import OOLDJsonSchemaParser
 from osw.utils.oold import (
     AggregateGeneratedSchemasParam,
     AggregateGeneratedSchemasParamMode,
@@ -380,6 +381,9 @@ class FetchSchemaParam(BaseModel):
         )
         legacy_generator: Optional[bool] = False
         """uses legacy command line for code generation if true"""
+        generate_annotations: Optional[bool] = True
+        """generate custom schema keywords in Fields and Classes.
+        Required to update the schema in OSW without information loss"""
         offline_pages: Optional[Dict[str, WtPage]] = None
         """pages to be used offline instead of fetching them from the OSW instance"""
         result_model_path: Optional[Union[str, pathlib.Path]] = None
@@ -410,6 +414,7 @@ def fetch_schema(self, fetchSchemaParam: FetchSchemaParam = None) -> None:
                     schema_title=schema_title,
                     mode=mode,
                     legacy_generator=fetchSchemaParam.legacy_generator,
+                    generate_annotations=fetchSchemaParam.generate_annotations,
                     offline_pages=fetchSchemaParam.offline_pages,
                     result_model_path=fetchSchemaParam.result_model_path,
                 )
@@ -438,6 +443,9 @@ class _FetchSchemaParam(BaseModel):
         )
         legacy_generator: Optional[bool] = False
         """uses legacy command line for code generation if true"""
+        generate_annotations: Optional[bool] = False
+        """generate custom schema keywords in Fields and Classes.
+        Required to update the schema in OSW without information loss"""
         offline_pages: Optional[Dict[str, WtPage]] = None
         """pages to be used offline instead of fetching them from the OSW instance"""
         result_model_path: Optional[Union[str, pathlib.Path]] = None
@@ -596,6 +604,12 @@ def _fetch_schema(self, fetchSchemaParam: _FetchSchemaParam = None) -> None:
                 # suppress deprecation warnings from pydantic
                 # see https://github.com/koxudaxi/datamodel-code-generator/issues/2213
                 warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20)
+
+                if fetchSchemaParam.generate_annotations:
+                    # monkey patch class
+                    datamodel_code_generator.parser.jsonschema.JsonSchemaParser = (
+                        OOLDJsonSchemaParser
+                    )
                 datamodel_code_generator.generate(
                     input_=pathlib.Path(schema_path),
                     input_file_type="jsonschema",
@@ -620,6 +634,43 @@ def _fetch_schema(self, fetchSchemaParam: _FetchSchemaParam = None) -> None:
                 )
                 warnings.filterwarnings("default", category=PydanticDeprecatedSince20)
 
+                # note: we could use OOLDJsonSchemaParser directly (see below),
+                # but datamodel_code_generator.generate
+                # does some pre- and postprocessing we do not want to duplicate
+
+                # data_model_type = datamodel_code_generator.DataModelType.PydanticBaseModel
+                # #data_model_type = DataModelType.PydanticV2BaseModel
+                # target_python_version = datamodel_code_generator.PythonVersion.PY_38
+                # data_model_types = datamodel_code_generator.model.get_data_model_types(
+                #   data_model_type, target_python_version
+                # )
+                # parser = OOLDJsonSchemaParserFixedRefs(
+                #     source=pathlib.Path(schema_path),
+
+                #     base_class="osw.model.static.OswBaseModel",
+                #     data_model_type=data_model_types.data_model,
+                #     data_model_root_type=data_model_types.root_model,
+                #     data_model_field_type=data_model_types.field_model,
+                #     data_type_manager_type=data_model_types.data_type_manager,
+                #     target_python_version=target_python_version,
+
+                #     #use_default=True,
+                #     apply_default_values_for_required_fields=True,
+                #     use_unique_items_as_set=True,
+                #     enum_field_as_literal=datamodel_code_generator.LiteralType.All,
+                #     use_title_as_name=True,
+                #     use_schema_description=True,
+                #     use_field_description=True,
+                #     encoding="utf-8",
+                #     use_double_quotes=True,
+                #     collapse_root_models=True,
+                #     reuse_model=True,
+                #     #field_include_all_keys=True
+                # )
+                # result = parser.parse()
+                # with open(temp_model_path, "w", encoding="utf-8") as f:
+                #     f.write(result)
+
             # see https://koxudaxi.github.io/datamodel-code-generator/
             # --base-class OswBaseModel: use a custom base class
             # --custom-template-dir src/model/template_data/
@@ -693,7 +744,6 @@ def _fetch_schema(self, fetchSchemaParam: _FetchSchemaParam = None) -> None:
                     r"class\s*([\S]*)\s*\(\s*[\S\s]*?\s*\)\s*:.*\n"
                 )  # match class definition [\s\S]*(?:[^\S\n]*\n){2,}
                 for cls in re.findall(pattern, org_content):
-                    print(cls)
                     content = re.sub(
                         r"(class\s*"
                         + cls

src/osw/model/entity.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  Item.json
-#   timestamp: 2024-09-12T12:35:11+00:00
+#   timestamp: 2024-12-10T05:09:51+00:00
 
 from __future__ import annotations
 
@@ -13,29 +13,71 @@
 
 
 class ReadAccess(OswBaseModel):
+    class Config:
+        schema_extra = {"title": "Read access", "title*": {"de": "Lesezugriff"}}
+
     level: Optional[Literal["public", "internal", "restricted"]] = Field(
         None, title="Level"
     )
 
 
 class AccessRestrictions(OswBaseModel):
+    class Config:
+        schema_extra = {
+            "title": "Access restrictions",
+            "title*": {"de": "Zugriffsbeschränkungen"},
+            "eval_template": {
+                "$comment": "See https://www.mediawiki.org/wiki/Extension:Semantic_ACL",
+                "type": "mustache-wikitext",
+                "mode": "render",
+                "value": "{{entry_access.read.level}} {{=<% %>=}} {{#set: |Visible to= {{#switch: <%={{ }}=%> {{{entry_access.read.level}}} {{=<% %>=}} |public=public |internal=users |restricted=whitelist |#default=}} }} <%={{ }}=%>",
+            },
+        }
+
     read: Optional[ReadAccess] = Field(None, title="Read access")
 
 
 class Label(OswBaseModel):
+    class Config:
+        schema_extra = {
+            "@context": {
+                "rdf": "http://www.w3.org/2000/01/rdf-schema#",
+                "text": {"@id": "@value"},
+                "lang": {"@id": "@language"},
+            },
+            "title": "Label",
+        }
+
     text: constr(min_length=1) = Field(..., title="Text")
     lang: Optional[Literal["en", "de"]] = Field("en", title="Lang code")
 
 
-class Description(Label):
-    pass
+class Description(OswBaseModel):
+    class Config:
+        schema_extra = {
+            "@context": {
+                "rdf": "http://www.w3.org/2000/01/rdf-schema#",
+                "text": {"@id": "@value"},
+                "lang": {"@id": "@language"},
+            },
+            "title": "Description",
+        }
+
+    text: constr(min_length=1) = Field(..., title="Text")
+    lang: Optional[Literal["en", "de"]] = Field("en", title="Lang code")
 
 
 class WikiPage(OswBaseModel):
     """
     The wiki page containing this entity
     """
 
+    class Config:
+        schema_extra = {
+            "title": "Wiki page",
+            "description": "The wiki page containing this entity",
+        }
+
     title: Optional[str] = Field(None, title="Title")
     """
     The page title
@@ -47,6 +89,14 @@ class WikiPage(OswBaseModel):
 
 
 class Meta(OswBaseModel):
+    class Config:
+        schema_extra = {
+            "@context": {
+                "change_id": {"@id": "Property:HasChangeId", "@type": "xsd:string"}
+            },
+            "title": "Meta",
+        }
+
     uuid: UUID = Field(default_factory=uuid4, title="UUID")
     wiki_page: Optional[WikiPage] = Field(None, title="Wiki page")
     """
@@ -59,6 +109,70 @@ class Meta(OswBaseModel):
 
 
 class Entity(OswBaseModel):
+    class Config:
+        schema_extra = {
+            "@context": {
+                "bvco": "https://bvco.ontology.link/",
+                "databatt": "http://www.databatt.org/",
+                "emmo": "https://w3id.org/emmo#",
+                "emmobattery": "https://w3id.org/emmo/domain/battery#",
+                "emmochameo": "https://w3id.org/emmo/domain/chameo#",
+                "emmochemicals": "https://w3id.org/emmo/domain/chemicalsubstance#",
+                "emmoelch": "https://w3id.org/emmo/domain/electrochemistry#",
+                "schema": "https://schema.org/",
+                "skos": "https://www.w3.org/TR/skos-reference/",
+                "xsd": "http://www.w3.org/2001/XMLSchema#",
+                "wiki": "https://wiki-dev.open-semantic-lab.org/id/",
+                "Category": {"@id": "wiki:Category-3A", "@prefix": True},
+                "File": {
+                    "@id": "https://wiki-dev.open-semantic-lab.org/wiki/Special:Redirect/file/",
+                    "@prefix": True,
+                },
+                "Property": {"@id": "wiki:Property-3A", "@prefix": True},
+                "Item": {"@id": "wiki:Item-3A", "@prefix": True},
+                "attachments*": {"@id": "Property:HasFileAttachment", "@type": "@id"},
+                "based_on": {"@id": "skos:isBasedOn", "@type": "@id"},
+                "based_on*": {"@id": "Property:IsBasedOn", "@type": "@id"},
+                "description": {"@id": "skos:definition", "@type": "@id"},
+                "description*": {"@id": "Property:HasDescription", "@type": "@id"},
+                "image": {"@id": "schema:image", "@type": "@id"},
+                "image*": {"@id": "Property:HasImage", "@type": "@id"},
+                "label": {"@id": "skos:prefLabel", "@type": "@id"},
+                "label*": {"@id": "Property:HasLabel", "@type": "@id"},
+                "lang": {"@id": "@language"},
+                "meta": {
+                    "@id": "Property:HasMeta",
+                    "@type": "@id",
+                    "@context": {
+                        "change_id": {
+                            "@id": "Property:HasChangeId",
+                            "@type": "xsd:string",
+                        }
+                    },
+                },
+                "name*": {"@id": "Property:HasName"},
+                "ordering_categories": {"@id": "Property:Category", "@type": "@id"},
+                "ordering_categories*": {
+                    "@id": "Property:HasClassificationCategory",
+                    "@type": "@id",
+                },
+                "query_label": {"@id": "Property:HasLabel", "@type": "@id"},
+                "rdf_type": {"@id": "@type", "@type": "@id"},
+                "rdf_type*": {"@id": "schema:additionalType", "@type": "@id"},
+                "rdf_type**": {"@id": "owl:sameAs", "@type": "@id"},
+                "rdf_type***": {"@id": "Property:Equivalent_URI", "@type": "@id"},
+                "short_name": {"@id": "Property:HasShortName"},
+                "keywords": {"@id": "schema:keywords", "@type": "@id"},
+                "keywords*": {"@id": "Property:HasKeyword", "@type": "@id"},
+                "statements": {"@id": "Property:HasStatement", "@type": "@id"},
+                "text": {"@id": "@value"},
+                "uuid*": {"@id": "Property:HasUuid"},
+            },
+            "uuid": "ce353767-c628-45bd-9d88-d6eb3009aec0",
+            "title": "Entity",
+            "defaultProperties": ["description"],
+        }
+
     rdf_type: Optional[Set[str]] = Field(None, title="Additional RDF type(s)")
     """
     Declares additional type(s) for this entity, e.g., to state that this entity has the same meaning as a term in a controlled vocabulary or ontology. This property is synonymous to the schema:additionalType and owl:sameAs. The default syntax is ontology:TermName. The ontology prefix has to be defined in the @context of the Entity, the category or any of the parent categories. The term name has to be a valid identifier in the ontology.
@@ -83,9 +197,11 @@ class Entity(OswBaseModel):
     query_label: Optional[str] = Field(None, title="Query label")
     description: Optional[List[Description]] = Field(None, title="Description")
     image: Optional[str] = Field(None, title="Image")
-    ordering_categories: Optional[List[str]] = Field(None, title="Ordering categories")
+    ordering_categories: Optional[List[str]] = Field(
+        None, title="Classification categories"
+    )
     """
-    Ordering categories are used to categorize instances, e.g., according to their use but not their properties. When querying for instances of a here listed ordering category, this instance will be returned. Note: Ordering categories define no properties, while 'regular' categories define properties, which an instance assigns values to.
+    Classification categories are used to categorize instances, e.g., according to their use but not their properties. When querying for instances of a here listed classification category, this instance will be returned. Note: Classification categories define no properties, while 'regular' categories define properties, which an instance assigns values to.
     """
     keywords: Optional[List[str]] = Field(None, title="Keywords / Tags")
     """
@@ -103,6 +219,9 @@ class Entity(OswBaseModel):
 
 
 class ObjectStatement(OswBaseModel):
+    class Config:
+        schema_extra = {"title": "Object Statement"}
+
     rdf_type: Optional[Any] = "rdf:Statement"
     uuid: UUID = Field(default_factory=uuid4, title="UUID")
     label: Optional[List[Label]] = Field(None, title="Label")
@@ -118,6 +237,9 @@ class ObjectStatement(OswBaseModel):
 
 
 class DataStatement(OswBaseModel):
+    class Config:
+        schema_extra = {"title": "Data Statement"}
+
     rdf_type: Optional[Any] = "rdf:Statement"
     uuid: UUID = Field(default_factory=uuid4, title="UUID")
     label: Optional[List[Label]] = Field(None, title="Label")
@@ -133,6 +255,9 @@ class DataStatement(OswBaseModel):
 
 
 class QuantityStatement(OswBaseModel):
+    class Config:
+        schema_extra = {"title": "Quantity Statement"}
+
     rdf_type: Optional[Any] = "rdf:Statement"
     uuid: UUID = Field(default_factory=uuid4, title="UUID")
     label: Optional[List[Label]] = Field(None, title="Label")
@@ -151,6 +276,18 @@ class QuantityStatement(OswBaseModel):
 
 
 class Item(Entity):
+    class Config:
+        schema_extra = {
+            "@context": [
+                "/wiki/Category:Entity?action=raw&slot=jsonschema",
+                {
+                    "type": {"@id": "Property:HasType", "@type": "@id"},
+                    "type*": {"@id": "Property:HasSchema", "@type": "@id"},
+                },
+            ],
+            "title": "Item",
+        }
+
     type: Optional[List[str]] = Field(
         ["Category:Item"], min_items=1, title="Types/Categories"
     )

src/osw/utils/codegen.py

@@ -0,0 +1,59 @@
+from pathlib import Path
+from typing import Any, Dict
+
+from datamodel_code_generator import load_yaml_from_path
+from datamodel_code_generator.model import pydantic as pydantic_v1_model
+from datamodel_code_generator.model import pydantic_v2 as pydantic_v2_model
+from datamodel_code_generator.parser.jsonschema import (
+    JsonSchemaObject,
+    JsonSchemaParser,
+)
+
+# https://docs.pydantic.dev/1.10/usage/schema/#schema-customization
+# https://docs.pydantic.dev/latest/concepts/json_schema/#using-json_schema_extra-with-a-dict
+# https://docs.pydantic.dev/latest/concepts/json_schema/#field-level-customization
+
+
+class PydanticV1Config(pydantic_v1_model.Config):
+    # schema_extra: Optional[Dict[str, Any]] = None
+    schema_extra: str = None
+
+
+class PydanticV2Config(pydantic_v2_model.ConfigDict):
+    # schema_extra: Optional[Dict[str, Any]] = None
+    json_schema_extra: str = None
+
+
+class OOLDJsonSchemaParser(JsonSchemaParser):
+    """Custom parser for OO-LD schemas.
+    You can use this class directly or monkey-patch the datamodel_code_generator module:
+    `datamodel_code_generator.parser.jsonschema.JsonSchemaParser = OOLDJsonSchemaParser`
+    """
+
+    def set_additional_properties(self, name: str, obj: JsonSchemaObject) -> None:
+        schema_extras = repr(obj.extras)  # keeps 'False' and 'True' boolean literals
+        if self.data_model_type == pydantic_v1_model.BaseModel:
+            self.extra_template_data[name]["config"] = PydanticV1Config(
+                schema_extra=schema_extras
+            )
+        if self.data_model_type == pydantic_v2_model.BaseModel:
+            self.extra_template_data[name]["config"] = PydanticV2Config(
+                json_schema_extra=schema_extras
+            )
+        return super().set_additional_properties(name, obj)
+
+
+class OOLDJsonSchemaParserFixedRefs(OOLDJsonSchemaParser):
+    """Overwrite # overwrite the original `_get_ref_body_from_remote` function
+    to fix wrongly composed paths. This issue occurs only when using this parser class directy
+    and occurs not if used through mokey patching and `datamodel_code_generator.generate()`
+    """
+
+    def _get_ref_body_from_remote(self, resolved_ref: str) -> Dict[Any, Any]:
+        # full_path = self.base_path / resolved_ref
+        # fix: merge the paths correctly
+        full_path = self.base_path / Path(resolved_ref).parts[-1]
+        return self.remote_object_cache.get_or_put(
+            str(full_path),
+            default_factory=lambda _: load_yaml_from_path(full_path, self.encoding),
+        )