diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb6c8f5d..da2009ef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Additions +- Add support of properties indirect reference ($ref) resolution; Add support of inner properties direct and indirect reference resolution to its owner model/enum (#329). Thanks @p1-ra! - New `--meta` command line option for specifying what type of metadata should be generated: - `poetry` is the default value, same behavior you're used to in previous versions - `setup` will generate a pyproject.toml with no Poetry information, and instead create a `setup.py` with the diff --git a/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py b/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py index 9237d2428..97ae9d56d 100644 --- a/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py +++ b/end_to_end_tests/golden-record-custom/custom_e2e/models/a_model.py @@ -27,6 +27,9 @@ class AModel: a_nullable_date: Optional[datetime.date] required_nullable: Optional[str] nullable_model: Optional[AModelNullableModel] + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + direct_ref_to_itself: Union["AModel", Unset] = UNSET + indirect_ref_to_itself: Union["AModel", Unset] = UNSET nested_list_of_enums: Union[Unset, List[List[DifferentEnum]]] = UNSET a_not_required_date: Union[Unset, datetime.date] = UNSET attr_1_leading_digit: Union[Unset, str] = UNSET @@ -48,6 +51,18 @@ def to_dict(self) -> Dict[str, Any]: required_not_nullable = self.required_not_nullable model = self.model.to_dict() + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + if not isinstance(self.an_enum_indirect_ref, Unset): + an_enum_indirect_ref = self.an_enum_indirect_ref + + direct_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.direct_ref_to_itself, Unset): + direct_ref_to_itself = self.direct_ref_to_itself.to_dict() + + indirect_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.indirect_ref_to_itself, Unset): + indirect_ref_to_itself = self.indirect_ref_to_itself.to_dict() + nested_list_of_enums: Union[Unset, List[Any]] = UNSET if not isinstance(self.nested_list_of_enums, Unset): nested_list_of_enums = [] @@ -94,6 +109,12 @@ def to_dict(self) -> Dict[str, Any]: "nullable_model": nullable_model, } ) + if an_enum_indirect_ref is not UNSET: + field_dict["an_enum_indirect_ref"] = an_enum_indirect_ref + if direct_ref_to_itself is not UNSET: + field_dict["direct_ref_to_itself"] = direct_ref_to_itself + if indirect_ref_to_itself is not UNSET: + field_dict["indirect_ref_to_itself"] = indirect_ref_to_itself if nested_list_of_enums is not UNSET: field_dict["nested_list_of_enums"] = nested_list_of_enums if a_not_required_date is not UNSET: @@ -137,6 +158,21 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat model = AModelModel.from_dict(d.pop("model")) + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + _an_enum_indirect_ref = d.pop("an_enum_indirect_ref", UNSET) + if not isinstance(_an_enum_indirect_ref, Unset): + an_enum_indirect_ref = AnEnum(_an_enum_indirect_ref) + + direct_ref_to_itself: Union[AModel, Unset] = UNSET + _direct_ref_to_itself = d.pop("direct_ref_to_itself", UNSET) + if not isinstance(_direct_ref_to_itself, Unset): + direct_ref_to_itself = AModel.from_dict(_direct_ref_to_itself) + + indirect_ref_to_itself: Union[AModel, Unset] = UNSET + _indirect_ref_to_itself = d.pop("indirect_ref_to_itself", UNSET) + if not isinstance(_indirect_ref_to_itself, Unset): + indirect_ref_to_itself = AModel.from_dict(_indirect_ref_to_itself) + nested_list_of_enums = [] _nested_list_of_enums = d.pop("nested_list_of_enums", UNSET) for nested_list_of_enums_item_data in _nested_list_of_enums or []: @@ -188,6 +224,9 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat a_date=a_date, required_not_nullable=required_not_nullable, model=model, + an_enum_indirect_ref=an_enum_indirect_ref, + direct_ref_to_itself=direct_ref_to_itself, + indirect_ref_to_itself=indirect_ref_to_itself, nested_list_of_enums=nested_list_of_enums, a_nullable_date=a_nullable_date, a_not_required_date=a_not_required_date, diff --git a/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py b/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py index 9237d2428..97ae9d56d 100644 --- a/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py +++ b/end_to_end_tests/golden-record/my_test_api_client/models/a_model.py @@ -27,6 +27,9 @@ class AModel: a_nullable_date: Optional[datetime.date] required_nullable: Optional[str] nullable_model: Optional[AModelNullableModel] + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + direct_ref_to_itself: Union["AModel", Unset] = UNSET + indirect_ref_to_itself: Union["AModel", Unset] = UNSET nested_list_of_enums: Union[Unset, List[List[DifferentEnum]]] = UNSET a_not_required_date: Union[Unset, datetime.date] = UNSET attr_1_leading_digit: Union[Unset, str] = UNSET @@ -48,6 +51,18 @@ def to_dict(self) -> Dict[str, Any]: required_not_nullable = self.required_not_nullable model = self.model.to_dict() + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + if not isinstance(self.an_enum_indirect_ref, Unset): + an_enum_indirect_ref = self.an_enum_indirect_ref + + direct_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.direct_ref_to_itself, Unset): + direct_ref_to_itself = self.direct_ref_to_itself.to_dict() + + indirect_ref_to_itself: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.indirect_ref_to_itself, Unset): + indirect_ref_to_itself = self.indirect_ref_to_itself.to_dict() + nested_list_of_enums: Union[Unset, List[Any]] = UNSET if not isinstance(self.nested_list_of_enums, Unset): nested_list_of_enums = [] @@ -94,6 +109,12 @@ def to_dict(self) -> Dict[str, Any]: "nullable_model": nullable_model, } ) + if an_enum_indirect_ref is not UNSET: + field_dict["an_enum_indirect_ref"] = an_enum_indirect_ref + if direct_ref_to_itself is not UNSET: + field_dict["direct_ref_to_itself"] = direct_ref_to_itself + if indirect_ref_to_itself is not UNSET: + field_dict["indirect_ref_to_itself"] = indirect_ref_to_itself if nested_list_of_enums is not UNSET: field_dict["nested_list_of_enums"] = nested_list_of_enums if a_not_required_date is not UNSET: @@ -137,6 +158,21 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat model = AModelModel.from_dict(d.pop("model")) + an_enum_indirect_ref: Union[Unset, AnEnum] = UNSET + _an_enum_indirect_ref = d.pop("an_enum_indirect_ref", UNSET) + if not isinstance(_an_enum_indirect_ref, Unset): + an_enum_indirect_ref = AnEnum(_an_enum_indirect_ref) + + direct_ref_to_itself: Union[AModel, Unset] = UNSET + _direct_ref_to_itself = d.pop("direct_ref_to_itself", UNSET) + if not isinstance(_direct_ref_to_itself, Unset): + direct_ref_to_itself = AModel.from_dict(_direct_ref_to_itself) + + indirect_ref_to_itself: Union[AModel, Unset] = UNSET + _indirect_ref_to_itself = d.pop("indirect_ref_to_itself", UNSET) + if not isinstance(_indirect_ref_to_itself, Unset): + indirect_ref_to_itself = AModel.from_dict(_indirect_ref_to_itself) + nested_list_of_enums = [] _nested_list_of_enums = d.pop("nested_list_of_enums", UNSET) for nested_list_of_enums_item_data in _nested_list_of_enums or []: @@ -188,6 +224,9 @@ def _parse_a_camel_date_time(data: Any) -> Union[datetime.datetime, datetime.dat a_date=a_date, required_not_nullable=required_not_nullable, model=model, + an_enum_indirect_ref=an_enum_indirect_ref, + direct_ref_to_itself=direct_ref_to_itself, + indirect_ref_to_itself=indirect_ref_to_itself, nested_list_of_enums=nested_list_of_enums, a_nullable_date=a_nullable_date, a_not_required_date=a_not_required_date, diff --git a/end_to_end_tests/openapi.json b/end_to_end_tests/openapi.json index fcd83e460..db4ae351a 100644 --- a/end_to_end_tests/openapi.json +++ b/end_to_end_tests/openapi.json @@ -675,6 +675,15 @@ "required": ["an_enum_value", "aCamelDateTime", "a_date", "a_nullable_date", "required_nullable", "required_not_nullable", "model", "nullable_model"], "type": "object", "properties": { + "an_enum_indirect_ref": { + "$ref": "#/components/schemas/AnEnumDeeperIndirectReference" + }, + "direct_ref_to_itself": { + "$ref": "#/components/schemas/AModel" + }, + "indirect_ref_to_itself": { + "$ref": "#/components/schemas/AModelDeeperIndirectReference" + }, "an_enum_value": { "$ref": "#/components/schemas/AnEnum" }, @@ -716,7 +725,7 @@ "a_not_required_date": { "title": "A Nullable Date", "type": "string", - "format": "date", + "format": "date" }, "1_leading_digit": { "title": "Leading Digit", @@ -782,11 +791,23 @@ "description": "A Model for testing all the ways custom objects can be used ", "additionalProperties": false }, + "AModelIndirectReference": { + "$ref": "#/components/schemas/AModel" + }, + "AModelDeeperIndirectReference": { + "$ref": "#/components/schemas/AModelIndirectReference" + }, + "AnEnumIndirectReference": { + "$ref": "#/components/schemas/AnEnum" + }, "AnEnum": { "title": "AnEnum", "enum": ["FIRST_VALUE", "SECOND_VALUE"], "description": "For testing Enums in all the ways they can be used " }, + "AnEnumDeeperIndirectReference": { + "$ref": "#/components/schemas/AnEnumIndirectReference" + }, "AnIntEnum": { "title": "AnIntEnum", "enum": [-1, 1, 2], diff --git a/openapi_python_client/parser/properties/__init__.py b/openapi_python_client/parser/properties/__init__.py index 992d726df..18304058d 100644 --- a/openapi_python_client/parser/properties/__init__.py +++ b/openapi_python_client/parser/properties/__init__.py @@ -1,5 +1,6 @@ +import copy from itertools import chain -from typing import Any, ClassVar, Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar, Union +from typing import Any, ClassVar, Dict, Generic, Iterable, Iterator, List, Optional, Set, Tuple, TypeVar, Union, cast import attr @@ -14,6 +15,92 @@ from .schemas import Schemas +class LazyReferencePropertyProxy: + + __GLOBAL_SCHEMAS_REF: Schemas = Schemas() + __PROXIES: List[Tuple["LazyReferencePropertyProxy", oai.Reference]] = [] + + @classmethod + def update_schemas(cls, schemas: Schemas) -> None: + cls.__GLOBAL_SCHEMAS_REF = schemas + + @classmethod + def create(cls, name: str, required: bool, data: oai.Reference, parent_name: str) -> "LazyReferencePropertyProxy": + proxy = LazyReferencePropertyProxy(name, required, data, parent_name) + cls.__PROXIES.append((proxy, data)) + return proxy + + @classmethod + def created_proxies(cls) -> List[Tuple["LazyReferencePropertyProxy", oai.Reference]]: + return cls.__PROXIES + + @classmethod + def flush_internal_references(cls) -> None: + cls.__PROXIES = [] + cls.__GLOBAL_SCHEMAS_REF = Schemas() + + def __init__(self, name: str, required: bool, data: oai.Reference, parent_name: str): + self._name = name + self._required = required + self._data = data + self._parent_name = parent_name + self._reference: Reference = Reference.from_ref(data.ref) + self._reference_to_itself: bool = self._reference.class_name == parent_name + self._resolved: Union[Property, None] = None + + def get_instance_type_string(self) -> str: + return self.get_type_string(no_optional=True) + + def get_type_string(self, no_optional: bool = False) -> str: + resolved = self.resolve() + if resolved: + return resolved.get_type_string(no_optional) + return "LazyReferencePropertyProxy" + + def get_imports(self, *, prefix: str) -> Set[str]: + resolved = self.resolve() + if resolved: + return resolved.get_imports(prefix=prefix) + return set() + + def to_string(self) -> str: + resolved = cast(Property, self.resolve(False)) + p_repr = resolved.to_string() + return p_repr.replace(f"{self._parent_name}", f"'{self._parent_name}'") + + def __copy__(self) -> Property: + resolved = cast(Property, self.resolve(False)) + return copy.copy(resolved) + + def __deepcopy__(self, memo: Any) -> Property: + resolved = cast(Property, self.resolve(False)) + return copy.deepcopy(resolved, memo) + + def __getattr__(self, name: str) -> Any: + if name == "nullable": + return False + elif name == "required": + return self._required + else: + resolved = self.resolve(False) + return resolved.__getattribute__(name) + + def resolve(self, allow_lazyness: bool = True) -> Union[Property, None]: + if not self._resolved: + schemas = LazyReferencePropertyProxy.__GLOBAL_SCHEMAS_REF + class_name = self._reference.class_name + existing = schemas.enums.get(class_name) or schemas.models.get(class_name) + if existing: + self._resolved = attr.evolve(existing, required=self._required, name=self._name) + + if self._resolved: + return self._resolved + elif allow_lazyness: + return None + else: + raise RuntimeError(f"Reference {self._data} shall have been resolved.") + + @attr.s(auto_attribs=True, frozen=True) class NoneProperty(Property): """ A property that is always None (used for empty schemas) """ @@ -235,7 +322,13 @@ def _string_based_property( def build_model_property( - *, data: oai.Schema, name: str, schemas: Schemas, required: bool, parent_name: Optional[str] + *, + data: oai.Schema, + name: str, + schemas: Schemas, + required: bool, + parent_name: Optional[str], + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[ModelProperty, PropertyError], Schemas]: """ A single ModelProperty from its OAI data @@ -259,7 +352,12 @@ def build_model_property( for key, value in (data.properties or {}).items(): prop_required = key in required_set prop, schemas = property_from_data( - name=key, required=prop_required, data=value, schemas=schemas, parent_name=class_name + name=key, + required=prop_required, + data=value, + schemas=schemas, + parent_name=class_name, + lazy_references=lazy_references, ) if isinstance(prop, PropertyError): return prop, schemas @@ -285,6 +383,7 @@ def build_model_property( data=data.additionalProperties, schemas=schemas, parent_name=class_name, + lazy_references=lazy_references, ) if isinstance(additional_properties, PropertyError): return additional_properties, schemas @@ -385,12 +484,23 @@ def build_enum_property( def build_union_property( - *, data: oai.Schema, name: str, required: bool, schemas: Schemas, parent_name: str + *, + data: oai.Schema, + name: str, + required: bool, + schemas: Schemas, + parent_name: str, + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[UnionProperty, PropertyError], Schemas]: sub_properties: List[Property] = [] for sub_prop_data in chain(data.anyOf, data.oneOf): sub_prop, schemas = property_from_data( - name=name, required=required, data=sub_prop_data, schemas=schemas, parent_name=parent_name + name=name, + required=required, + data=sub_prop_data, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, ) if isinstance(sub_prop, PropertyError): return PropertyError(detail=f"Invalid property in union {name}", data=sub_prop_data), schemas @@ -410,12 +520,23 @@ def build_union_property( def build_list_property( - *, data: oai.Schema, name: str, required: bool, schemas: Schemas, parent_name: str + *, + data: oai.Schema, + name: str, + required: bool, + schemas: Schemas, + parent_name: str, + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[ListProperty[Any], PropertyError], Schemas]: if data.items is None: return PropertyError(data=data, detail="type array must have items defined"), schemas inner_prop, schemas = property_from_data( - name=f"{name}_item", required=True, data=data.items, schemas=schemas, parent_name=parent_name + name=f"{name}_item", + required=True, + data=data.items, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, ) if isinstance(inner_prop, PropertyError): return PropertyError(data=inner_prop.data, detail=f"invalid data in items of array {name}"), schemas @@ -437,10 +558,14 @@ def _property_from_data( data: Union[oai.Reference, oai.Schema], schemas: Schemas, parent_name: str, + lazy_references: Dict[str, oai.Reference], ) -> Tuple[Union[Property, PropertyError], Schemas]: """ Generate a Property from the OpenAPI dictionary representation of it """ name = utils.remove_string_escapes(name) if isinstance(data, oai.Reference): + if not _is_local_reference(data): + return PropertyError(data=data, detail="Remote reference schemas are not supported."), schemas + reference = Reference.from_ref(data.ref) existing = schemas.enums.get(reference.class_name) or schemas.models.get(reference.class_name) if existing: @@ -448,13 +573,47 @@ def _property_from_data( attr.evolve(existing, required=required, name=name), schemas, ) - return PropertyError(data=data, detail="Could not find reference in parsed models or enums"), schemas + else: + + def lookup_is_reference_to_itself( + ref_name: str, + owner_class_name: str, + lazy_references: Dict[str, oai.Reference], + ) -> bool: + if ref_name in lazy_references: + next_ref_name = _reference_pointer_name(lazy_references[ref_name]) + return lookup_is_reference_to_itself( + next_ref_name, + owner_class_name, + lazy_references, + ) + + return ref_name.casefold() == owner_class_name.casefold() + + reference_name = _reference_pointer_name(data) + if lookup_is_reference_to_itself(reference_name, parent_name, lazy_references): + return cast(Property, LazyReferencePropertyProxy.create(name, required, data, parent_name)), schemas + else: + return PropertyError(data=data, detail="Could not find reference in parsed models or enums."), schemas + if data.enum: return build_enum_property( - data=data, name=name, required=required, schemas=schemas, enum=data.enum, parent_name=parent_name + data=data, + name=name, + required=required, + schemas=schemas, + enum=data.enum, + parent_name=parent_name, ) if data.anyOf or data.oneOf: - return build_union_property(data=data, name=name, required=required, schemas=schemas, parent_name=parent_name) + return build_union_property( + data=data, + name=name, + required=required, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, + ) if not data.type: return NoneProperty(name=name, required=required, nullable=False, default=None), schemas @@ -491,9 +650,23 @@ def _property_from_data( schemas, ) elif data.type == "array": - return build_list_property(data=data, name=name, required=required, schemas=schemas, parent_name=parent_name) + return build_list_property( + data=data, + name=name, + required=required, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, + ) elif data.type == "object": - return build_model_property(data=data, name=name, schemas=schemas, required=required, parent_name=parent_name) + return build_model_property( + data=data, + name=name, + schemas=schemas, + required=required, + parent_name=parent_name, + lazy_references=lazy_references, + ) return PropertyError(data=data, detail=f"unknown type {data.type}"), schemas @@ -504,52 +677,182 @@ def property_from_data( data: Union[oai.Reference, oai.Schema], schemas: Schemas, parent_name: str, + lazy_references: Optional[Dict[str, oai.Reference]] = None, ) -> Tuple[Union[Property, PropertyError], Schemas]: + if lazy_references is None: + lazy_references = dict() + try: - return _property_from_data(name=name, required=required, data=data, schemas=schemas, parent_name=parent_name) + return _property_from_data( + name=name, + required=required, + data=data, + schemas=schemas, + parent_name=parent_name, + lazy_references=lazy_references, + ) except ValidationError: return PropertyError(detail="Failed to validate default value", data=data), schemas -def update_schemas_with_data(name: str, data: oai.Schema, schemas: Schemas) -> Union[Schemas, PropertyError]: +def update_schemas_with_data( + name: str, data: oai.Schema, schemas: Schemas, lazy_references: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: prop: Union[PropertyError, ModelProperty, EnumProperty] if data.enum is not None: prop, schemas = build_enum_property( - data=data, name=name, required=True, schemas=schemas, enum=data.enum, parent_name=None + data=data, + name=name, + required=True, + schemas=schemas, + enum=data.enum, + parent_name=None, ) else: - prop, schemas = build_model_property(data=data, name=name, schemas=schemas, required=True, parent_name=None) + prop, schemas = build_model_property( + data=data, name=name, schemas=schemas, required=True, parent_name=None, lazy_references=lazy_references + ) + if isinstance(prop, PropertyError): return prop else: return schemas +def resolve_reference_and_update_schemas( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: + if _is_local_reference(data): + return _resolve_local_reference_schema(name, data, schemas, references_by_name) + else: + return _resolve_remote_reference_schema(name, data, schemas, references_by_name) + + +def _resolve_local_reference_schema( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: + resolved_model_or_enum = _resolve_model_or_enum_reference(name, data, schemas, references_by_name) + + if resolved_model_or_enum: + model_name = utils.pascal_case(name) + + if isinstance(resolved_model_or_enum, EnumProperty): + schemas.enums[model_name] = resolved_model_or_enum + + elif isinstance(resolved_model_or_enum, ModelProperty): + schemas.models[model_name] = resolved_model_or_enum + + return schemas + else: + return PropertyError(data=data, detail="Failed to resolve local reference schemas.") + + +def _resolve_model_or_enum_reference( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[EnumProperty, ModelProperty, None]: + target_model = _reference_model_name(data) + target_name = _reference_pointer_name(data) + + if target_model == name or target_name == name: + return None # Avoid infinite loop + + if target_name in references_by_name: + return _resolve_model_or_enum_reference( + target_name, references_by_name[target_name], schemas, references_by_name + ) + + if target_model in schemas.enums: + return schemas.enums[target_model] + elif target_model in schemas.models: + return schemas.models[target_model] + + return None + + +def _resolve_remote_reference_schema( + name: str, data: oai.Reference, schemas: Schemas, references_by_name: Dict[str, oai.Reference] +) -> Union[Schemas, PropertyError]: + return PropertyError(data=data, detail="Remote reference schemas are not supported.") + + +def _is_local_reference(reference: oai.Reference) -> bool: + return reference.ref.startswith("#", 0) + + +def _reference_model_name(reference: oai.Reference) -> str: + return Reference.from_ref(reference.ref).class_name + + +def _reference_pointer_name(reference: oai.Reference) -> str: + parts = reference.ref.split("/") + return parts[-1] + + def build_schemas(*, components: Dict[str, Union[oai.Reference, oai.Schema]]) -> Schemas: """ Get a list of Schemas from an OpenAPI dict """ schemas = Schemas() to_process: Iterable[Tuple[str, Union[oai.Reference, oai.Schema]]] = components.items() processing = True errors: List[PropertyError] = [] + lazy_self_references: Dict[str, oai.Reference] = dict() + visited: List[str] = [] + references_by_name: Dict[str, oai.Reference] = dict() + references_to_process: List[Tuple[str, oai.Reference]] = list() + LazyReferencePropertyProxy.flush_internal_references() # Cleanup side effects # References could have forward References so keep going as long as we are making progress while processing: processing = False errors = [] next_round = [] + # Only accumulate errors from the last round, since we might fix some along the way for name, data in to_process: + visited.append(name) + if isinstance(data, oai.Reference): - schemas.errors.append(PropertyError(data=data, detail="Reference schemas are not supported.")) + references_by_name[name] = data + references_to_process.append((name, data)) continue - schemas_or_err = update_schemas_with_data(name, data, schemas) + + schemas_or_err = update_schemas_with_data(name, data, schemas, lazy_self_references) + if isinstance(schemas_or_err, PropertyError): next_round.append((name, data)) errors.append(schemas_or_err) else: schemas = schemas_or_err processing = True # We made some progress this round, do another after it's done + to_process = next_round + + for name, reference in references_to_process: + schemas_or_err = resolve_reference_and_update_schemas(name, reference, schemas, references_by_name) + + if isinstance(schemas_or_err, PropertyError): + if _reference_pointer_name(reference) in visited and name not in lazy_self_references: + # It's a reference to an already visited Enum|Model; not yet resolved + # It's an indirect reference toward this Enum|Model; + # It will be lazy proxified and resolved later on + lazy_self_references[name] = reference + processing = True + else: + errors.append(schemas_or_err) + schemas.errors.extend(errors) + for name in lazy_self_references.keys(): + schemas_or_err = resolve_reference_and_update_schemas( + name, lazy_self_references[name], schemas, references_by_name + ) + if isinstance(schemas_or_err, PropertyError): + schemas.errors.extend(errors) + + LazyReferencePropertyProxy.update_schemas(schemas) + for reference_proxy, data in LazyReferencePropertyProxy.created_proxies(): + if not reference_proxy.resolve(): + schemas.errors.append( + PropertyError(data=data, detail="Could not find reference in parsed models or enums.") + ) + return schemas diff --git a/tests/test_parser/test_properties/test_init.py b/tests/test_parser/test_properties/test_init.py index ee07d3973..2edc7fa17 100644 --- a/tests/test_parser/test_properties/test_init.py +++ b/tests/test_parser/test_properties/test_init.py @@ -548,7 +548,7 @@ def test_property_from_data_ref_enum(self): from openapi_python_client.parser.properties import EnumProperty, Reference, Schemas, property_from_data name = "some_enum" - data = oai.Reference.construct(ref="MyEnum") + data = oai.Reference.construct(ref="#MyEnum") existing_enum = EnumProperty( name="an_enum", required=True, @@ -579,7 +579,7 @@ def test_property_from_data_ref_model(self): name = "new_name" required = False class_name = "MyModel" - data = oai.Reference.construct(ref=class_name) + data = oai.Reference.construct(ref=f"#{class_name}") existing_model = ModelProperty( name="old_name", required=True, @@ -611,7 +611,12 @@ def test_property_from_data_ref_model(self): assert schemas == new_schemas def test_property_from_data_ref_not_found(self, mocker): - from openapi_python_client.parser.properties import PropertyError, Schemas, property_from_data + from openapi_python_client.parser.properties import ( + LazyReferencePropertyProxy, + PropertyError, + Schemas, + property_from_data, + ) name = mocker.MagicMock() required = mocker.MagicMock() @@ -621,11 +626,16 @@ def test_property_from_data_ref_not_found(self, mocker): schemas = Schemas() prop, new_schemas = property_from_data( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) from_ref.assert_called_once_with(data.ref) - assert prop == PropertyError(data=data, detail="Could not find reference in parsed models or enums") + assert prop == PropertyError(data=data, detail="Could not find reference in parsed models or enums.") assert schemas == new_schemas def test_property_from_data_string(self, mocker): @@ -698,7 +708,12 @@ def test_property_from_data_array(self, mocker): assert response == build_list_property.return_value build_list_property.assert_called_once_with( - data=data, name=name, required=required, schemas=schemas, parent_name="parent" + data=data, + name=name, + required=required, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_property_from_data_object(self, mocker): @@ -717,7 +732,12 @@ def test_property_from_data_object(self, mocker): assert response == build_model_property.return_value build_model_property.assert_called_once_with( - data=data, name=name, required=required, schemas=schemas, parent_name="parent" + data=data, + name=name, + required=required, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_property_from_data_union(self, mocker): @@ -739,7 +759,12 @@ def test_property_from_data_union(self, mocker): assert response == build_union_property.return_value build_union_property.assert_called_once_with( - data=data, name=name, required=required, schemas=schemas, parent_name="parent" + data=data, + name=name, + required=required, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_property_from_data_unsupported_type(self, mocker): @@ -794,7 +819,12 @@ def test_build_list_property_no_items(self, mocker): schemas = properties.Schemas() p, new_schemas = properties.build_list_property( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) assert p == PropertyError(data=data, detail="type array must have items defined") @@ -817,14 +847,24 @@ def test_build_list_property_invalid_items(self, mocker): ) p, new_schemas = properties.build_list_property( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) assert p == PropertyError(data="blah", detail=f"invalid data in items of array {name}") assert new_schemas == second_schemas assert schemas != new_schemas, "Schema was mutated" property_from_data.assert_called_once_with( - name=f"{name}_item", required=True, data=data.items, schemas=schemas, parent_name="parent" + name=f"{name}_item", + required=True, + data=data.items, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) def test_build_list_property(self, mocker): @@ -845,7 +885,12 @@ def test_build_list_property(self, mocker): mocker.patch("openapi_python_client.utils.to_valid_python_identifier", return_value=name) p, new_schemas = properties.build_list_property( - name=name, required=required, data=data, schemas=schemas, parent_name="parent" + name=name, + required=required, + data=data, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) assert isinstance(p, properties.ListProperty) @@ -853,7 +898,12 @@ def test_build_list_property(self, mocker): assert new_schemas == second_schemas assert schemas != new_schemas, "Schema was mutated" property_from_data.assert_called_once_with( - name=f"{name}_item", required=True, data=data.items, schemas=schemas, parent_name="parent" + name=f"{name}_item", + required=True, + data=data.items, + schemas=schemas, + parent_name="parent", + lazy_references=dict(), ) @@ -1009,10 +1059,18 @@ def test_build_schemas(mocker): build_model_property.assert_has_calls( [ - mocker.call(data=in_data["1"], name="1", schemas=Schemas(), required=True, parent_name=None), - mocker.call(data=in_data["2"], name="2", schemas=schemas_1, required=True, parent_name=None), - mocker.call(data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None), - mocker.call(data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None), + mocker.call( + data=in_data["1"], name="1", schemas=Schemas(), required=True, parent_name=None, lazy_references=dict() + ), + mocker.call( + data=in_data["2"], name="2", schemas=schemas_1, required=True, parent_name=None, lazy_references=dict() + ), + mocker.call( + data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None, lazy_references=dict() + ), + mocker.call( + data=in_data["3"], name="3", schemas=schemas_2, required=True, parent_name=None, lazy_references=dict() + ), ] ) # schemas_3 was the last to come back from build_model_property, but it should be ignored because it's an error @@ -1020,13 +1078,43 @@ def test_build_schemas(mocker): assert result.errors == [error] -def test_build_parse_error_on_reference(): +def test_build_parse_error_on_unknown_local_reference(): from openapi_python_client.parser.openapi import build_schemas - ref_schema = oai.Reference.construct() + ref_schema = oai.Reference.construct(ref="#/foobar") in_data = {"1": ref_schema} result = build_schemas(components=in_data) - assert result.errors[0] == PropertyError(data=ref_schema, detail="Reference schemas are not supported.") + assert result.errors[0] == PropertyError(data=ref_schema, detail="Failed to resolve local reference schemas.") + + +def test_build_parse_success_on_known_local_reference(mocker): + from openapi_python_client.parser.openapi import build_schemas + + build_model_property = mocker.patch(f"{MODULE_NAME}.build_model_property") + schemas = mocker.MagicMock() + build_enum_property = mocker.patch(f"{MODULE_NAME}.build_enum_property", return_value=(mocker.MagicMock(), schemas)) + in_data = {"1": oai.Reference.construct(ref="#/foobar"), "foobar": mocker.MagicMock(enum=["val1", "val2", "val3"])} + + result = build_schemas(components=in_data) + + assert len(result.errors) == 0 + assert result.enums["1"] == result.enums["foobar"] + + +def test_build_parse_error_on_remote_reference(): + from openapi_python_client.parser.openapi import build_schemas + + ref_schemas = [ + oai.Reference.construct(ref="http://foobar/../foobar.yaml#/foobar"), + oai.Reference.construct(ref="https://foobar/foobar.yaml#/foobar"), + oai.Reference.construct(ref="../foobar.yaml#/foobar"), + oai.Reference.construct(ref="foobar.yaml#/foobar"), + oai.Reference.construct(ref="//foobar#/foobar"), + ] + for ref_schema in ref_schemas: + in_data = {"1": ref_schema} + result = build_schemas(components=in_data) + assert result.errors[0] == PropertyError(data=ref_schema, detail="Remote reference schemas are not supported.") def test_build_enums(mocker): @@ -1078,6 +1166,7 @@ def test_build_model_property(additional_properties_schema, expected_additional_ schemas=schemas, required=True, parent_name="parent", + lazy_references=dict(), ) assert new_schemas != schemas @@ -1124,6 +1213,7 @@ def test_build_model_property_conflict(): schemas=schemas, required=True, parent_name=None, + lazy_references=dict(), ) assert new_schemas == schemas @@ -1141,11 +1231,7 @@ def test_build_model_property_bad_prop(): schemas = Schemas(models={"OtherModel": None}) err, new_schemas = build_model_property( - data=data, - name="prop", - schemas=schemas, - required=True, - parent_name=None, + data=data, name="prop", schemas=schemas, required=True, parent_name=None, lazy_references=dict() ) assert new_schemas == schemas @@ -1170,6 +1256,7 @@ def test_build_model_property_bad_additional_props(): schemas=schemas, required=True, parent_name=None, + lazy_references=dict(), ) assert new_schemas == schemas @@ -1216,3 +1303,376 @@ def test_build_enum_property_bad_default(): assert schemas == schemas assert err == PropertyError(detail="B is an invalid default for enum Existing", data=data) + + +def test__is_local_reference(): + from openapi_python_client.parser.properties import _is_local_reference + + data_set = [ + ("//foobar#foobar", False), + ("foobar#/foobar", False), + ("foobar.json", False), + ("foobar.yaml", False), + ("../foo/bar.json#/foobar", False), + ("#/foobar", True), + ("#/foo/bar", True), + ] + + for data, expected_result in data_set: + ref = oai.Reference.construct(ref=data) + assert _is_local_reference(ref) == expected_result + + +def test__reference_pointer_name(): + from openapi_python_client.parser.properties import _reference_pointer_name + + data_set = [ + ("#/foobar", "foobar"), + ("#/foo/bar", "bar"), + ] + + for data, expected_result in data_set: + ref = oai.Reference.construct(ref=data) + assert _reference_pointer_name(ref) == expected_result + + +def test__reference_model_name(): + from openapi_python_client.parser.properties import _reference_model_name + + data_set = [ + ("#/foobar", "Foobar"), + ("#/foo/bar", "Bar"), + ] + + for data, expected_result in data_set: + ref = oai.Reference.construct(ref=data) + assert _reference_model_name(ref) == expected_result + + +def test__resolve_model_or_enum_reference(mocker): + from openapi_python_client.parser.properties import _resolve_model_or_enum_reference + from openapi_python_client.parser.properties.schemas import Schemas + + references_by_name = { + "FooBarReferenceLoop": oai.Reference.construct(ref="#/foobar"), + "FooBarDeeperReferenceLoop": oai.Reference.construct(ref="#/FooBarReferenceLoop"), + "BarFooReferenceLoop": oai.Reference.construct(ref="#/barfoo"), + "BarFooDeeperReferenceLoop": oai.Reference.construct(ref="#/BarFooReferenceLoop"), + "InfiniteReferenceLoop": oai.Reference.construct(ref="#/InfiniteReferenceLoop"), + "UnknownReference": oai.Reference.construct(ref="#/unknown"), + } + schemas = Schemas(enums={"Foobar": 1}, models={"Barfoo": 2}) + + res_1 = _resolve_model_or_enum_reference( + "FooBarReferenceLoop", references_by_name["FooBarReferenceLoop"], schemas, references_by_name + ) + res_2 = _resolve_model_or_enum_reference( + "FooBarDeeperReferenceLoop", references_by_name["FooBarDeeperReferenceLoop"], schemas, references_by_name + ) + res_3 = _resolve_model_or_enum_reference( + "BarFooReferenceLoop", references_by_name["BarFooReferenceLoop"], schemas, references_by_name + ) + res_4 = _resolve_model_or_enum_reference( + "BarFooDeeperReferenceLoop", references_by_name["BarFooDeeperReferenceLoop"], schemas, references_by_name + ) + res_5 = _resolve_model_or_enum_reference( + "InfiniteReferenceLoop", references_by_name["InfiniteReferenceLoop"], schemas, references_by_name + ) + res_6 = _resolve_model_or_enum_reference( + "UnknownReference", references_by_name["UnknownReference"], schemas, references_by_name + ) + + assert res_1 == schemas.enums["Foobar"] + assert res_2 == schemas.enums["Foobar"] + assert res_3 == schemas.models["Barfoo"] + assert res_4 == schemas.models["Barfoo"] + assert res_5 == None + assert res_6 == None + + +def test__resolve_local_reference_schema(mocker): + from openapi_python_client.parser.properties import _resolve_local_reference_schema + from openapi_python_client.parser.properties.enum_property import EnumProperty + from openapi_python_client.parser.properties.model_property import ModelProperty + from openapi_python_client.parser.properties.schemas import Schemas + + references_by_name = { + "FooBarReferenceLoop": oai.Reference.construct(ref="#/foobar"), + "FooBarDeeperReferenceLoop": oai.Reference.construct(ref="#/FooBarReferenceLoop"), + "fooBarLowerCaseReferenceLoop": oai.Reference.construct(ref="#/foobar"), + "fooBarLowerCaseDeeperReferenceLoop": oai.Reference.construct(ref="#/fooBarLowerCaseReferenceLoop"), + "BarFooReferenceLoop": oai.Reference.construct(ref="#/barfoo"), + "BarFooDeeperReferenceLoop": oai.Reference.construct(ref="#/BarFooReferenceLoop"), + "InfiniteReferenceLoop": oai.Reference.construct(ref="#/InfiniteReferenceLoop"), + "UnknownReference": oai.Reference.construct(ref="#/unknown"), + } + schemas = Schemas( + enums={ + "Foobar": EnumProperty( + name="Foobar", + required=False, + nullable=True, + default="foobar", + values=["foobar"], + value_type="str", + reference="", + ) + }, + models={ + "Barfoo": ModelProperty( + name="Barfoo", + required=False, + nullable=True, + default="barfoo", + reference="", + required_properties=[], + optional_properties=[], + description="", + relative_imports=[], + additional_properties=[], + ) + }, + ) + + res_1 = _resolve_local_reference_schema( + "FooBarReferenceLoop", references_by_name["FooBarReferenceLoop"], schemas, references_by_name + ) + res_2 = _resolve_local_reference_schema( + "FooBarDeeperReferenceLoop", references_by_name["FooBarDeeperReferenceLoop"], schemas, references_by_name + ) + res_3 = _resolve_local_reference_schema( + "BarFooReferenceLoop", references_by_name["BarFooReferenceLoop"], schemas, references_by_name + ) + res_4 = _resolve_local_reference_schema( + "BarFooDeeperReferenceLoop", references_by_name["BarFooDeeperReferenceLoop"], schemas, references_by_name + ) + res_5 = _resolve_local_reference_schema( + "fooBarLowerCaseReferenceLoop", references_by_name["fooBarLowerCaseReferenceLoop"], schemas, references_by_name + ) + res_6 = _resolve_local_reference_schema( + "fooBarLowerCaseDeeperReferenceLoop", + references_by_name["fooBarLowerCaseDeeperReferenceLoop"], + schemas, + references_by_name, + ) + res_7 = _resolve_local_reference_schema( + "InfiniteReferenceLoop", references_by_name["InfiniteReferenceLoop"], schemas, references_by_name + ) + res_8 = _resolve_local_reference_schema( + "UnknownReference", references_by_name["UnknownReference"], schemas, references_by_name + ) + + assert res_1 == res_2 == res_3 == res_4 == res_5 == res_6 == schemas + assert schemas.enums["FooBarReferenceLoop"] == schemas.enums["Foobar"] + assert schemas.enums["FooBarDeeperReferenceLoop"] == schemas.enums["Foobar"] + assert schemas.models["BarFooReferenceLoop"] == schemas.models["Barfoo"] + assert schemas.models["BarFooDeeperReferenceLoop"] == schemas.models["Barfoo"] + assert schemas.enums["FooBarLowerCaseReferenceLoop"] == schemas.enums["Foobar"] + assert schemas.enums["FooBarLowerCaseDeeperReferenceLoop"] == schemas.enums["Foobar"] + assert isinstance(res_7, PropertyError) + assert isinstance(res_8, PropertyError) + + +def _base_api_data(): + return """ +--- +openapi: 3.0.2 +info: + title: test + description: test + version: 1.0.0 +paths: + /tests/: + get: + operationId: getTests + description: test + responses: + '200': + description: test + content: + application/json: + schema: + $ref: '#/components/schemas/fooBar' +""" + + +def test_lazy_proxy_reference_unresolved(): + import copy + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import LazyReferencePropertyProxy, Schemas + + LazyReferencePropertyProxy.update_schemas(Schemas()) + lazy_reference_proxy = LazyReferencePropertyProxy.create( + "childProperty", False, oai.Reference(ref="#/foobar"), "AModel" + ) + + assert lazy_reference_proxy.get_instance_type_string() == "LazyReferencePropertyProxy" + assert lazy_reference_proxy.get_type_string(no_optional=False) == "LazyReferencePropertyProxy" + assert lazy_reference_proxy.get_type_string(no_optional=True) == "LazyReferencePropertyProxy" + assert lazy_reference_proxy.get_imports(prefix="..") == set() + assert lazy_reference_proxy.resolve() == None + assert lazy_reference_proxy.required == False + assert lazy_reference_proxy.nullable == False + with pytest.raises(RuntimeError): + lazy_reference_proxy.resolve(False) + with pytest.raises(RuntimeError): + copy.copy(lazy_reference_proxy) + with pytest.raises(RuntimeError): + copy.deepcopy(lazy_reference_proxy) + with pytest.raises(RuntimeError): + lazy_reference_proxy.name + with pytest.raises(RuntimeError): + lazy_reference_proxy.default + with pytest.raises(RuntimeError): + lazy_reference_proxy.python_name + with pytest.raises(RuntimeError): + lazy_reference_proxy.template + + +def test_lazy_proxy_reference_resolved(): + import copy + + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import LazyReferencePropertyProxy, Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: number +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + schemas = build_schemas(components=openapi.components.schemas) + foobar = schemas.models.get("FooBar") + + LazyReferencePropertyProxy.update_schemas(schemas) + lazy_reference_proxy = LazyReferencePropertyProxy.create( + "childProperty", True, oai.Reference(ref="#/components/schemas/fooBar"), "AModel" + ) + + assert foobar + assert lazy_reference_proxy.get_instance_type_string() == foobar.get_instance_type_string() + assert lazy_reference_proxy.get_type_string(no_optional=False) == foobar.get_type_string(no_optional=False) + assert lazy_reference_proxy.get_type_string(no_optional=True) == foobar.get_type_string(no_optional=True) + assert lazy_reference_proxy.get_imports(prefix="..") == foobar.get_imports(prefix="..") + assert lazy_reference_proxy.name == "childProperty" and foobar.name == "fooBar" + assert lazy_reference_proxy.nullable == foobar.nullable + assert lazy_reference_proxy.default == foobar.default + assert lazy_reference_proxy.python_name == "child_property" and foobar.python_name == "foo_bar" + assert lazy_reference_proxy.template == foobar.template + try: + copy.copy(lazy_reference_proxy) + copy.deepcopy(lazy_reference_proxy) + except Exception as e: + pytest.fail(e) + + +def test_build_schemas_resolve_inner_property_remote_reference(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + items: + $ref: 'AnOtherDocument#/components/schemas/bar' +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + assert len(schemas.errors) == 1 + assert schemas.errors[0] == PropertyError( + data=oai.Reference(ref="AnOtherDocument#/components/schemas/bar"), + detail="invalid data in items of array childSettings", + ) + + +def test_build_schemas_lazy_resolve_inner_property_self_direct_reference(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + items: + $ref: '#/components/schemas/fooBar' +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + foo_bar = schemas.models.get("FooBar") + assert len(schemas.errors) == 0 + assert foo_bar + child_settings = foo_bar.optional_properties[0] + assert child_settings.inner_property.reference == foo_bar.reference + + +def test_build_schemas_lazy_resolve_known_inner_property_self_indirect_reference(): + import yaml + + import openapi_python_client.schema as oai + from openapi_python_client.parser.properties import Schemas, build_schemas + + data = yaml.safe_load( + f""" +{_base_api_data()} +components: + schemas: + fooBar: + type: object + properties: + childSettings: + type: array + description: test + items: + $ref: '#/components/schemas/FoobarSelfIndirectReference' + FoobarSelfIndirectReference: + $ref: '#/components/schemas/foobarSelfDeeperIndirectReference' + foobarSelfDeeperIndirectReference: + $ref: '#/components/schemas/fooBar' +""" + ) + openapi = oai.OpenAPI.parse_obj(data) + + schemas = build_schemas(components=openapi.components.schemas) + + assert len(schemas.errors) == 0 + foobar = schemas.models.get("FooBar") + foobar_indirect_ref = schemas.models.get("FoobarSelfIndirectReference") + foobar_deep_indirect_ref = schemas.models.get("FoobarSelfDeeperIndirectReference") + assert foobar is not None and foobar_indirect_ref is not None and foobar_deep_indirect_ref is not None + assert foobar == foobar_indirect_ref == foobar_deep_indirect_ref + + child_settings = foobar.optional_properties[0] + assert child_settings.inner_property.reference == foobar.reference