Skip to content

Commit 6d6bbed

Browse files
committed
fix parameters propagation issues for concurrent cursors
1 parent 4475c57 commit 6d6bbed

File tree

1 file changed

+12
-0
lines changed

1 file changed

+12
-0
lines changed

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1278,6 +1278,12 @@ def create_concurrent_cursor_from_datetime_based_cursor(
12781278
f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead"
12791279
)
12801280

1281+
# FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases:
1282+
# * The ComponentDefinition comes from model.__dict__ in which case we have `parameters`
1283+
# * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters`
1284+
# We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory.
1285+
if "$parameters" not in component_definition and "parameters" in component_definition:
1286+
component_definition["$parameters"] = component_definition.get("parameters")
12811287
datetime_based_cursor_model = model_type.parse_obj(component_definition)
12821288

12831289
if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel):
@@ -1582,6 +1588,12 @@ def create_concurrent_cursor_from_perpartition_cursor(
15821588
f"Expected manifest component of type {model_type.__name__}, but received {component_type} instead"
15831589
)
15841590

1591+
# FIXME the interfaces of the concurrent cursor are kind of annoying as they take a `ComponentDefinition` instead of the actual model. This was done because the ConcurrentDeclarativeSource didn't have access to the models [here for example](https://github.com/airbytehq/airbyte-python-cdk/blob/f525803b3fec9329e4cc8478996a92bf884bfde9/airbyte_cdk/sources/declarative/concurrent_declarative_source.py#L354C54-L354C91). So now we have two cases:
1592+
# * The ComponentDefinition comes from model.__dict__ in which case we have `parameters`
1593+
# * The ComponentDefinition comes from the manifest as a dict in which case we have `$parameters`
1594+
# We should change those interfaces to use the model once we clean up the code in CDS at which point the parameter propagation should happen as part of the ModelToComponentFactory.
1595+
if "$parameters" not in component_definition and "parameters" in component_definition:
1596+
component_definition["$parameters"] = component_definition.get("parameters")
15851597
datetime_based_cursor_model = model_type.parse_obj(component_definition)
15861598

15871599
if not isinstance(datetime_based_cursor_model, DatetimeBasedCursorModel):

0 commit comments

Comments
 (0)