Skip to content

Commit 1aae4b7

Browse files
refactor code
1 parent b82d7a5 commit 1aae4b7

File tree

1 file changed

+13
-13
lines changed

1 file changed

+13
-13
lines changed

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -934,6 +934,17 @@ def create_concurrency_level(
934934
parameters={},
935935
)
936936

937+
@staticmethod
938+
def apply_stream_state_migrations(
939+
stream_state_migrations: List[Any], stream_state: MutableMapping[str, Any]
940+
) -> MutableMapping[str, Any]:
941+
if stream_state_migrations:
942+
for state_migration in stream_state_migrations:
943+
if state_migration.should_migrate(stream_state):
944+
# The state variable is expected to be mutable but the migrate method returns an immutable mapping.
945+
stream_state = dict(state_migration.migrate(stream_state))
946+
return stream_state
947+
937948
def create_concurrent_cursor_from_datetime_based_cursor(
938949
self,
939950
model_type: Type[BaseModel],
@@ -954,11 +965,7 @@ def create_concurrent_cursor_from_datetime_based_cursor(
954965
if "stream_state" not in kwargs
955966
else kwargs["stream_state"]
956967
)
957-
if stream_state_migrations:
958-
for state_migration in stream_state_migrations:
959-
if state_migration.should_migrate(stream_state):
960-
# The state variable is expected to be mutable but the migrate method returns an immutable mapping.
961-
stream_state = dict(state_migration.migrate(stream_state))
968+
stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
962969

963970
component_type = component_definition.get("type")
964971
if component_definition.get("type") != model_type.__name__:
@@ -1246,12 +1253,7 @@ def create_concurrent_cursor_from_perpartition_cursor(
12461253
stream_state_migrations=stream_state_migrations,
12471254
)
12481255
)
1249-
1250-
if stream_state_migrations:
1251-
for state_migration in stream_state_migrations:
1252-
if state_migration.should_migrate(stream_state):
1253-
# The state variable is expected to be mutable but the migrate method returns an immutable mapping.
1254-
stream_state = dict(state_migration.migrate(stream_state))
1256+
stream_state = self.apply_stream_state_migrations(stream_state_migrations, stream_state)
12551257

12561258
# Return the concurrent cursor and state converter
12571259
return ConcurrentPerPartitionCursor(
@@ -1711,15 +1713,13 @@ def _merge_stream_slicers(
17111713
config=config or {},
17121714
stream_state={},
17131715
partition_router=stream_slicer,
1714-
stream_state_migrations=model.state_migrations,
17151716
)
17161717
return self.create_concurrent_cursor_from_datetime_based_cursor( # type: ignore # This is a known issue that we are creating and returning a ConcurrentCursor which does not technically implement the (low-code) StreamSlicer. However, (low-code) StreamSlicer and ConcurrentCursor both implement StreamSlicer.stream_slices() which is the primary method needed for checkpointing
17171718
model_type=DatetimeBasedCursorModel,
17181719
component_definition=model.incremental_sync.__dict__,
17191720
stream_name=model.name or "",
17201721
stream_namespace=None,
17211722
config=config or {},
1722-
stream_state_migrations=model.state_migrations,
17231723
)
17241724

17251725
incremental_sync_model = model.incremental_sync

0 commit comments

Comments
 (0)