Skip to content

Commit c18f0d7

Browse files
aldogonzalez8aldogonzalez8
authored
Merge branch 'main' into aldogonzalez8/changes-for-custom-retriever
2 parents 513a533 + f8cb659 commit c18f0d7

File tree

62 files changed

+2326
-746
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+2326
-746
lines changed

‎.github/workflows/pypi_publish.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ name: Packaging and Publishing
1010
on:
1111
push:
1212
tags:
13-
- 'v*'
13+
- "v*"
1414
workflow_dispatch:
1515
inputs:
1616
version:
@@ -146,7 +146,7 @@ jobs:
146146
(github.event_name == 'push' &&
147147
startsWith(github.ref, 'refs/tags/v')
148148
) || github.event.inputs.publish_to_pypi == 'true'
149-
uses: pypa/[email protected].2
149+
uses: pypa/[email protected].3
150150

151151
publish_sdm:
152152
name: Publish SDM to DockerHub

‎airbyte_cdk/__init__.py‎

Lines changed: 93 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -48,27 +48,46 @@
4848
# Once those issues are resolved, the below can be sorted with isort.
4949
import dunamai as _dunamai
5050

51-
from .destinations import Destination
52-
from .models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type, FailureType, AirbyteStream, AdvancedAuth, DestinationSyncMode, ConnectorSpecification, OAuthConfigSpecification, OrchestratorType, ConfiguredAirbyteStream, SyncMode, AirbyteLogMessage, Level, AirbyteRecordMessage
53-
54-
from .sources import Source
55-
from .config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
51+
from .config_observation import (
52+
create_connector_config_control_message,
53+
emit_configuration_as_airbyte_control_message,
54+
)
5655
from .connector import BaseConnector, Connector
57-
58-
from .entrypoint import launch, AirbyteEntrypoint
59-
56+
from .destinations import Destination
57+
from .entrypoint import AirbyteEntrypoint, launch
6058
from .logger import AirbyteLogFormatter, init_logger
61-
from .sources import AbstractSource
59+
from .models import (
60+
AdvancedAuth,
61+
AirbyteConnectionStatus,
62+
AirbyteLogMessage,
63+
AirbyteMessage,
64+
AirbyteRecordMessage,
65+
AirbyteStream,
66+
ConfiguredAirbyteCatalog,
67+
ConfiguredAirbyteStream,
68+
ConnectorSpecification,
69+
DestinationSyncMode,
70+
FailureType,
71+
Level,
72+
OAuthConfigSpecification,
73+
OrchestratorType,
74+
Status,
75+
SyncMode,
76+
Type,
77+
)
78+
from .sources import AbstractSource, Source
6279
from .sources.concurrent_source.concurrent_source import ConcurrentSource
6380
from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
6481
from .sources.config import BaseConfig
65-
from .sources.types import Config, Record, StreamSlice
6682
from .sources.connector_state_manager import ConnectorStateManager
6783
from .sources.declarative.auth import DeclarativeOauth2Authenticator
68-
from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
69-
from .sources.declarative.auth.declarative_authenticator import NoAuth
84+
from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth
7085
from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
71-
from .sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator, ApiKeyAuthenticator
86+
from .sources.declarative.auth.token import (
87+
ApiKeyAuthenticator,
88+
BasicHttpAuthenticator,
89+
BearerAuthenticator,
90+
)
7291
from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
7392
from .sources.declarative.declarative_stream import DeclarativeStream
7493
from .sources.declarative.decoders import Decoder, JsonDecoder
@@ -77,48 +96,89 @@
7796
from .sources.declarative.extractors.record_extractor import RecordExtractor
7897
from .sources.declarative.extractors.record_filter import RecordFilter
7998
from .sources.declarative.incremental import DatetimeBasedCursor
80-
from .sources.declarative.interpolation import InterpolatedString, InterpolatedBoolean
99+
from .sources.declarative.interpolation import InterpolatedBoolean, InterpolatedString
81100
from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
82-
from .sources.declarative.migrations.legacy_to_per_partition_state_migration import LegacyToPerPartitionStateMigration
83-
84-
from .sources.declarative.partition_routers import CartesianProductStreamSlicer, SinglePartitionRouter, SubstreamPartitionRouter
101+
from .sources.declarative.migrations.legacy_to_per_partition_state_migration import (
102+
LegacyToPerPartitionStateMigration,
103+
)
104+
from .sources.declarative.partition_routers import (
105+
CartesianProductStreamSlicer,
106+
SinglePartitionRouter,
107+
SubstreamPartitionRouter,
108+
)
85109
from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
86-
from .sources.declarative.requesters import Requester, HttpRequester
87-
110+
from .sources.declarative.requesters import HttpRequester, Requester
88111
from .sources.declarative.requesters.error_handlers import BackoffStrategy
89112
from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
90-
from .sources.declarative.requesters.paginators.strategies import OffsetIncrement, CursorPaginationStrategy, PageIncrement, StopConditionPaginationStrategyDecorator
91-
113+
from .sources.declarative.requesters.paginators.strategies import (
114+
CursorPaginationStrategy,
115+
OffsetIncrement,
116+
PageIncrement,
117+
StopConditionPaginationStrategyDecorator,
118+
)
92119
from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
93-
94-
from .sources.declarative.requesters.request_options.default_request_options_provider import DefaultRequestOptionsProvider
95-
from .sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider
120+
from .sources.declarative.requesters.request_options.default_request_options_provider import (
121+
DefaultRequestOptionsProvider,
122+
)
123+
from .sources.declarative.requesters.request_options.interpolated_request_input_provider import (
124+
InterpolatedRequestInputProvider,
125+
)
96126
from .sources.declarative.requesters.requester import HttpMethod
97127
from .sources.declarative.retrievers import SimpleRetriever
98128
from .sources.declarative.schema import JsonFileSchemaLoader
99-
from .sources.declarative.transformations.add_fields import AddFields, AddedFieldDefinition
129+
from .sources.declarative.transformations.add_fields import AddedFieldDefinition, AddFields
100130
from .sources.declarative.transformations.transformation import RecordTransformation
101131
from .sources.declarative.types import FieldPointer
102132
from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
103133
from .sources.message import InMemoryMessageRepository, MessageRepository
104134
from .sources.source import TState
105135
from .sources.streams.availability_strategy import AvailabilityStrategy
106-
from .sources.streams.call_rate import AbstractAPIBudget, HttpAPIBudget, HttpRequestMatcher, MovingWindowCallRatePolicy, Rate, CachedLimiterSession, LimiterSession
136+
from .sources.streams.call_rate import (
137+
AbstractAPIBudget,
138+
CachedLimiterSession,
139+
HttpAPIBudget,
140+
HttpRequestMatcher,
141+
LimiterSession,
142+
MovingWindowCallRatePolicy,
143+
Rate,
144+
)
107145
from .sources.streams.checkpoint import Cursor as LegacyCursor
108146
from .sources.streams.checkpoint import ResumableFullRefreshCursor
109147
from .sources.streams.concurrent.adapters import StreamFacade
110-
from .sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, FinalStateCursor
111-
from .sources.streams.concurrent.cursor import Cursor
112-
from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter, IsoMillisConcurrentStreamStateConverter
113-
from .sources.streams.core import Stream, IncrementalMixin, package_name_from_class
148+
from .sources.streams.concurrent.cursor import (
149+
ConcurrentCursor,
150+
Cursor,
151+
CursorField,
152+
FinalStateCursor,
153+
)
154+
from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import (
155+
EpochValueConcurrentStreamStateConverter,
156+
IsoMillisConcurrentStreamStateConverter,
157+
)
158+
from .sources.streams.core import IncrementalMixin, Stream, package_name_from_class
114159
from .sources.streams.http import HttpStream, HttpSubStream
115160
from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
116-
from .sources.streams.http.exceptions import BaseBackoffException, DefaultBackoffException, UserDefinedBackoffException
161+
from .sources.streams.http.exceptions import (
162+
BaseBackoffException,
163+
DefaultBackoffException,
164+
UserDefinedBackoffException,
165+
)
117166
from .sources.streams.http.rate_limiting import default_backoff_handler
118-
from .sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator, SingleUseRefreshTokenOauth2Authenticator
167+
from .sources.streams.http.requests_native_auth import (
168+
Oauth2Authenticator,
169+
SingleUseRefreshTokenOauth2Authenticator,
170+
TokenAuthenticator,
171+
)
119172
from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
173+
from .sources.types import Config, Record, StreamSlice
120174
from .sources.utils import casing
121-
from .sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader, check_config_against_spec_or_exit, split_config, expand_refs
175+
from .sources.utils.schema_helpers import (
176+
InternalConfig,
177+
ResourceSchemaLoader,
178+
check_config_against_spec_or_exit,
179+
expand_refs,
180+
split_config,
181+
)
122182
from .sources.utils.transform import TransformConfig, TypeTransformer
123183
from .utils import AirbyteTracedException, is_cloud_environment
124184
from .utils.constants import ENV_REQUEST_CACHE_PATH
@@ -127,7 +187,6 @@
127187
from .utils.spec_schema_transformations import resolve_refs
128188
from .utils.stream_status_utils import as_airbyte_message
129189

130-
131190
__all__ = [
132191
# Availability strategy
133192
"AvailabilityStrategy",

‎airbyte_cdk/cli/source_declarative_manifest/__init__.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
from airbyte_cdk.cli.source_declarative_manifest._run import run
22

3-
43
__all__ = [
54
"run",
65
]

‎airbyte_cdk/models/__init__.py‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
# of airbyte-cdk rather than a standalone package.
88
from .airbyte_protocol import (
99
AdvancedAuth,
10-
AirbyteStateStats,
1110
AirbyteAnalyticsTraceMessage,
1211
AirbyteCatalog,
1312
AirbyteConnectionStatus,
@@ -22,13 +21,14 @@
2221
AirbyteRecordMessage,
2322
AirbyteStateBlob,
2423
AirbyteStateMessage,
24+
AirbyteStateStats,
2525
AirbyteStateType,
2626
AirbyteStream,
2727
AirbyteStreamState,
2828
AirbyteStreamStatus,
29-
AirbyteStreamStatusTraceMessage,
3029
AirbyteStreamStatusReason,
3130
AirbyteStreamStatusReasonType,
31+
AirbyteStreamStatusTraceMessage,
3232
AirbyteTraceMessage,
3333
AuthFlowType,
3434
ConfiguredAirbyteCatalog,
@@ -48,6 +48,14 @@
4848
TraceType,
4949
Type,
5050
)
51+
from .airbyte_protocol_serializers import (
52+
AirbyteMessageSerializer,
53+
AirbyteStateMessageSerializer,
54+
AirbyteStreamStateSerializer,
55+
ConfiguredAirbyteCatalogSerializer,
56+
ConfiguredAirbyteStreamSerializer,
57+
ConnectorSpecificationSerializer,
58+
)
5159
from .well_known_types import (
5260
BinaryData,
5361
Boolean,
@@ -61,12 +69,3 @@
6169
TimeWithoutTimezone,
6270
TimeWithTimezone,
6371
)
64-
65-
from .airbyte_protocol_serializers import (
66-
AirbyteStreamStateSerializer,
67-
AirbyteStateMessageSerializer,
68-
AirbyteMessageSerializer,
69-
ConfiguredAirbyteCatalogSerializer,
70-
ConfiguredAirbyteStreamSerializer,
71-
ConnectorSpecificationSerializer,
72-
)

‎airbyte_cdk/sources/declarative/async_job/job_orchestrator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
TypeVar,
2121
)
2222

23-
from airbyte_cdk import StreamSlice
2423
from airbyte_cdk.logger import lazy_log
2524
from airbyte_cdk.models import FailureType
2625
from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
@@ -31,6 +30,7 @@
3130
from airbyte_cdk.sources.declarative.async_job.repository import AsyncJobRepository
3231
from airbyte_cdk.sources.declarative.async_job.status import AsyncJobStatus
3332
from airbyte_cdk.sources.message import MessageRepository
33+
from airbyte_cdk.sources.types import StreamSlice
3434
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
3535
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
3636

‎airbyte_cdk/sources/declarative/auth/__init__.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,7 @@
22
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
33
#
44

5-
from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
65
from airbyte_cdk.sources.declarative.auth.jwt import JwtAuthenticator
6+
from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator
77

8-
__all__ = [
9-
"DeclarativeOauth2Authenticator",
10-
"JwtAuthenticator"
11-
]
8+
__all__ = ["DeclarativeOauth2Authenticator", "JwtAuthenticator"]

‎airbyte_cdk/sources/declarative/auth/oauth.py‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -43,28 +43,32 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
4343
message_repository (MessageRepository): the message repository used to emit logs on HTTP requests
4444
"""
4545

46-
token_refresh_endpoint: Union[InterpolatedString, str]
4746
client_id: Union[InterpolatedString, str]
4847
client_secret: Union[InterpolatedString, str]
4948
config: Mapping[str, Any]
5049
parameters: InitVar[Mapping[str, Any]]
50+
token_refresh_endpoint: Optional[Union[InterpolatedString, str]] = None
5151
refresh_token: Optional[Union[InterpolatedString, str]] = None
5252
scopes: Optional[List[str]] = None
5353
token_expiry_date: Optional[Union[InterpolatedString, str]] = None
5454
_token_expiry_date: Optional[pendulum.DateTime] = field(init=False, repr=False, default=None)
5555
token_expiry_date_format: Optional[str] = None
5656
token_expiry_is_time_of_expiration: bool = False
5757
access_token_name: Union[InterpolatedString, str] = "access_token"
58+
access_token_value: Optional[Union[InterpolatedString, str]] = None
5859
expires_in_name: Union[InterpolatedString, str] = "expires_in"
5960
refresh_request_body: Optional[Mapping[str, Any]] = None
6061
grant_type: Union[InterpolatedString, str] = "refresh_token"
6162
message_repository: MessageRepository = NoopMessageRepository()
6263

6364
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
6465
super().__init__()
65-
self._token_refresh_endpoint = InterpolatedString.create(
66-
self.token_refresh_endpoint, parameters=parameters
67-
)
66+
if self.token_refresh_endpoint is not None:
67+
self._token_refresh_endpoint: Optional[InterpolatedString] = InterpolatedString.create(
68+
self.token_refresh_endpoint, parameters=parameters
69+
)
70+
else:
71+
self._token_refresh_endpoint = None
6872
self._client_id = InterpolatedString.create(self.client_id, parameters=parameters)
6973
self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters)
7074
if self.refresh_token is not None:
@@ -92,20 +96,31 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
9296
if self.token_expiry_date
9397
else pendulum.now().subtract(days=1) # type: ignore # substract does not have type hints
9498
)
95-
self._access_token: Optional[str] = None # access_token is initialized by a setter
99+
if self.access_token_value is not None:
100+
self._access_token_value = InterpolatedString.create(
101+
self.access_token_value, parameters=parameters
102+
).eval(self.config)
103+
else:
104+
self._access_token_value = None
105+
106+
self._access_token: Optional[str] = (
107+
self._access_token_value if self.access_token_value else None
108+
)
96109

97110
if self.get_grant_type() == "refresh_token" and self._refresh_token is None:
98111
raise ValueError(
99112
"OAuthAuthenticator needs a refresh_token parameter if grant_type is set to `refresh_token`"
100113
)
101114

102-
def get_token_refresh_endpoint(self) -> str:
103-
refresh_token: str = self._token_refresh_endpoint.eval(self.config)
104-
if not refresh_token:
105-
raise ValueError(
106-
"OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter"
107-
)
108-
return refresh_token
115+
def get_token_refresh_endpoint(self) -> Optional[str]:
116+
if self._token_refresh_endpoint is not None:
117+
refresh_token_endpoint: str = self._token_refresh_endpoint.eval(self.config)
118+
if not refresh_token_endpoint:
119+
raise ValueError(
120+
"OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter"
121+
)
122+
return refresh_token_endpoint
123+
return None
109124

110125
def get_client_id(self) -> str:
111126
client_id: str = self._client_id.eval(self.config)

0 commit comments

Comments
 (0)