Merge branch 'main' into tolik0/concurrent-perpartitioncursor

Author: tolik0

.github/dependabot.yml

@@ -21,7 +21,7 @@ updates:
     commit-message:
       prefix: "ci(deps): "
     schedule:
-      interval: daily
+      interval: monthly
     labels:
       - ci
     groups:

.github/workflows/connector-tests.yml

@@ -88,10 +88,10 @@ jobs:
           #   cdk_extra: n/a
           # TODO: These are manifest connectors and won't work as expected until we
           #       add `--use-local-cdk` support for manifest connectors.
-          # - connector: source-the-guardian-api
-          #   cdk_extra: n/a
-          # - connector: source-pokeapi
-          #   cdk_extra: n/a
+          - connector: source-the-guardian-api
+            cdk_extra: n/a
+          - connector: source-pokeapi
+            cdk_extra: n/a
 
     name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
     permissions:
@@ -163,9 +163,10 @@ jobs:
           fi
           echo -e "\n[Download Job Output](${{steps.upload_job_output.outputs.artifact-url}})" >> $GITHUB_STEP_SUMMARY
           if [ "${success}" != "true" ]; then
-            echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. Check the logs for more details."
+            echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. "
             exit 1
           fi
+          echo "See the execution report for details: ${html_report_url}"
           echo "success=${success}" >> $GITHUB_OUTPUT
           echo "html_report_url=${html_report_url}" >> $GITHUB_OUTPUT
 

.github/workflows/publish_sdm_connector.yml

@@ -14,7 +14,7 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: "Version. The version to publish, ie 1.0.0 or 1.0.0-dev1. In most cases, you can leave this blank. If run from a release tag (recommended), the version number will be inferred from the git tag."
+        description: "Note that this workflow is intended for prereleases. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow from main or from a dev branch, please enter the desired version number here, for instance 1.2.3dev0 or 1.2.3rc1."
         required: false
       publish_to_pypi:
         description: "Publish to PyPI. If true, the workflow will publish to PyPI."
@@ -30,7 +30,7 @@ on:
         description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder."
         type: boolean
         required: true
-        default: true
+        default: false
 
 jobs:
   build:

.github/workflows/pypi_publish.yml

@@ -1,4 +1,4 @@
-FROM docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
+FROM docker.io/airbyte/python-connector-base:3.0.0@sha256:1a0845ff2b30eafa793c6eee4e8f4283c2e52e1bbd44eed6cb9e9abd5d34d844
 
 WORKDIR /airbyte/integration_code
 
@@ -23,6 +23,10 @@ RUN mkdir -p source_declarative_manifest \
 # Remove unnecessary build files
 RUN rm -rf dist/ pyproject.toml poetry.lock README.md
 
+# Set ownership of /airbyte to the non-root airbyte user and group (1000:1000)
+RUN chown -R 1000:1000 /airbyte
+
 # Set the entrypoint
 ENV AIRBYTE_ENTRYPOINT="python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
+USER airbyte

Dockerfile

@@ -30,7 +30,7 @@ def __new__(  # type: ignore[misc]
         try:
             selected_key = str(
                 dpath.get(
-                    config,  # type: ignore [arg-type]  # Dpath wants mutable mapping but doesn't need it.
+                    config,  # type: ignore[arg-type]  # Dpath wants mutable mapping but doesn't need it.
                     authenticator_selection_path,
                 )
             )

airbyte_cdk/sources/declarative/auth/selective_authenticator.py

@@ -59,8 +59,9 @@
 
 
 class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
-    # By default, we defer to a value of 1 which represents running a connector using the Concurrent CDK engine on only one thread.
-    SINGLE_THREADED_CONCURRENCY_LEVEL = 1
+    # By default, we defer to a value of 2. A value lower than than could cause a PartitionEnqueuer to be stuck in a state of deadlock
+    # because it has hit the limit of futures but not partition reader is consuming them.
+    _LOWEST_SAFE_CONCURRENCY_LEVEL = 2
 
     def __init__(
         self,
@@ -110,8 +111,8 @@ def __init__(
                 concurrency_level // 2, 1
             )  # Partition_generation iterates using range based on this value. If this is floored to zero we end up in a dead lock during start up
         else:
-            concurrency_level = self.SINGLE_THREADED_CONCURRENCY_LEVEL
-            initial_number_of_partitions_to_generate = self.SINGLE_THREADED_CONCURRENCY_LEVEL
+            concurrency_level = self._LOWEST_SAFE_CONCURRENCY_LEVEL
+            initial_number_of_partitions_to_generate = self._LOWEST_SAFE_CONCURRENCY_LEVEL // 2
 
         self._concurrent_source = ConcurrentSource.create(
             num_workers=concurrency_level,