Skip to content

Commit 37efbae

Browse files
tolik0tolik0
committed
Merge branch 'main' into tolik0/concurrent-perpartitioncursor
2 parents d326a26 + cd1bd1c commit 37efbae

File tree

304 files changed

+4466
-1300
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

304 files changed

+4466
-1300
lines changed

‎.github/release-drafter.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ categories:
1515
- "chore"
1616
- "ci"
1717
- "refactor"
18+
- "testing"
1819
- title: "Documentation 📖"
1920
label: "docs"
2021
change-template: "- $TITLE (#$NUMBER)"
@@ -38,6 +39,9 @@ autolabeler:
3839
- label: "chore"
3940
title:
4041
- '/^chore(\(.*\))?\:/i'
42+
- label: "testing"
43+
title:
44+
- '/^tests(\(.*\))?\:/i'
4145
- label: "ci"
4246
title:
4347
- '/^ci(\(.*\))?\:/i'

‎.github/workflows/connector-tests.yml‎

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -74,29 +74,34 @@ jobs:
7474
cdk_extra: n/a
7575
- connector: source-chargebee
7676
cdk_extra: n/a
77-
# Currently not passing CI (unrelated)
78-
# - connector: source-zendesk-support
79-
# cdk_extra: n/a
8077
- connector: source-s3
8178
cdk_extra: file-based
8279
- connector: destination-pinecone
8380
cdk_extra: vector-db-based
8481
- connector: destination-motherduck
8582
cdk_extra: sql
83+
# ZenDesk currently failing (as of 2024-12-02)
84+
# TODO: Re-enable once fixed
85+
# - connector: source-zendesk-support
86+
# cdk_extra: n/a
8687
# TODO: These are manifest connectors and won't work as expected until we
8788
# add `--use-local-cdk` support for manifest connectors.
88-
- connector: source-the-guardian-api
89-
cdk_extra: n/a
90-
- connector: source-pokeapi
91-
cdk_extra: n/a
89+
# - connector: source-the-guardian-api
90+
# cdk_extra: n/a
91+
# - connector: source-pokeapi
92+
# cdk_extra: n/a
9293

9394
name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
9495
steps:
9596
- name: Abort if extra not changed (${{matrix.cdk_extra}})
9697
id: no_changes
97-
if: ${{ matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }}
98+
if: ${{ needs.cdk_changes.outputs['src'] == 'false' || matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }}
9899
run: |
99-
echo "Aborting job as specified extra not changed: ${{matrix.cdk_extra}} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}"
100+
echo "Aborting job."
101+
echo "Source code changed: ${{ needs.cdk_changes.outputs['src'] }}"
102+
if [ "${{ matrix.cdk_extra }}" != "n/a" ]; then
103+
echo "Extra not changed: ${{ matrix.cdk_extra }} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}"
104+
fi
100105
echo "> Skipped '${{matrix.connector}}' (no relevant changes)" >> $GITHUB_STEP_SUMMARY
101106
echo "status=cancelled" >> $GITHUB_OUTPUT
102107
exit 0
@@ -112,8 +117,7 @@ jobs:
112117
if: steps.no_changes.outputs.status != 'cancelled'
113118
with:
114119
repository: airbytehq/airbyte
115-
# TODO: Revert to `master` after Airbyte CI released:
116-
ref: aj/airbyte-ci/update-python-local-cdk-code
120+
ref: master
117121
path: airbyte
118122
- name: Test Connector
119123
if: steps.no_changes.outputs.status != 'cancelled'
@@ -133,16 +137,7 @@ jobs:
133137
--skip-step qa_checks \
134138
--skip-step connector_live_tests
135139
136-
# Upload the job output to the artifacts
137-
- name: Upload Job Output
138-
id: upload_job_output
139-
if: always() && steps.no_changes.outputs.status != 'cancelled'
140-
uses: actions/upload-artifact@v4
141-
with:
142-
name: ${{matrix.connector}}-job-output
143-
path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports
144-
145-
- name: Evaluate Job Output
140+
- name: Evaluate Test Output
146141
if: always() && steps.no_changes.outputs.status != 'cancelled'
147142
run: |
148143
# save job output json file as ci step output
@@ -162,3 +157,12 @@ jobs:
162157
echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. Check the logs for more details."
163158
exit 1
164159
fi
160+
161+
# Upload the job output to the artifacts
162+
- name: Upload Job Output
163+
id: upload_job_output
164+
if: always() && steps.no_changes.outputs.status != 'cancelled'
165+
uses: actions/upload-artifact@v4
166+
with:
167+
name: ${{matrix.connector}}-job-output
168+
path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports

‎.github/workflows/publish_sdm_connector.yml‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
# This flow publishes the Source-Declarative-Manifest (SDM)
2+
# connector to DockerHub as a Docker image.
3+
4+
name: Publish SDM Connector
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
version:
10+
description:
11+
The version to publish, ie 1.0.0 or 1.0.0-dev1.
12+
If omitted, and if run from a release branch, the version will be
13+
inferred from the git tag.
14+
If omitted, and if run from a non-release branch, then only a SHA-based
15+
Docker tag will be created.
16+
required: false
17+
dry_run:
18+
description: If true, the workflow will not push to DockerHub.
19+
type: boolean
20+
required: false
21+
default: false
22+
23+
jobs:
24+
build:
25+
runs-on: ubuntu-latest
26+
steps:
27+
28+
- name: Detect Release Tag Version
29+
if: startsWith(github.ref, 'refs/tags/v')
30+
run: |
31+
DETECTED_VERSION=${{ github.ref_name }}
32+
echo "Version ref set to '${DETECTED_VERSION}'"
33+
# Remove the 'v' prefix if it exists
34+
DETECTED_VERSION="${DETECTED_VERSION#v}"
35+
echo "Setting version to '$DETECTED_VERSION'"
36+
echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV
37+
38+
- name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
39+
id: set_version
40+
if: github.event_name == 'workflow_dispatch'
41+
run: |
42+
INPUT_VERSION=${{ github.event.inputs.version }}
43+
echo "Version input set to '${INPUT_VERSION}'"
44+
# Exit with success if both detected and input versions are empty
45+
if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then
46+
echo "No version detected or input. Will publish to SHA tag instead."
47+
echo 'VERSION=' >> $GITHUB_ENV
48+
exit 0
49+
fi
50+
# Remove the 'v' prefix if it exists
51+
INPUT_VERSION="${INPUT_VERSION#v}"
52+
# Fail if detected version is non-empty and different from the input version
53+
if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
54+
echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
55+
exit 1
56+
fi
57+
# Set the version to the input version if non-empty, otherwise the detected version
58+
VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
59+
# Fail if the version is still empty
60+
if [ -z "$VERSION" ]; then
61+
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
62+
exit 1
63+
fi
64+
echo "Setting version to '$VERSION'"
65+
echo "VERSION=${VERSION}" >> $GITHUB_ENV
66+
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
67+
# Check if version is a prerelease version (will not tag 'latest')
68+
if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
69+
echo "IS_PRERELEASE=false" >> $GITHUB_ENV
70+
echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT
71+
else
72+
echo "IS_PRERELEASE=true" >> $GITHUB_ENV
73+
echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
74+
fi
75+
76+
- uses: actions/checkout@v4
77+
with:
78+
fetch-depth: 0
79+
80+
- uses: hynek/build-and-inspect-python-package@v2
81+
name: Build package with version ref '${{ env.VERSION || '0.0.0dev0' }}'
82+
env:
83+
# Pass in the evaluated version from the previous step
84+
# More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables
85+
POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}}
86+
87+
- uses: actions/upload-artifact@v4
88+
with:
89+
name: Packages-${{ github.run_id }}
90+
path: |
91+
/tmp/baipp/dist/*.whl
92+
/tmp/baipp/dist/*.tar.gz
93+
outputs:
94+
VERSION: ${{ steps.set_version.outputs.VERSION }}
95+
IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }}
96+
97+
publish_sdm:
98+
name: Publish SDM to DockerHub
99+
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
100+
runs-on: ubuntu-latest
101+
needs: [build]
102+
environment:
103+
name: DockerHub
104+
url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
105+
env:
106+
VERSION: ${{ needs.build.outputs.VERSION }}
107+
IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
108+
109+
steps:
110+
- uses: actions/checkout@v4
111+
with:
112+
fetch-depth: 0
113+
114+
# We need to download the build artifact again because the previous job was on a different runner
115+
- name: Download Build Artifact
116+
uses: actions/download-artifact@v4
117+
with:
118+
name: Packages-${{ github.run_id }}
119+
path: dist
120+
121+
- name: Set up QEMU for multi-platform builds
122+
uses: docker/setup-qemu-action@v3
123+
124+
- name: Set up Docker Buildx
125+
uses: docker/setup-buildx-action@v3
126+
127+
- name: Login to Docker Hub
128+
uses: docker/login-action@v3
129+
with:
130+
username: ${{ secrets.DOCKER_HUB_USERNAME }}
131+
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
132+
133+
- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
134+
if: env.VERSION != ''
135+
run: |
136+
tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
137+
if [ -z "$tag" ]; then
138+
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
139+
exit 1
140+
fi
141+
echo "Checking if tag '$tag' exists on DockerHub..."
142+
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
143+
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
144+
exit 1
145+
fi
146+
echo "No existing tag '$tag' found. Proceeding with publish."
147+
148+
- name: Build and push (sha tag)
149+
# Only run if the version is not set
150+
if: env.VERSION == '' && github.event.inputs.dry_run == 'false'
151+
uses: docker/build-push-action@v5
152+
with:
153+
context: .
154+
platforms: linux/amd64,linux/arm64
155+
push: true
156+
tags: |
157+
airbyte/source-declarative-manifest:${{ github.sha }}
158+
159+
- name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
160+
# Only run if the version is set
161+
if: env.VERSION != '' && github.event.inputs.dry_run == 'false'
162+
uses: docker/build-push-action@v5
163+
with:
164+
context: .
165+
platforms: linux/amd64,linux/arm64
166+
push: true
167+
tags: |
168+
airbyte/source-declarative-manifest:${{ env.VERSION }}
169+
170+
171+
- name: Build and push ('latest' tag)
172+
# Only run if version is set and IS_PRERELEASE is false
173+
if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false'
174+
uses: docker/build-push-action@v5
175+
with:
176+
context: .
177+
platforms: linux/amd64,linux/arm64
178+
push: true
179+
tags: |
180+
airbyte/source-declarative-manifest:latest

‎.github/workflows/pypi_publish.yml‎

Lines changed: 0 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -59,81 +59,3 @@ jobs:
5959

6060
- name: Publish to PyPI
6161
uses: pypa/[email protected]
62-
63-
publish_sdm:
64-
name: Publish SDM to DockerHub
65-
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
66-
runs-on: ubuntu-latest
67-
needs: [publish]
68-
environment:
69-
name: DockerHub
70-
url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
71-
72-
steps:
73-
- uses: actions/checkout@v4
74-
with:
75-
fetch-depth: 0
76-
77-
- name: Set Version (workflow_dispatch)
78-
if: github.event_name == 'workflow_dispatch'
79-
run: |
80-
VERSION=${{ github.event.inputs.version }}
81-
echo "Version input set to '${VERSION}'"
82-
# Remove the 'v' prefix if it exists
83-
VERSION=${VERSION#v}
84-
echo "Setting version to '$VERSION'"
85-
echo "VERSION=${VERSION}" >> $GITHUB_ENV
86-
87-
- name: Set Version (tag)
88-
if: startsWith(github.ref, 'refs/tags/v')
89-
run: |
90-
VERSION=${{ github.ref_name }}
91-
echo "Version ref set to '${VERSION}'"
92-
# Remove the 'v' prefix if it exists
93-
VERSION=${VERSION#v}
94-
echo "Setting version to '$VERSION'"
95-
echo "VERSION=${VERSION}" >> $GITHUB_ENV
96-
97-
# We need to download the build artifact again because the previous job was on a different runner
98-
- name: Download Build Artifact
99-
uses: actions/download-artifact@v4
100-
with:
101-
name: Packages-${{ github.run_id }}
102-
path: dist
103-
104-
- name: Set up QEMU for multi-platform builds
105-
uses: docker/setup-qemu-action@v3
106-
107-
- name: Set up Docker Buildx
108-
uses: docker/setup-buildx-action@v3
109-
110-
- name: Login to Docker Hub
111-
uses: docker/login-action@v3
112-
with:
113-
username: ${{ secrets.DOCKER_HUB_USERNAME }}
114-
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
115-
116-
- name: Check for existing tag
117-
run: |
118-
tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
119-
if [ -z "$tag" ]; then
120-
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
121-
exit 1
122-
fi
123-
echo "Checking if tag '$tag' exists on DockerHub..."
124-
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
125-
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
126-
exit 1
127-
fi
128-
echo "No existing tag '$tag' found. Proceeding with publish."
129-
130-
- name: Build and push
131-
uses: docker/build-push-action@v5
132-
with:
133-
context: .
134-
platforms: linux/amd64,linux/arm64
135-
push: true
136-
tags: |
137-
airbyte/source-declarative-manifest:latest
138-
airbyte/source-declarative-manifest:${{ env.VERSION }}
139-
airbyte/source-declarative-manifest:${{ github.sha }}

‎.github/workflows/pytest_fast.yml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,7 @@ on:
44
push:
55
branches:
66
- main
7-
paths:
8-
- 'airbyte_cdk/**'
9-
- 'poetry.lock'
10-
- 'pyproject.toml'
117
pull_request:
12-
paths:
13-
- 'airbyte_cdk/**'
14-
- 'poetry.lock'
15-
- 'pyproject.toml'
168

179
jobs:
1810
pytest-fast:

0 commit comments

Comments
 (0)