Skip to content

Commit e80c173

Browse files
pedroslopezgithub-advanced-security[bot]
pedroslopez
and
github-advanced-security[bot]
authored
feat: add manifest server (#719)
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
1 parent 2aa7b58 commit e80c173

38 files changed

+3170
-28
lines changed

‎.github/actions/check-docker-tag/action.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: 'Check Docker Tag Exists'
2+
description: 'Check if a Docker tag exists on DockerHub to prevent overwrites'
3+
inputs:
4+
image_name:
5+
description: 'Docker image name (e.g. airbyte/source-declarative-manifest)'
6+
required: true
7+
tag:
8+
description: 'Docker tag to check'
9+
required: true
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: "Check for existing tag (${{ inputs.image_name }}:${{ inputs.tag }})"
14+
shell: bash
15+
run: |
16+
image="${{ inputs.image_name }}"
17+
tag_input="${{ inputs.tag }}"
18+
if [ -z "$image" ] || [ -z "$tag_input" ]; then
19+
echo "Error: image_name and tag are required."
20+
exit 1
21+
fi
22+
tag="${image}:${tag_input}"
23+
echo "Checking if tag '$tag' exists on DockerHub..."
24+
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
25+
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
26+
exit 1
27+
fi
28+
echo "No existing tag '$tag' found. Proceeding with publish."

‎.github/workflows/docker-build-check.yml‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
name: Docker Build Check
2+
permissions:
3+
contents: read
24

35
on:
46
pull_request:
57
branches:
68
- main
79

810
jobs:
9-
docker-build-check:
10-
name: SDM Docker Image Build # Renamed job to be more descriptive
11+
sdm-docker-build-check:
12+
name: SDM Docker Image Build
1113
runs-on: ubuntu-24.04
1214
steps:
1315
- name: Checkout code
@@ -42,3 +44,29 @@ jobs:
4244
push: false
4345
tags: airbyte/source-declarative-manifest:pr-${{ github.event.pull_request.number }}
4446
outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=SDM Docker image for PR ${{ github.event.pull_request.number }}
47+
48+
manifest-server-docker-build-check:
49+
name: Manifest Server Docker Image Build
50+
runs-on: ubuntu-24.04
51+
steps:
52+
- name: Checkout code
53+
uses: actions/checkout@v4
54+
with:
55+
fetch-depth: 0
56+
57+
- name: Set up QEMU for multi-platform builds
58+
uses: docker/setup-qemu-action@v3
59+
60+
- name: Set up Docker Buildx
61+
uses: docker/setup-buildx-action@v3
62+
63+
- name: Build Manifest Server Docker image for multiple platforms
64+
id: manifest-server-build
65+
uses: docker/build-push-action@v5
66+
with:
67+
context: .
68+
file: airbyte_cdk/manifest_server/Dockerfile
69+
platforms: linux/amd64,linux/arm64
70+
push: false
71+
tags: airbyte/manifest-server:pr-${{ github.event.pull_request.number }}
72+
outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=Manifest Server Docker image for PR ${{ github.event.pull_request.number }}

‎.github/workflows/pypi_publish.yml‎ renamed to ‎.github/workflows/publish.yml‎

Lines changed: 95 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
# we have to also update the Trusted Publisher settings on PyPI.
77

88
name: CDK Publish
9+
permissions:
10+
contents: read
911

1012
on:
1113
push:
@@ -31,6 +33,11 @@ on:
3133
type: boolean
3234
required: true
3335
default: true
36+
publish_manifest_server:
37+
description: "Publish Manifest Server to DockerHub. If true, the workflow will publish the Manifest Server to DockerHub."
38+
type: boolean
39+
required: true
40+
default: true
3441
update_connector_builder:
3542
description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder."
3643
type: boolean
@@ -204,18 +211,10 @@ jobs:
204211

205212
- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
206213
if: env.VERSION != ''
207-
run: |
208-
tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
209-
if [ -z "$tag" ]; then
210-
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
211-
exit 1
212-
fi
213-
echo "Checking if tag '$tag' exists on DockerHub..."
214-
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
215-
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
216-
exit 1
217-
fi
218-
echo "No existing tag '$tag' found. Proceeding with publish."
214+
uses: ./.github/actions/check-docker-tag
215+
with:
216+
image_name: airbyte/source-declarative-manifest
217+
tag: ${{ env.VERSION }}
219218

220219
- name: "Build and push (sha tag: '${{ github.sha }}')"
221220
# Only run if the version is not set
@@ -250,6 +249,90 @@ jobs:
250249
tags: |
251250
airbyte/source-declarative-manifest:latest
252251
252+
publish_manifest_server:
253+
name: Publish Manifest Server to DockerHub
254+
if: >
255+
(github.event_name == 'push' &&
256+
startsWith(github.ref, 'refs/tags/v')) ||
257+
(github.event_name == 'workflow_dispatch' &&
258+
github.event.inputs.publish_manifest_server == 'true'
259+
)
260+
runs-on: ubuntu-24.04
261+
needs: [build]
262+
environment:
263+
name: DockerHub
264+
url: https://hub.docker.com/r/airbyte/manifest-server/tags
265+
env:
266+
VERSION: ${{ needs.build.outputs.VERSION }}
267+
IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
268+
269+
steps:
270+
- uses: actions/checkout@v4
271+
with:
272+
fetch-depth: 0
273+
274+
# We need to download the build artifact again because the previous job was on a different runner
275+
- name: Download Build Artifact
276+
uses: actions/download-artifact@v4
277+
with:
278+
name: Packages-${{ github.run_id }}
279+
path: dist
280+
281+
- name: Set up QEMU for multi-platform builds
282+
uses: docker/setup-qemu-action@v3
283+
284+
- name: Set up Docker Buildx
285+
uses: docker/setup-buildx-action@v3
286+
287+
- name: Login to Docker Hub
288+
uses: docker/login-action@v3
289+
with:
290+
username: ${{ secrets.DOCKER_HUB_USERNAME }}
291+
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
292+
293+
- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
294+
if: env.VERSION != ''
295+
uses: ./.github/actions/check-docker-tag
296+
with:
297+
image_name: airbyte/manifest-server
298+
tag: ${{ env.VERSION }}
299+
300+
- name: "Build and push (sha tag: '${{ github.sha }}')"
301+
# Only run if the version is not set
302+
if: env.VERSION == ''
303+
uses: docker/build-push-action@v5
304+
with:
305+
context: .
306+
file: airbyte_cdk/manifest_server/Dockerfile
307+
platforms: linux/amd64,linux/arm64
308+
push: true
309+
tags: |
310+
airbyte/manifest-server:${{ github.sha }}
311+
312+
- name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
313+
# Only run if the version is set
314+
if: env.VERSION != ''
315+
uses: docker/build-push-action@v5
316+
with:
317+
context: .
318+
file: airbyte_cdk/manifest_server/Dockerfile
319+
platforms: linux/amd64,linux/arm64
320+
push: true
321+
tags: |
322+
airbyte/manifest-server:${{ env.VERSION }}
323+
324+
- name: Build and push ('latest' tag)
325+
# Only run if version is set and IS_PRERELEASE is false
326+
if: env.VERSION != '' && env.IS_PRERELEASE == 'false'
327+
uses: docker/build-push-action@v5
328+
with:
329+
context: .
330+
file: airbyte_cdk/manifest_server/Dockerfile
331+
platforms: linux/amd64,linux/arm64
332+
push: true
333+
tags: |
334+
airbyte/manifest-server:latest
335+
253336
update-connector-builder:
254337
# Create a PR against the Builder, to update the CDK version that it uses.
255338
# In the future, Builder may use the SDM docker image instead of the Python CDK package.

‎airbyte_cdk/manifest_server/Dockerfile‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Dockerfile for the Airbyte Manifest Server.
2+
#
3+
# This Dockerfile should be built from the root of the repository.
4+
#
5+
# Example:
6+
# docker build -f airbyte_cdk/manifest_server/Dockerfile -t airbyte/manifest-server .
7+
8+
FROM python:3.12-slim-bookworm
9+
10+
# Install git (needed for dynamic versioning) and poetry
11+
RUN apt-get update && \
12+
apt-get install -y git && \
13+
rm -rf /var/lib/apt/lists/* && \
14+
pip install poetry==1.8.3
15+
16+
# Configure poetry to not create virtual environments and disable interactive mode
17+
ENV POETRY_NO_INTERACTION=1 \
18+
POETRY_CACHE_DIR=/tmp/poetry_cache
19+
20+
WORKDIR /app
21+
22+
# Copy poetry files (build from project root)
23+
COPY pyproject.toml poetry.lock ./
24+
25+
# Copy the project source code (needed for dynamic versioning)
26+
COPY . /app
27+
28+
# Install dependencies and package directly to system Python
29+
RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
30+
poetry config virtualenvs.create false && \
31+
poetry install --extras manifest-server
32+
33+
# Create a non-root user and group
34+
RUN groupadd --gid 1000 airbyte && \
35+
useradd --uid 1000 --gid airbyte --shell /bin/bash --create-home airbyte
36+
37+
# Change ownership
38+
RUN chown -R airbyte:airbyte /app
39+
40+
# Run app as non-root user
41+
USER airbyte:airbyte
42+
43+
EXPOSE 8080
44+
45+
CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]

0 commit comments

Comments
 (0)