feat(cli): Add image build command to airbyte-cdk CLI

airbyte_cdk/cli/__init__.py

@@ -1 +1,28 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Airbyte CDK CLI.
+
+The Airbyte CDK provides command-line tools for working with connectors and related resources.
+
+
+```bash
+pip install airbyte-cdk
+
+pipx run airbyte-cdk [command]
+```
+
+
+
+- `airbyte-cdk-build`: Build connector Docker images (legacy entry point)
+- `source-declarative-manifest`: Run a declarative YAML manifest connector
+- `airbyte-cdk`: Main CLI entry point with subcommands
+
+
+The `airbyte-cdk` command includes subcommands organized by category:
+
+```bash
+airbyte-cdk image build [OPTIONS] CONNECTOR_DIR
+```
+
+
+For details on specific commands, see the documentation for each command module.
+"""

airbyte_cdk/cli/build/__init__.py

@@ -0,0 +1,41 @@
+"""Airbyte CDK Build Command.
+
+The `airbyte-cdk-build` command provides a simplified way to build connector Docker images without requiring the full Airbyte CI pipeline.
+
+
+```bash
+pip install airbyte-cdk
+
+pipx run airbyte-cdk-build [arguments]
+```
+
+
+```bash
+airbyte-cdk-build /path/to/connector
+
+airbyte-cdk image build /path/to/connector
+
+airbyte-cdk image build /path/to/connector --tag custom_tag
+
+airbyte-cdk image build /path/to/connector --no-verify
+
+airbyte-cdk image build /path/to/connector --verbose
+```
+
+
+- `connector_dir`: Path to the connector directory (required)
+- `--tag`: Tag to apply to the built image (default: "dev")
+- `--no-verify`: Skip verification of the built image
+- `--verbose`, `-v`: Enable verbose logging
+
+
+The command reads the connector's metadata from the `metadata.yaml` file, builds a Docker image using the connector's Dockerfile, and verifies the image by running the `spec` command. The image is tagged according to the repository name specified in the metadata and the provided tag.
+
+This command is designed to be a simpler alternative to the `airbyte-ci build` command, using Docker directly on the host machine instead of Dagger.
+"""
+
+from airbyte_cdk.cli.build._run import run
+
+__all__ = [
+    "run",
+]

airbyte_cdk/cli/build/_run.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Implements the `airbyte-cdk build` command for building connector Docker images.
+
+This command provides a simplified way to build connector Docker images without
+requiring the full Airbyte CI pipeline, which uses Dagger.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from airbyte_cdk.utils.docker.build import run_command as docker_run_command
+
+
+def run() -> None:
+    """Entry point for the airbyte-cdk build command."""
+    parser = argparse.ArgumentParser(description="Build connector Docker images")
+    parser.add_argument("connector_dir", type=str, help="Path to the connector directory")
+    parser.add_argument(
+        "--tag", type=str, default="dev", help="Tag to apply to the built image (default: dev)"
+    )
+    parser.add_argument(
+        "--no-verify", action="store_true", help="Skip verification of the built image"
+    )
+    parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
+
+    args = parser.parse_args(sys.argv[1:])
+
+    sys.exit(
+        docker_run_command(
+            connector_dir=Path(args.connector_dir),
+            tag=args.tag,
+            no_verify=args.no_verify,
+            verbose=args.verbose,
+        )
+    )
+
+
+if __name__ == "__main__":
+    run()

airbyte_cdk/cli/build/models.py

@@ -0,0 +1,54 @@
+"""Models for the build command."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ConnectorLanguage(str, Enum):
+    """Connector implementation language."""
+
+    PYTHON = "python"
+    JAVA = "java"
+    LOW_CODE = "low-code"
+    MANIFEST_ONLY = "manifest-only"
+    UNKNOWN = "unknown"
+
+
+class ConnectorBuildOptions(BaseModel):
+    """Connector build options from metadata.yaml."""
+
+    model_config = {"extra": "allow"}
+
+    baseImage: Optional[str] = Field(
+        None, description="Base image to use for building the connector"
+    )
+    path: Optional[str] = Field(
+        None, description="Path to the connector code within the repository"
+    )
+
+
+class ConnectorMetadata(BaseModel):
+    """Connector metadata from metadata.yaml."""
+
+    model_config = {"extra": "allow"}
+
+    dockerRepository: str = Field(..., description="Docker repository for the connector image")
+    dockerImageTag: str = Field(..., description="Docker image tag for the connector")
+    language: Optional[ConnectorLanguage] = Field(
+        None, description="Language of the connector implementation"
+    )
+    connectorBuildOptions: Optional[ConnectorBuildOptions] = Field(
+        None, description="Options for building the connector"
+    )
+
+
+class MetadataFile(BaseModel):
+    """Represents the structure of a metadata.yaml file."""
+
+    model_config = {"extra": "allow"}
+
+    data: ConnectorMetadata = Field(..., description="Connector metadata")

airbyte_cdk/cli/commands/__init__.py

@@ -0,0 +1 @@
+"""CLI commands for Airbyte CDK."""

airbyte_cdk/cli/commands/cli.py

@@ -0,0 +1,14 @@
+"""Main CLI command group for Airbyte CDK."""
+
+import click
+
+from airbyte_cdk.cli.commands.image import image
+
+
+@click.group()
+def cli() -> None:
+    """Airbyte CDK command-line interface."""
+    pass
+
+
+cli.add_command(image)

airbyte_cdk/cli/commands/image.py

@@ -0,0 +1,98 @@
+"""Image-related commands for the Airbyte CDK CLI."""
+
+import sys
+from pathlib import Path
+
+import click
+
+from airbyte_cdk.utils.docker.build import (
+    build_from_base_image,
+    build_from_dockerfile,
+    read_metadata,
+    set_up_logging,
+    verify_docker_installation,
+    verify_image,
+)
+
+
+@click.group()
+def image() -> None:
+    """Commands for working with connector Docker images."""
+    pass
+
+
+@image.command()
+@click.argument(
+    "connector_dir", type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path)
+)
+@click.option("--tag", default="dev", help="Tag to apply to the built image (default: dev)")
+@click.option("--no-verify", is_flag=True, help="Skip verification of the built image")
+@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging")
+def build(connector_dir: Path, tag: str, no_verify: bool, verbose: bool) -> None:
+    """Build a connector Docker image.
+
+    This command builds a Docker image for a connector, using either
+    the connector's Dockerfile or a base image specified in the metadata.
+    The image is built for both AMD64 and ARM64 architectures.
+    """
+    set_up_logging(verbose)
+
+    if not verify_docker_installation():
+        click.echo(
+            "Docker is not installed or not running. Please install Docker and try again.", err=True
+        )
+        sys.exit(1)
+
+    try:
+        metadata = read_metadata(connector_dir)
+        click.echo(f"Connector: {metadata.dockerRepository}")
+        click.echo(f"Version: {metadata.dockerImageTag}")
+
+        if metadata.language:
+            click.echo(f"Connector language from metadata: {metadata.language}")
+        else:
+            click.echo("Connector language not specified in metadata")
+
+        try:
+            import subprocess
+            result = subprocess.run(
+                ["docker", "buildx", "inspect"], 
+                capture_output=True, 
+                text=True, 
+                check=False
+            )
+
+            if "linux/amd64" in result.stdout and "linux/arm64" in result.stdout:
+                platforms = "linux/amd64,linux/arm64"
+                click.echo(f"Building for platforms: {platforms}")
+            else:
+                platforms = "linux/amd64"
+                click.echo(f"Multi-platform build not available. Building for platform: {platforms}")
+                click.echo("To enable multi-platform builds, configure Docker buildx with: docker buildx create --use")
+        except Exception:
+            platforms = "linux/amd64"
+            click.echo(f"Multi-platform build check failed. Building for platform: {platforms}")
+
+        if metadata.connectorBuildOptions and metadata.connectorBuildOptions.baseImage:
+            image_name = build_from_base_image(connector_dir, metadata, tag, platforms)
+        else:
+            image_name = build_from_dockerfile(connector_dir, metadata, tag, platforms)
+
+        if not no_verify:
+            if verify_image(image_name):
+                click.echo(f"Build completed successfully: {image_name}")
+                sys.exit(0)
+            else:
+                click.echo(f"Built image failed verification: {image_name}", err=True)
+                sys.exit(1)
+        else:
+            click.echo(f"Build completed successfully (without verification): {image_name}")
+            sys.exit(0)
+
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        if verbose:
+            import traceback
+
+            click.echo(traceback.format_exc(), err=True)
+        sys.exit(1)

airbyte_cdk/cli/entrypoint/__init__.py

@@ -0,0 +1,5 @@
+"""CLI entrypoint module for Airbyte CDK."""
+
+from airbyte_cdk.cli.entrypoint.cli import run
+
+__all__ = ["run"]

airbyte_cdk/cli/entrypoint/cli.py

@@ -0,0 +1,8 @@
+"""Entrypoint for the Airbyte CDK CLI."""
+
+from airbyte_cdk.cli.commands.cli import cli
+
+
+def run() -> None:
+    """Run the Airbyte CDK CLI."""
+    cli(prog_name="airbyte-cdk")

airbyte_cdk/models/connector_metadata.py

@@ -0,0 +1,54 @@
+"""Models for connector metadata."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class ConnectorLanguage(str, Enum):
+    """Connector implementation language."""
+
+    PYTHON = "python"
+    JAVA = "java"
+    LOW_CODE = "low-code"
+    MANIFEST_ONLY = "manifest-only"
+    UNKNOWN = "unknown"
+
+
+class ConnectorBuildOptions(BaseModel):
+    """Connector build options from metadata.yaml."""
+
+    model_config = {"extra": "allow"}
+
+    baseImage: Optional[str] = Field(
+        None, description="Base image to use for building the connector"
+    )
+    path: Optional[str] = Field(
+        None, description="Path to the connector code within the repository"
+    )
+
+
+class ConnectorMetadata(BaseModel):
+    """Connector metadata from metadata.yaml."""
+
+    model_config = {"extra": "allow"}
+
+    dockerRepository: str = Field(..., description="Docker repository for the connector image")
+    dockerImageTag: str = Field(..., description="Docker image tag for the connector")
+    language: Optional[ConnectorLanguage] = Field(
+        None, description="Language of the connector implementation"
+    )
+    connectorBuildOptions: Optional[ConnectorBuildOptions] = Field(
+        None, description="Options for building the connector"
+    )
+
+
+class MetadataFile(BaseModel):
+    """Represents the structure of a metadata.yaml file."""
+
+    model_config = {"extra": "allow"}
+
+    data: ConnectorMetadata = Field(..., description="Connector metadata")

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2315,7 +2315,7 @@ definitions:
     properties:
       type:
         type: string
-        enum: [ KeyTransformation ]
+        enum: [KeyTransformation]
       prefix:
         title: Key Prefix
         description: Prefix to add for object keys. If not provided original keys remain unchanged.

airbyte_cdk/utils/docker/__init__.py

@@ -0,0 +1 @@
+"""Docker utilities for Airbyte CDK."""