docker-compose.yml

# By default, this docker compose script maps all services to localhost only.
# If you need to make services available outside of your machine, add
# appropriate service mappings to the .env file. See .env.example file for
# configuration example.
#
# Notes on image versions:
#  - For the key services such as postgres and pulsar we are trying to run
#    against the oldest supported version
#  - For the zookeeper and kafka we are trying to use the oldest supported
#    version that has arm64 images
#  - For everything else we are trying to run against the latest version.
#
# To run against the latest image versions update .env file. See .env.example
# file for configuration examples. You might need to remove the old images
# first if they are already tagged latest and volumes if their content is
# incompatible with the latest version, as in case of postgres.

name: quickwit

networks:
  default:
    name: quickwit-network
    ipam:
      config:
      - subnet: 172.16.7.0/24
        gateway: 172.16.7.1

services:
  localstack:
    image: localstack/localstack:${LOCALSTACK_VERSION:-3.5.0}
    container_name: localstack
    ports:
      - "${MAP_HOST_LOCALSTACK:-127.0.0.1}:4566:4566"
      - "${MAP_HOST_LOCALSTACK:-127.0.0.1}:4571:4571"
      - "${MAP_HOST_LOCALSTACK:-127.0.0.1}:8080:8080"
    profiles:
      - all
      - localstack
    environment:
      SERVICES: kinesis,s3,sqs
      PERSISTENCE: 1
    volumes:
      - .localstack:/etc/localstack/init/ready.d
      - localstack_data:/var/lib/localstack
    healthcheck:
      test: ["CMD", "curl", "-k", "-f", "https://localhost:4566/quickwit-integration-tests"]
      interval: 1s
      timeout: 5s
      retries: 100

  postgres:
    # The oldest supported version. EOL November 14, 2024
    image: postgres:${POSTGRES_VERSION:-12.17-alpine}
    container_name: postgres
    ports:
      - "${MAP_HOST_POSTGRES:-127.0.0.1}:5432:5432"
    profiles:
      - all
      - postgres
    environment:
      PGDATA: /var/lib/postgresql/data/pgdata
      POSTGRES_USER: ${POSTGRES_USER:-quickwit-dev}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-quickwit-dev}
      POSTGRES_DB: ${POSTGRES_DB:-quickwit-metastore-dev}
    volumes:
      - postgres_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready"]
      interval: 1s
      timeout: 5s
      retries: 100

  pulsar-broker:
    # The oldest version with arm64 docker images. EOL May 2 2025
    image: apachepulsar/pulsar:${PULSAR_VERSION:-3.0.0}
    container_name: pulsar-broker
    command: bin/pulsar standalone
    ports:
      - "${MAP_HOST_PULSAR:-127.0.0.1}:6650:6650"
      - "${MAP_HOST_PULSAR:-127.0.0.1}:8081:8080"
    environment:
      PULSAR_MEM: "-Xms384M -Xmx384M"
    profiles:
      - all
      - pulsar

  kafka-broker:
    # The oldest supported version with arm64 docker images. EOL October 27, 2023
    image: confluentinc/cp-kafka:${CP_VERSION:-7.0.9}
    container_name: kafka-broker
    depends_on:
      - zookeeper
    ports:
      - "${MAP_HOST_KAFKA:-127.0.0.1}:9092:9092"
      - "${MAP_HOST_KAFKA:-127.0.0.1}:9101:9101"
      - "${MAP_HOST_KAFKA:-127.0.0.1}:29092:29092"
    profiles:
      - all
      - kafka
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT_HOST://localhost:9092,PLAINTEXT://kafka-broker:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_HEAP_OPTS: -Xms256M -Xmx256M
    healthcheck:
      test: ["CMD", "cub", "kafka-ready", "-b", "localhost:9092", "1", "5"]
      start_period: 5s
      interval: 5s
      timeout: 10s
      retries: 100

  zookeeper:
    # The oldest supported version with arm64 images. EOL October 27, 2023
    image: confluentinc/cp-zookeeper:${CP_VERSION:-7.0.9}
    container_name: zookeeper
    ports:
      - "${MAP_HOST_ZOOKEEPER:-127.0.0.1}:2181:2181"
    profiles:
      - all
      - kafka
    environment:
      KAFKA_HEAP_OPTS: -Xms256M -Xmx256M
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    healthcheck:
      test: ["CMD", "cub", "zk-ready", "localhost:2181", "5"]
      start_period: 5s
      interval: 5s
      timeout: 10s
      retries: 100

  azurite:
    image: mcr.microsoft.com/azure-storage/azurite:${AZURITE_VERSION:-3.24.0}
    container_name: azurite
    ports:
        - "${MAP_HOST_AZURITE:-127.0.0.1}:10000:10000" # Blob store port
    profiles:
      - all
      - azurite
    volumes:
        - azurite_data:/data
    command: azurite --blobHost 0.0.0.0 --loose

  fake-gcs-server:
    image: fsouza/fake-gcs-server:${FAKE_GCS_SERVER_VERSION:-1.47.7}
    container_name: fake-gcs-server
    ports:
      - "${MAP_HOST_FAKE_GCS_SERVER:-127.0.0.1}:4443:4443" # Blob store port
    profiles:
      - all
      - fake-gcs-server
    volumes:
      - fake_gcs_server_data:/data
    command: -scheme http

  grafana:
    image: grafana/grafana-oss:${GRAFANA_VERSION:-10.4.1}
    container_name: grafana
    ports:
      - "${MAP_HOST_GRAFANA:-127.0.0.1}:3000:3000"
    profiles:
      - grafana
      - monitoring
    environment:
      GF_AUTH_DISABLE_LOGIN_FORM: "true"
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
    volumes:
      - grafana_conf:/etc/grafana
      - grafana_data:/var/lib/grafana
      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning

  jaeger:
    image: jaegertracing/all-in-one:${JAEGER_VERSION:-1.48.0}
    container_name: jaeger
    ports:
      - "${MAP_HOST_JAEGER:-127.0.0.1}:16686:16686" # Frontend
    profiles:
      - jaeger
      - monitoring

  otel-collector:
    image: otel/opentelemetry-collector:${OTEL_VERSION:-0.84.0}
    container_name: otel-collector
    ports:
      - "${MAP_HOST_OTEL:-127.0.0.1}:1888:1888"   # pprof extension
      - "${MAP_HOST_OTEL:-127.0.0.1}:8888:8888"   # Prometheus metrics exposed by the collector
      - "${MAP_HOST_OTEL:-127.0.0.1}:8889:8889"   # Prometheus exporter metrics
      - "${MAP_HOST_OTEL:-127.0.0.1}:13133:13133" # health_check extension
      - "${MAP_HOST_OTEL:-127.0.0.1}:4317:4317"   # OTLP gRPC receiver
      - "${MAP_HOST_OTEL:-127.0.0.1}:4318:4318"   # OTLP http receiver
      - "${MAP_HOST_OTEL:-127.0.0.1}:55679:55679" # zpages extension
    profiles:
      - otel
      - monitoring
    volumes:
      - ./monitoring/otel-collector-config.yaml:/etc/otel-collector-config.yaml
    command: ["--config=/etc/otel-collector-config.yaml"]

  prometheus:
    image: prom/prometheus:${PROMETHEUS_VERSION:-v2.43.0}
    container_name: prometheus
    ports:
      - "${MAP_HOST_PROMETHEUS:-127.0.0.1}:9090:9090"
    profiles:
      - prometheus
      - monitoring
    volumes:
      - ./monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
    extra_hosts:
      - "host.docker.internal:host-gateway"

  gcp-pubsub-emulator:
    # It is not an official docker image
    # if we prefer we can build a docker from the official docker image (gcloud cli)
    # and install the pubsub emulator https://cloud.google.com/pubsub/docs/emulator
    image: thekevjames/gcloud-pubsub-emulator:${GCLOUD_EMULATOR:-455.0.0}
    container_name: gcp-pubsub-emulator
    ports:
      - "${MAP_HOST_GCLOUD_EMULATOR:-127.0.0.1}:8681:8681"
    environment:
      # create a fake gcp project and a topic / subscription
      - PUBSUB_PROJECT1=quickwit-emulator,emulator_topic:emulator_subscription
    profiles:
      - all
      - gcp-pubsub

volumes:
  azurite_data:
  fake_gcs_server_data:
  grafana_conf:
  grafana_data:
  localstack_data:
  postgres_data: