feat: gcs support for storage

Cargo.toml

@@ -10,14 +10,19 @@ build = "build.rs"
 [dependencies]
 # Arrow and DataFusion ecosystem
 arrow = "54.0.0"
-arrow-array = "54.0.0" 
+arrow-array = "54.0.0"
 arrow-flight = { version = "54.0.0", features = ["tls"] }
 arrow-ipc = { version = "54.0.0", features = ["zstd"] }
 arrow-json = "54.0.0"
 arrow-schema = { version = "54.0.0", features = ["serde"] }
 arrow-select = "54.0.0"
 datafusion = "45.0.0"
-object_store = { version = "0.11.2", features = ["cloud", "aws", "azure"] }
+object_store = { version = "0.11.2", features = [
+    "cloud",
+    "aws",
+    "azure",
+    "gcp",
+] }
 parquet = "54.0.0"
 
 # Web server and HTTP-related
@@ -34,7 +39,11 @@ tower-http = { version = "0.6.1", features = ["cors"] }
 url = "2.4.0"
 
 # Connectors dependencies
-rdkafka = { version = "0.37", optional = true, features = ["cmake-build", "tracing", "libz-static"] }
+rdkafka = { version = "0.37", optional = true, features = [
+    "cmake-build",
+    "tracing",
+    "libz-static",
+] }
 sasl2-sys = { version = "0.1.22", optional = true, features = ["vendored"] }
 
 # Authentication and Security
@@ -144,7 +153,14 @@ assets-sha1 = "3e703ef8bedf8ae55fd31713f6267ad14ad3d29d"
 
 [features]
 debug = []
-kafka = ["rdkafka", "rdkafka/ssl-vendored", "rdkafka/ssl", "rdkafka/sasl", "sasl2-sys", "sasl2-sys/vendored"]
+kafka = [
+    "rdkafka",
+    "rdkafka/ssl-vendored",
+    "rdkafka/ssl",
+    "rdkafka/sasl",
+    "sasl2-sys",
+    "sasl2-sys/vendored",
+]
 
 [profile.release-lto]
 inherits = "release"

Dockerfile.debug

@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # build stage
-FROM rust:1.84.0-bookworm AS builder
+FROM docker.io/rust:1.84.0-bookworm AS builder
 
 LABEL org.opencontainers.image.title="Parseable"
 LABEL maintainer="Parseable Team <[email protected]>"

docker-compose-gcs-distributed-test.yaml

@@ -0,0 +1,110 @@
+networks:
+  parseable-internal:
+
+services:
+  # query server
+  parseable-query:
+    container_name: parseable-query
+    build:
+      context: .
+      dockerfile: Dockerfile.debug
+    platform: linux/amd64
+    command: ["parseable", "gcs-store"]
+    ports:
+      - "8000:8000"
+    environment:
+      - P_GCS_BUCKET=parseable-test-gcs-local
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=query
+      - RUST_LOG=warn
+      - GOOGLE_APPLICATION_CREDENTIALS=/parseable/svc/mpt-randd-8217aef869fd.json
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts:
+    volumes:
+      - "/home/opeyemi/Downloads/:/parseable/svc/:ro,z"
+  # ingest server one
+  parseable-ingest-one:
+    container_name: parseable-ingest-one
+    build:
+      context: .
+      dockerfile: Dockerfile.debug
+    platform: linux/amd64
+    command: ["parseable", "gcs-store"]
+    ports:
+      - "8000"
+    environment:
+      - P_GCS_BUCKET=parseable-test-gcs-local
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=ingest
+      - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
+      - RUST_LOG=warn
+      - GOOGLE_APPLICATION_CREDENTIALS=/parseable/svc/${GCS_CREDENTIALS_FILE:-key.json}
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      parseable-query:
+        condition: service_healthy
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+    volumes:
+      - "${GCS_CREDENTIALS_PATH:-./credentials}:/parseable/svc/:ro,z"
+
+  quest:
+    platform: linux/amd64
+    image: ghcr.io/parseablehq/quest:main
+    pull_policy: always
+    command:
+      [
+        "load",
+        "http://parseable-query:8000",
+        "parseableadmin",
+        "parseableadmin",
+        "20",
+        "10",
+        "5m",
+        "storage.googleapis.com",
+        "",
+        "",
+        "parseable-test-gcs-local",
+        "http://parseable-ingest-one:8000",
+        "parseableadmin",
+        "parseableadmin",
+      ]
+    networks:
+      - parseable-internal
+    depends_on:
+      parseable-query:
+        condition: service_healthy
+      parseable-ingest-one:
+        condition: service_healthy
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3

helm/templates/ingestor-statefulset.yaml

@@ -66,6 +66,12 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+      volumes:
+        - name: {{ .Values.parseable.gcsModeSecret.auth.secret_name }}
+          secret:
+            secretName: {{ .Values.parseable.gcsModeSecret.auth.secret_name }}
+      {{- end }}
       containers:
       - name: {{ .Chart.Name }}
         securityContext:
@@ -74,19 +80,19 @@ spec:
         imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
         args:
           - /usr/bin/parseable
-          - {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}
+          - {{ .Values.parseable.store | quote }}
           - --ingestor-endpoint=$(HOSTNAME).{{ include "parseable.fullname" . }}-ingestor-headless.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.parseable.highAvailability.ingestor.port }}
         env:
-          {{- range $key, $value :=  .Values.parseable.highAvailability.ingestor.env }}
-          - name: {{ $key }}
-            value: {{ tpl $value $ | quote }}
           - name: HOSTNAME
             valueFrom:
               fieldRef:
                 apiVersion: v1
                 fieldPath: metadata.name
+          {{- range $key, $value := .Values.parseable.highAvailability.ingestor.env }}
+          - name: {{ $key }}
+            value: {{ tpl $value $ | quote }}
           {{- end }}
-          
+
           {{- if .Values.parseable.auditLogging.enabled }}
           - name: P_AUDIT_LOGGER
             value: {{ .Values.parseable.auditLogging.p_server | quote }}
@@ -111,11 +117,13 @@ spec:
           {{- end }}
 
           {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: {{ .Values.parseable.gcsModeSecret.auth.mount_path }}
           {{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
           {{- range $key := $secret.keys }}
           {{- $envPrefix := $secret.prefix | default "" | upper }}
           {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
-          - name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
+          - name: {{ $envPrefix }}{{ $envKey }}
             valueFrom:
               secretKeyRef:
                 name: {{ $secret.name }}
@@ -137,30 +145,39 @@ spec:
           {{- end }}
           {{- end }}
           {{- end }}
+
           - name: P_MODE
             value: "ingest"
+
           {{- if .Values.parseable.kafkaConnector.enabled }}
-            {{- range $key, $value := .Values.parseable.kafkaConnector.env }}
+          {{- range $key, $value := .Values.parseable.kafkaConnector.env }}
           - name: {{ $key }}
             value: {{ tpl $value $ | quote }}
-            {{- end }}
+          {{- end }}
           {{- end }}
 
         ports:
           - containerPort: {{ .Values.parseable.highAvailability.ingestor.port }}
-      {{- with .Values.readinessProbe }}
+        {{- with .Values.readinessProbe }}
         readinessProbe:
-          {{ toYaml . | nindent 12 }}
-      {{- end }}
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
         resources:
-          {{- toYaml .Values.parseable.highAvailability.ingestor.resources | nindent 12 }}
-        {{- if .Values.parseable.persistence.ingestor.enabled }}
+          {{- toYaml .Values.parseable.highAvailability.ingestor.resources | nindent 10 }}
+        {{- if or .Values.parseable.persistence.ingestor.enabled (and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled) }}
         volumeMounts:
+        {{- if .Values.parseable.persistence.ingestor.enabled }}
         - mountPath: "/parseable/staging"
           name: stage-volume
         {{- end }}
-  volumeClaimTemplates:
+        {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+        - mountPath: {{ .Values.parseable.gcsModeSecret.auth.mount_path }}
+          name: {{ .Values.parseable.gcsModeSecret.auth.secret_name }}
+          readOnly: true
+        {{- end }}
+        {{- end }}
   {{- if .Values.parseable.persistence.ingestor.enabled }}
+  volumeClaimTemplates:
   - metadata:
       name: stage-volume
     spec:
@@ -171,4 +188,4 @@ spec:
         requests:
           storage: {{ .Values.parseable.persistence.ingestor.size | quote }}
   {{- end }}
-{{- end }}
+{{- end }}