[Store] Add commands to setup/drop a store

.github/workflows/integration-tests.yaml

@@ -0,0 +1,150 @@
+name: Integration Tests
+
+on:
+    push:
+        paths-ignore:
+            - 'src/*/doc/**'
+            - 'src/**/*.md'
+    pull_request:
+        paths-ignore:
+            - 'src/*/doc/**'
+            - 'src/**/*.md'
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+    cancel-in-progress: true
+
+env:
+    REQUIRED_PHP_EXTENSIONS: 'mongodb'
+
+jobs:
+    php:
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                php-version: ['8.2', '8.3', '8.4']
+                dependency-version: ['']
+                symfony-version: ['']
+                include:
+                    # lowest deps
+                    -   php-version: '8.2'
+                        dependency-version: 'lowest'
+                    # LTS version of Symfony
+                    -   php-version: '8.2'
+                        symfony-version: '6.4.*'
+        services:
+            clickhouse:
+                image: clickhouse/clickhouse-server
+                env:
+                    CLICKHOUSE_DB: 'symfony'
+                    CLICKHOUSE_USER: 'symfony'
+                    CLICKHOUSE_PASSWORD: 'symfony'
+                ports:
+                    - '8123:8123'
+
+            chromadb:
+                image: chromadb/chroma
+                ports:
+                    - '8001:8000'
+
+            mariadb:
+                image: mariadb:11.7
+                env:
+                    MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
+                    MARIADB_DATABASE: my_database
+                ports:
+                    - '3309:3306'
+
+            postgres:
+                image: pgvector/pgvector:0.8.0-pg17
+                env:
+                    POSTGRES_DB: my_database
+                    POSTGRES_USER: postgres
+                    POSTGRES_PASSWORD: postgres
+                ports:
+                    - '5432:5432'
+
+            meilisearch:
+                image: getmeili/meilisearch:v1.15
+                env:
+                    MEILI_MASTER_KEY: '${MEILISEARCH_MASTER_KEY:-changeMe}'
+                ports:
+                    - '7700:7700'
+
+            mongodb:
+                image: mongodb/mongodb-community-server:7.0-ubi9
+                env:
+                    MONGO_INITDB_ROOT_USERNAME: 'symfony'
+                    MONGO_INITDB_ROOT_PASSWORD: 'symfony'
+                ports:
+                    - '27017:27017'
+
+            neo4j:
+                image: neo4j
+                env:
+                    NEO4J_AUTH: 'neo4j/${NEO4J_PASSWORD:-symfonyai}'
+                ports:
+                    - '7474:7474'
+                    - '7687:7687'
+
+            qdrant:
+                image: qdrant/qdrant
+                env:
+                    QDRANT__SERVICE__API_KEY: '${QDRAT_SERVICE_API_KEY:-changeMe}'
+                ports:
+                    - '6333:6333'
+
+            typesense:
+                image: typesense/typesense:29.0
+                env:
+                    TYPESENSE_API_KEY: '${TYPESENSE_API_KEY:-changeMe}'
+                    TYPESENSE_DATA_DIR: '/data'
+                ports:
+                    - '8108:8108'
+
+        env:
+            SYMFONY_REQUIRE: ${{ matrix.symfony-version || '>=6.4' }}
+
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Configure environment
+              run: |
+                  echo COLUMNS=120 >> $GITHUB_ENV
+                  echo COMPOSER_UP='composer update ${{ matrix.dependency-version == 'lowest' && '--prefer-lowest --prefer-stable' || '' }} --no-progress --no-interaction --ansi --ignore-platform-req=ext-mongodb' >> $GITHUB_ENV
+                  echo PHPUNIT='vendor/bin/phpunit' >> $GITHUB_ENV
+                  [ 'lowest' = '${{ matrix.dependency-version }}' ] && export SYMFONY_DEPRECATIONS_HELPER=weak
+
+                  PACKAGES=$(find src/ -mindepth 2 -type f -name composer.json -not -path "*/vendor/*" -printf '%h\n' | sed 's/^src\///' | grep -Ev "examples" | sort |  tr '\n' ' ')
+                  echo "Packages: $PACKAGES"
+                  echo "PACKAGES=$PACKAGES" >> $GITHUB_ENV
+
+            - name: Setup PHP
+              uses: shivammathur/setup-php@v2
+              with:
+                  php-version: ${{ matrix.php-version }}
+                  tools: flex
+                  extensions: "${{ env.REQUIRED_PHP_EXTENSIONS }}"
+
+            - name: Get composer cache directory
+              id: composer-cache
+              run: |
+                  echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
+
+            - name: Cache packages dependencies
+              uses: actions/cache@v4
+              with:
+                  path: ${{ steps.composer-cache.outputs.dir }}
+                  key: ${{ runner.os }}-composer-packages-${{ matrix.php-version }}-${{ matrix.dependency-version }}-${{ matrix.symfony-version }}-${{ hashFiles('src/**/composer.json') }}
+                  restore-keys: |
+                      ${{ runner.os }}-composer-packages-${{ matrix.php-version }}-${{ matrix.dependency-version }}-${{ matrix.symfony-version }}
+
+            - name: Install root dependencies
+              uses: ramsey/composer-install@v3
+
+            - name: Install examples dependencies
+              run: cd examples && composer install && ../link
+
+            - name: Run commands examples
+              run: php examples/commands/stores.php

examples/.env

@@ -60,8 +60,13 @@ BRAVE_API_KEY=
 FIRECRAWL_HOST=https://api.firecrawl.dev
 FIRECRAWL_API_KEY=
 
-# For using MongoDB Atlas (store)
-MONGODB_URI=
+# For using Clickhouse (store)
+CLICKHOUSE_HOST=http://symfony:[email protected]:8123
+CLICKHOUSE_DATABASE=symfony
+CLICKHOUSE_TABLE=symfony
+
+# For using MongoDB Atlas / Community Edition (store)
+MONGODB_URI=mongodb://symfony:[email protected]:27017
 
 # For using Pinecone (store)
 PINECONE_API_KEY=

examples/commands/stores.php

@@ -0,0 +1,124 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+use Doctrine\DBAL\DriverManager;
+use Doctrine\DBAL\Tools\DsnParser;
+use MongoDB\Client as MongoDbClient;
+use Symfony\AI\Store\Bridge\Clickhouse\Store as ClickhouseStore;
+use Symfony\AI\Store\Bridge\Local\CacheStore;
+use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+use Symfony\AI\Store\Bridge\MariaDb\Store as MariaDbStore;
+use Symfony\AI\Store\Bridge\Meilisearch\Store as MeilisearchStore;
+use Symfony\AI\Store\Bridge\Milvus\Store as MilvusStore;
+use Symfony\AI\Store\Bridge\MongoDb\Store as MongoDbStore;
+use Symfony\AI\Store\Bridge\Neo4j\Store as Neo4jStore;
+use Symfony\AI\Store\Bridge\Postgres\Store as PostgresStore;
+use Symfony\AI\Store\Bridge\Qdrant\Store as QdrantStore;
+use Symfony\AI\Store\Bridge\SurrealDb\Store as SurrealDbStore;
+use Symfony\AI\Store\Bridge\Typesense\Store as TypesenseStore;
+use Symfony\AI\Store\Command\DropStoreCommand;
+use Symfony\AI\Store\Command\SetupStoreCommand;
+use Symfony\Component\Cache\Adapter\ArrayAdapter;
+use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Input\ArrayInput;
+use Symfony\Component\Console\Output\ConsoleOutput;
+use Symfony\Component\DependencyInjection\ServiceLocator;
+use Symfony\Component\HttpClient\HttpClient;
+
+$factories = [
+    'cache' => static fn (): CacheStore => new CacheStore(new ArrayAdapter(), cacheKey: '_commands'),
+    'clickhouse' => static fn (): ClickhouseStore => new ClickhouseStore(
+        HttpClient::createForBaseUri(env('CLICKHOUSE_HOST')),
+        env('CLICKHOUSE_DATABASE'),
+        env('CLICKHOUSE_TABLE'),
+    ),
+    'mariadb' => static fn (): MariaDbStore => MariaDbStore::fromDbal(
+        DriverManager::getConnection((new DsnParser())->parse(env('MARIADB_URI'))),
+        'my_table_openai',
+        'my_index',
+    ),
+    'memory' => static fn (): InMemoryStore => new InMemoryStore(),
+    'meilisearch' => static fn (): MeilisearchStore => new MeilisearchStore(
+        http_client(),
+        env('MEILISEARCH_HOST'),
+        env('MEILISEARCH_API_KEY'),
+        '_commands',
+    ),
+    'milvus' => static fn (): MilvusStore => new MilvusStore(
+        http_client(),
+        env('MILVUS_HOST'),
+        env('MILVUS_API_KEY'),
+        env('MILVUS_DATABASE'),
+        '_commands',
+    ),
+    'mongodb' => static fn (): MongoDbStore => new MongoDbStore(
+        client: new MongoDbClient(env('MONGODB_URI')),
+        databaseName: 'my-database',
+        collectionName: 'my-collection',
+        indexName: 'my-index',
+        vectorFieldName: 'vector',
+    ),
+    'neo4j' => static fn (): Neo4jStore => new Neo4jStore(
+        httpClient: http_client(),
+        endpointUrl: env('NEO4J_HOST'),
+        username: env('NEO4J_USERNAME'),
+        password: env('NEO4J_PASSWORD'),
+        databaseName: env('NEO4J_DATABASE'),
+        vectorIndexName: 'Movies',
+        nodeName: 'movies',
+    ),
+    'postgres' => static fn (): PostgresStore => PostgresStore::fromDbal(
+        DriverManager::getConnection((new DsnParser())->parse(env('POSTGRES_URI'))),
+        'my_table',
+    ),
+    'qdrant' => static fn (): QdrantStore => new QdrantStore(
+        http_client(),
+        env('QDRANT_HOST'),
+        env('QDRANT_SERVICE_API_KEY'),
+        '_commands',
+    ),
+    'surrealdb' => static fn (): SurrealDbStore => new SurrealDbStore(
+        httpClient: http_client(),
+        endpointUrl: env('SURREALDB_HOST'),
+        user: env('SURREALDB_USER'),
+        password: env('SURREALDB_PASS'),
+        namespace: 'default',
+        database: '_commands',
+        table: '_commands',
+    ),
+    'typesense' => static fn (): TypesenseStore => new TypesenseStore(
+        http_client(),
+        env('TYPESENSE_HOST'),
+        env('TYPESENSE_API_KEY'),
+        '_commands',
+    ),
+];
+
+$storesIds = array_keys($factories);
+
+$application = new Application();
+$application->setAutoExit(false);
+$application->add(new SetupStoreCommand(new ServiceLocator($factories), $storesIds));
+$application->add(new DropStoreCommand(new ServiceLocator($factories), $storesIds));
+
+foreach ($storesIds as $store) {
+    $application->run(new ArrayInput([
+        'command' => 'ai:store:setup',
+        'store' => $store,
+    ]), new ConsoleOutput());
+
+    $application->run(new ArrayInput([
+        'command' => 'ai:store:drop',
+        'store' => $store,
+    ]), new ConsoleOutput());
+}

examples/compose.yaml

@@ -1,20 +1,40 @@
 services:
+  chromadb:
+      image: chromadb/chroma
+      environment:
+          - IS_PERSISTENT=TRUE
+          - PERSIST_DIRECTORY=/chroma/chroma # this is the default path, change it as needed
+          - ANONYMIZED_TELEMETRY=FALSE
+      volumes:
+          - chroma_vlm:/chroma/chroma
+      ports:
+          - '8001:8000'
+
+  clickhouse:
+      image: clickhouse/clickhouse-server
+      environment:
+          CLICKHOUSE_DB: 'symfony'
+          CLICKHOUSE_USER: 'symfony'
+          CLICKHOUSE_PASSWORD: 'symfony'
+      ports:
+          - '8123:8123'
+
   mariadb:
-    image: mariadb:11.7
-    environment:
-      MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
-      MARIADB_DATABASE: my_database
-    ports:
-      - '3309:3306'
+      image: mariadb:11.7
+      environment:
+          MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
+          MARIADB_DATABASE: my_database
+      ports:
+          - '3309:3306'
 
   postgres:
-    image: pgvector/pgvector:0.8.0-pg17
-    environment:
-        POSTGRES_DB: my_database
-        POSTGRES_USER: postgres
-        POSTGRES_PASSWORD: postgres
-    ports:
-        - '5432:5432'
+      image: pgvector/pgvector:0.8.0-pg17
+      environment:
+          POSTGRES_DB: my_database
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+      ports:
+          - '5432:5432'
 
   meilisearch:
       image: getmeili/meilisearch:v1.15
@@ -23,6 +43,14 @@ services:
       ports:
           - '7700:7700'
 
+  mongodb:
+      image: mongodb/mongodb-community-server:7.0-ubi9
+      environment:
+          MONGO_INITDB_ROOT_USERNAME: 'symfony'
+          MONGO_INITDB_ROOT_PASSWORD: 'symfony'
+      ports:
+          - '27017:27017'
+
   qdrant:
       image: qdrant/qdrant
       environment:
@@ -80,9 +108,6 @@ services:
       environment:
           MINIO_ACCESS_KEY: minioadmin
           MINIO_SECRET_KEY: minioadmin
-      ports:
-          - '9001:9001'
-          - '9000:9000'
       volumes:
           - minio_vlm:/minio_data
       command: minio server /minio_data --console-address ":9001"
@@ -122,3 +147,4 @@ volumes:
     etcd_vlm:
     minio_vlm:
     milvus_vlm:
+    chroma_vlm:

examples/composer.json

@@ -8,6 +8,7 @@
         "async-aws/bedrock-runtime": "^1.1",
         "codewithkyrian/transformers": "^0.6.1",
         "doctrine/dbal": "^3.3|^4.0",
+        "mongodb/mongodb": "^2.1",
         "mrmysql/youtube-transcript": "^0.0.5",
         "php-http/discovery": "^1.20",
         "probots-io/pinecone-php": "^1.1",
@@ -18,6 +19,7 @@
         "symfony/cache": "^6.4|^7.0",
         "symfony/console": "^6.4|^7.0",
         "symfony/css-selector": "^6.4|^7.0",
+        "symfony/dependency-injection": "^7.3",
         "symfony/dom-crawler": "^6.4|^7.0",
         "symfony/dotenv": "^6.4|^7.0",
         "symfony/event-dispatcher": "^6.4|^7.0",