diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index da91e79e..0ba3448a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -19,7 +19,9 @@ "ghcr.io/azure/azure-dev/azd:latest": { "version": "1.10.1" }, - "ghcr.io/prulloac/devcontainer-features/ollama:1": {} + "ghcr.io/prulloac/devcontainer-features/ollama:1": {}, + // az CLI is helpful for being able to login correctly with DefaultAzureCredential: + "ghcr.io/devcontainers/features/azure-cli": {} }, // Configure tool-specific properties. "customizations": { @@ -46,10 +48,6 @@ "source.fixAll": "explicit" }, "editor.defaultFormatter": "charliermarsh.ruff" - }, - "files.exclude": { - ".ruff_cache": true, - ".pytest_cache": true } } } diff --git a/.env.sample b/.env.sample index f6c8774c..91881fa6 100644 --- a/.env.sample +++ b/.env.sample @@ -13,12 +13,15 @@ OPENAI_EMBED_HOST=azure # You also need to `azd auth login` if running this locally AZURE_OPENAI_ENDPOINT=https://YOUR-AZURE-OPENAI-SERVICE-NAME.openai.azure.com AZURE_OPENAI_VERSION=2024-03-01-preview -AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-35-turbo -AZURE_OPENAI_CHAT_MODEL=gpt-35-turbo +AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o-mini +AZURE_OPENAI_CHAT_MODEL=gpt-4o-mini AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-ada-002 AZURE_OPENAI_EMBED_MODEL=text-embedding-ada-002 -AZURE_OPENAI_EMBED_MODEL_DIMENSIONS=1536 +AZURE_OPENAI_EMBED_DIMENSIONS=1536 AZURE_OPENAI_EMBEDDING_COLUMN=embedding_ada002 +AZURE_OPENAI_EVAL_DEPLOYMENT=gpt-4 +AZURE_OPENAI_EVAL_MODEL=gpt-4 +AZURE_TENANT_ID= # Only needed when using key-based Azure authentication: AZURE_OPENAI_KEY= # Needed for OpenAI.com: diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ff5e930f..d608252d 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -80,6 +80,11 @@ jobs: cd ./src/frontend npm install npm run build + - name: cache mypy + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ./.mypy_cache + key: mypy${{ matrix.os }}-${{ matrix.python_version }}-${{ hashFiles('requirements-dev.txt', 'src/backend/requirements.txt', 'src/backend/pyproject.toml') }} - name: Run MyPy run: python3 -m mypy . - name: Run Pytest diff --git a/.github/workflows/azure-dev.yaml b/.github/workflows/azure-dev.yaml index 4cc9a3c8..70675e6b 100644 --- a/.github/workflows/azure-dev.yaml +++ b/.github/workflows/azure-dev.yaml @@ -3,8 +3,7 @@ name: Deploy with azd on: workflow_dispatch: push: - branches: - - main + branches: [ main ] # GitHub Actions workflow to deploy to Azure using azd # To configure required secrets for connecting to Azure, simply run `azd pipeline config` diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml new file mode 100644 index 00000000..73220be3 --- /dev/null +++ b/.github/workflows/evaluate.yaml @@ -0,0 +1,170 @@ +name: Evaluate + +on: + workflow_dispatch: + issue_comment: + types: [created] + +# Set up permissions for deploying with secretless Azure federated credentials +# https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication +permissions: + id-token: write + contents: read + +jobs: + evaluate: + if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') + runs-on: ubuntu-latest + env: + AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} + steps: + - name: Check for evaluate hash tag + if: contains(github.event.comment.body, '#evaluate') + run: | + echo "Comment contains #evaluate hashtag" + + - uses: actions/checkout@v4 + - name: Install PostgreSQL development libraries + run: | + sudo apt update + sudo apt install postgresql-server-dev-14 + - name: Setup postgres + uses: ikalnytskyi/action-setup-postgres@v6 + with: + username: admin + password: postgres + database: postgres + + - name: Install pgvector on MacOS/Linux using install-pgvector.sh + run: .github/workflows/install-pgvector.sh + + - name: Install python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install azd + uses: Azure/setup-azd@v1.0.0 + + - name: Install dependencies + run: | + python -m pip install -r requirements-dev.txt + + - name: Install app as editable app + run: | + python -m pip install -e src/backend + + - name: Setup local database with seed data + run: | + python ./src/backend/fastapi_app/setup_postgres_database.py + python ./src/backend/fastapi_app/setup_postgres_seeddata.py + env: + POSTGRES_HOST: localhost + POSTGRES_USERNAME: admin + POSTGRES_PASSWORD: postgres + POSTGRES_DATABASE: postgres + POSTGRES_SSL: disable + + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Build frontend + run: | + cd ./src/frontend + npm install + npm run build + + - name: Install python packages + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Login to Azure + uses: azure/login@v2 + with: + client-id: ${{ env.AZURE_CLIENT_ID }} + tenant-id: ${{ env.AZURE_TENANT_ID }} + subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} + + - name: Set az account + uses: azure/CLI@v2 + with: + inlineScript: | + az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}} + + - name: Log in with Azure (Federated Credentials) + if: ${{ env.AZURE_CLIENT_ID != '' }} + run: | + azd auth login ` + --client-id "$Env:AZURE_CLIENT_ID" ` + --federated-credential-provider "github" ` + --tenant-id "$Env:AZURE_TENANT_ID" + shell: pwsh + + - name: Provision Infrastructure + run: azd provision --no-prompt + env: + AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }} + + - name: Run local server in background + run: | + RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) + env: + OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} + OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} + AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} + AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} + AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }} + AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} + AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} + AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} + POSTGRES_HOST: localhost + POSTGRES_USERNAME: admin + POSTGRES_PASSWORD: postgres + POSTGRES_DATABASE: postgres + POSTGRES_SSL: disable + - name: Evaluate local RAG flow + run: | + python evals/evaluate.py + env: + OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} + AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} + AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} + AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }} + AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }} + - name: Upload server logs as build artifact + uses: actions/upload-artifact@v4 + with: + name: server_logs + path: ./serverlogs.out + + - name: Upload server error logs as build artifact + uses: actions/upload-artifact@v4 + with: + name: server_error_logs + path: ./serverlogs.err + + - name: Upload eval results as build artifact + uses: actions/upload-artifact@v4 + with: + name: eval_result + path: ./src/api/evaluate/eval_results.jsonl + + - name: GitHub Summary Step + if: ${{ success() }} + working-directory: ./src/api + run: | + echo "" >> $GITHUB_STEP_SUMMARY + + echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY + cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY diff --git a/.vscode/settings.json b/.vscode/settings.json index aed07e4c..6c6ce906 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,5 +27,13 @@ "tests" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "files.exclude": { + ".ruff_cache": true, + ".pytest_cache": true, + "__pycache__": true, + "htmlcov": true, + ".mypy_cache": true, + ".coverage": true + } } diff --git a/README.md b/README.md index 458ee38c..43069b89 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain azd up ``` - You will be asked to select two locations, first a region for most of the resources (Container Apps, PostgreSQL), then a region specifically for the Azure OpenAI models. This project uses the gpt-3.5-turbo (version 0125) and text-embedding-ada-002 models which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region accordingly. + You will be asked to select two locations, first a region for most of the resources (Container Apps, PostgreSQL), then a region specifically for the Azure OpenAI models. This project uses the gpt-4o-mini and text-embedding-ada-002 models which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region accordingly. ## Local Development @@ -207,6 +207,7 @@ Further documentation is available in the `docs/` folder: * [Using Entra auth with PostgreSQL tools](docs/using_entra_auth.md) * [Monitoring with Azure Monitor](docs/monitoring.md) * [Load testing](docs/loadtesting.md) +* [Evaluation](docs/evaluation.md) Please post in the issue tracker with any questions or issues. diff --git a/azure.yaml b/azure.yaml index bcc84608..0a74a2d6 100644 --- a/azure.yaml +++ b/azure.yaml @@ -39,12 +39,21 @@ hooks: pipeline: variables: - DEPLOY_AZURE_OPENAI - - AZURE_OPENAI_CHAT_MODEL + - OPENAI_CHAT_HOST + - OPENAI_EMBED_HOST + - AZURE_OPENAI_ENDPOINT + - AZURE_OPENAI_VERSION - AZURE_OPENAI_CHAT_DEPLOYMENT + - AZURE_OPENAI_CHAT_MODEL - AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION - AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY - - AZURE_OPENAI_EMBED_MODEL + - AZURE_OPENAI_EVAL_DEPLOYMENT_SKU - AZURE_OPENAI_EMBED_DEPLOYMENT + - AZURE_OPENAI_EMBED_MODEL - AZURE_OPENAI_EMBED_DEPLOYMENT_VERSION - AZURE_OPENAI_EMBED_DEPLOYMENT_CAPACITY + - AZURE_OPENAI_EMBED_DEPLOYMENT_SKU - AZURE_OPENAI_EMBED_DIMENSIONS + - AZURE_OPENAI_EMBEDDING_COLUMN + - AZURE_OPENAI_EVAL_DEPLOYMENT + - AZURE_OPENAI_EVAL_MODEL diff --git a/docs/evaluation.md b/docs/evaluation.md new file mode 100644 index 00000000..4a6c5bd1 --- /dev/null +++ b/docs/evaluation.md @@ -0,0 +1,75 @@ +# Evaluating the RAG answer quality + +## Deploy a GPT-4 model + + +1. Run this command to tell `azd` to deploy a GPT-4 model for evaluation: + + ```shell + azd env set DEPLOY_EVAL_MODEL true + ``` + +2. Set the capacity to the highest possible value to ensure that the evaluation runs quickly. + + ```shell + azd env set AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY 100 + ``` + + By default, that will provision a `gpt-4` model, version `turbo-2024-04-09`. To change those settings, set the `AZURE_OPENAI_EVAL_DEPLOYMENT` and `AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION` environment variables. + +3. Then, run the following command to provision the model: + + ```shell + azd provision + ``` + +## Setup the evaluation environment + +Install all the dependencies for the evaluation script by running the following command: + +```bash +pip install -r requirements-dev.txt +``` + +## Generate ground truth data + +Generate ground truth data by running the following command: + +```bash +python evals/generate.py +``` + +Review the generated data after running that script, removing any question/answer pairs that don't seem like realistic user input. + +## Evaluate the RAG answer quality + +Review the configuration in `evals/eval_config.json` to ensure that everything is correctly setup. You may want to adjust the metrics used. [TODO: link to evaluator docs] + +By default, the evaluation script will evaluate every question in the ground truth data. +Run the evaluation script by running the following command: + +```bash +python evals/evaluate.py +``` + +## Review the evaluation results + +The evaluation script will output a summary of the evaluation results, inside the `evals/results` directory. + +You can see a summary of results across all evaluation runs by running the following command: + +```bash +python -m evaltools summary evals/results +``` + +Compare answers across runs by running the following command: + +```bash +python -m evaltools diff evals/results/baseline/ +``` + +## Run the evaluation in GitHub actions + + +# TODO: Add GPT-4 deployment with high capacity for evaluation +# TODO: Add CI workflow that can be triggered to run the evaluate on the local app diff --git a/evals/eval_config.json b/evals/eval_config.json new file mode 100644 index 00000000..c51f9e79 --- /dev/null +++ b/evals/eval_config.json @@ -0,0 +1,16 @@ +{ + "testdata_path": "ground_truth.jsonl", + "results_dir": "results/experiment", + "requested_metrics": ["gpt_groundedness", "gpt_relevance", "answer_length", "latency", "citation_match"], + "target_url": "http://127.0.0.1:8000/chat", + "target_parameters": { + "overrides": { + "use_advanced_flow": true, + "top": 3, + "retrieval_mode": "hybrid", + "temperature": 0.3 + } + }, + "target_response_answer_jmespath": "message.content", + "target_response_context_jmespath": "context.data_points" +} diff --git a/evals/evaluate.py b/evals/evaluate.py new file mode 100644 index 00000000..812a3f09 --- /dev/null +++ b/evals/evaluate.py @@ -0,0 +1,60 @@ +import logging +import os +from pathlib import Path +from typing import Any + +import azure.identity +from dotenv import load_dotenv +from evaltools.eval.evaluate import run_evaluate_from_config +from rich.logging import RichHandler + +logger = logging.getLogger("ragapp") + + +def get_openai_config() -> dict: + openai_config: dict[str, Any] + if os.environ.get("OPENAI_CHAT_HOST") == "azure": + azure_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] + azure_deployment = os.environ["AZURE_OPENAI_EVAL_DEPLOYMENT"] + if os.environ.get("AZURE_OPENAI_KEY"): + logger.info("Using Azure OpenAI Service with API Key from AZURE_OPENAI_KEY") + openai_config = { + "azure_endpoint": azure_endpoint, + "azure_deployment": azure_deployment, + "api_key": os.environ["AZURE_OPENAI_KEY"], + } + else: + if tenant_id := os.getenv("AZURE_TENANT_ID"): + logger.info("Authenticating to Azure using Azure Developer CLI Credential for tenant %s", tenant_id) + azure_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=tenant_id, process_timeout=60) + else: + logger.info("Authenticating to Azure using Azure Developer CLI Credential") + azure_credential = azure.identity.AzureDeveloperCliCredential(process_timeout=60) + openai_config = { + "azure_endpoint": azure_endpoint, + "azure_deployment": azure_deployment, + "credential": azure_credential, + } + # azure-ai-evaluate will call DefaultAzureCredential behind the scenes, + # so we must be logged in to Azure CLI with the correct tenant + openai_config["model"] = os.environ["AZURE_OPENAI_EVAL_MODEL"] + else: + logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY") + openai_config = {"api_key": os.environ["OPENAICOM_KEY"], "model": "gpt-4"} + return openai_config + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, format="%(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True)] + ) + load_dotenv(".env", override=True) + + openai_config = get_openai_config() + # TODO: specify the localhost URL using argument + # TODO: specify the experiment name (based on PR number) + # TODO: Specify the num questions using argument + + run_evaluate_from_config( + working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=2 + ) diff --git a/evals/generate_ground_truth.py b/evals/generate_ground_truth.py new file mode 100644 index 00000000..c0c1e0a6 --- /dev/null +++ b/evals/generate_ground_truth.py @@ -0,0 +1,95 @@ +import logging +import os +from collections.abc import Generator +from pathlib import Path + +from azure.identity import AzureDeveloperCliCredential +from dotenv import load_dotenv +from evaltools.gen.generate import generate_test_qa_data +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session + +from fastapi_app.postgres_models import Item + +logger = logging.getLogger("ragapp") + + +def source_retriever() -> Generator[dict, None, None]: + # Connect to the database + DBHOST = os.environ["POSTGRES_HOST"] + DBUSER = os.environ["POSTGRES_USERNAME"] + DBPASS = os.environ["POSTGRES_PASSWORD"] + DBNAME = os.environ["POSTGRES_DATABASE"] + DATABASE_URI = f"postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}" + engine = create_engine(DATABASE_URI, echo=False) + with Session(engine) as session: + # Fetch all products for a particular type + item_types = session.scalars(select(Item.type).distinct()) + for item_type in item_types: + records = list(session.scalars(select(Item).filter(Item.type == item_type).order_by(Item.id))) + # logger.info(f"Processing database records for type: {item_type}") + # yield { + # "citations": " ".join([f"[{record.id}] - {record.name}" for record in records]), + # "content": "\n\n".join([record.to_str_for_rag() for record in records]), + # } + # Fetch each item individually + records = list(session.scalars(select(Item).order_by(Item.id))) + for record in records: + logger.info(f"Processing database record: {record.name}") + yield {"id": record.id, "content": record.to_str_for_rag()} + + +def source_to_text(source) -> str: + return source["content"] + + +def answer_formatter(answer, source) -> str: + return f"{answer} [{source['id']}]" + + +def get_openai_config_dict() -> dict: + """Return a dictionary with OpenAI configuration based on environment variables.""" + OPENAI_CHAT_HOST = os.getenv("OPENAI_CHAT_HOST") + if OPENAI_CHAT_HOST == "azure": + if api_key := os.getenv("AZURE_OPENAI_KEY"): + logger.info("Using Azure OpenAI Service with API Key from AZURE_OPENAI_KEY") + api_key = os.environ["AZURE_OPENAI_KEY"] + else: + logger.info("Using Azure OpenAI Service with Azure Developer CLI Credential") + azure_credential = AzureDeveloperCliCredential(process_timeout=60) + api_key = azure_credential.get_token("https://cognitiveservices.azure.com/.default").token + openai_config = { + "api_type": "azure", + "api_base": os.environ["AZURE_OPENAI_ENDPOINT"], + "api_key": api_key, + "api_version": os.environ["AZURE_OPENAI_VERSION"], + "deployment": os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"], + "model": os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"], + } + elif OPENAI_CHAT_HOST == "ollama": + raise NotImplementedError("Ollama OpenAI Service is not supported. Switch to Azure or OpenAI.com") + else: + logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY") + openai_config = { + "api_type": "openai", + "api_key": os.environ["OPENAICOM_KEY"], + "model": os.environ["OPENAICOM_CHAT_MODEL"], + "deployment": "none-needed-for-openaicom", + } + return openai_config + + +if __name__ == "__main__": + logging.basicConfig(level=logging.WARNING) + logger.setLevel(logging.INFO) + load_dotenv(".env", override=True) + + generate_test_qa_data( + openai_config=get_openai_config_dict(), + num_questions_total=202, + num_questions_per_source=2, + output_file=Path(__file__).parent / "ground_truth.jsonl", + source_retriever=source_retriever, + source_to_text=source_to_text, + answer_formatter=answer_formatter, + ) diff --git a/evals/ground_truth.jsonl b/evals/ground_truth.jsonl new file mode 100644 index 00000000..c6f63907 --- /dev/null +++ b/evals/ground_truth.jsonl @@ -0,0 +1,199 @@ +{"question": "What is the description of the Wanderer Black Hiking Boots?", "truth": "The Wanderer Black Hiking Boots by Daybird are made with a waterproof leather upper and a durable rubber sole for superior traction. They feature a cushioned insole and padded collar for all-day comfort during outdoor adventures. [1]"} +{"question": "What is the price and brand of the Wanderer Black Hiking Boots?", "truth": "The Wanderer Black Hiking Boots are priced at 109.99 and are from the brand Daybird. [1]"} +{"question": "What are the key features of the Summit Pro Harness by Gravitator?", "truth": "The Summit Pro Harness by Gravitator is a lightweight and durable climbing harness with adjustable leg loops and waist belt for a customized fit. It also features a reinforced tie-in point and strong webbing loops for safety. [2]"} +{"question": "What is the price of the Summit Pro Harness by Gravitator and what type of product is it?", "truth": "The price of the Summit Pro Harness by Gravitator is 89.99. It is a climbing harness designed for conquering new heights with style and safety in mind. [2]"} +{"question": "What are the key features of the Alpine Fusion Goggles from WildRunner?", "truth": "The Alpine Fusion Goggles from WildRunner offer full UV protection, anti-fog lenses, a stylish silver frame with orange lenses, adjustable straps for a secure fit, and soft foam padding for comfort during skiing or snowboarding. [3]"} +{"question": "What is the price and brand of the Alpine Fusion Goggles?", "truth": "The Alpine Fusion Goggles from WildRunner are priced at 79.99 and are part of the WildRunner brand. [3]"} +{"question": "What is the description of the Expedition Backpack by Quester?", "truth": "The Expedition Backpack by Quester is described as a must-have for every outdoor enthusiast with its spacious interior, multiple pockets, and durable nylon fabric that can withstand tough conditions. It also features orange accents for added style. [4]"} +{"question": "What is the price of the Expedition Backpack by Quester?", "truth": "The price of the Expedition Backpack by Quester is $129.99. [4]"} +{"question": "What is the description of the Blizzard Rider Snowboard by B&R?", "truth": "The Blizzard Rider Snowboard by B&R is a versatile snowboard perfect for riders of all levels, featuring a medium flex and twin shape. It has a sleek black and blue color scheme and is suitable for carving turns or hitting the terrain park. [5]"} +{"question": "What is the price and brand of the Blizzard Rider Snowboard?", "truth": "The price of the Blizzard Rider Snowboard is 299.99, and it is manufactured by the brand B&R under the type Ski/boarding. [5]"} +{"question": "What are the key features of the Carbon Fiber Trekking Poles by Raptor Elite?", "truth": "The Carbon Fiber Trekking Poles by Raptor Elite are designed with lightweight carbon fiber shafts for excellent support and durability. They have comfortable and adjustable cork grips for a secure hold, and blue accents for a stylish touch. Additionally, they are compact and collapsible for easy transport and storage. [6]"} +{"question": "What is the price of the Carbon Fiber Trekking Poles by Raptor Elite?", "truth": "The price of the Carbon Fiber Trekking Poles by Raptor Elite is $69.99. [6]"} +{"question": "What are the key features of the Explorer 45L Backpack by Solstix?", "truth": "The Explorer 45L Backpack by Solstix is made with waterproof and tear-resistant materials, making it suitable for harsh weather conditions. It has a spacious main compartment, multiple pockets for organization, and a green and black color scheme for a rugged look. [7]"} +{"question": "How much does the Explorer 45L Backpack by Solstix cost and what brand manufactures it?", "truth": "The Explorer 45L Backpack by Solstix is priced at 149.99 and is manufactured by the brand Solstix. [7]"} +{"question": "What are the key features of the Frostbite Insulated Jacket by Grolltex?", "truth": "The Frostbite Insulated Jacket by Grolltex features a water-resistant outer shell, lightweight insulation, adjustable hood, multiple pockets, black and gray color combination, and the Grolltex logo for a touch of sophistication. [8]"} +{"question": "What is the price of the Frostbite Insulated Jacket and which brand manufactures it?", "truth": "The Frostbite Insulated Jacket is priced at 179.99 and is manufactured by Grolltex. [8]"} +{"question": "What features does the VenturePro GPS Watch by AirStrider offer?", "truth": "The VenturePro GPS Watch by AirStrider features a built-in GPS, altimeter, and compass, allowing users to navigate with confidence in any terrain. [9]"} +{"question": "What makes the VenturePro GPS Watch a must-have for adventurers?", "truth": "The VenturePro GPS Watch is described as rugged, durable, stylish, and practical with its sleek black design, easy-to-read display, and essential navigation features like GPS, altimeter, and compass. [9]"} +{"question": "What are the key features of the Trailblazer Bike Helmet by Green Equipment?", "truth": "The Trailblazer Bike Helmet by Green Equipment is lightweight and durable, with an adjustable fit system and ventilation for added comfort. It also features a vibrant green color and sleek design. [10]"} +{"question": "What types of cycling is the Trailblazer Bike Helmet suitable for?", "truth": "The Trailblazer Bike Helmet is perfect for all types of cycling, from mountain biking to road cycling. [10]"} +{"question": "What are the key features of the Vertical Journey Climbing Shoes from WildRunner?", "truth": "The Vertical Journey Climbing Shoes from WildRunner feature an aggressive down-turned toe, sticky rubber outsole, and a reinforced heel cup for added support, providing ultimate performance on challenging climbing routes. [11]"} +{"question": "What is the price of the Vertical Journey Climbing Shoes from WildRunner?", "truth": "The price of the Vertical Journey Climbing Shoes from WildRunner is $129.99. [11]"} +{"question": "What is the description of the Powder Pro Snowboard by Zephyr?", "truth": "The Powder Pro Snowboard by Zephyr is designed for the ultimate ride through deep snow with its floating camber for effortless turns and smooth maneuverability. Its lightweight carbon fiber construction ensures maximum control at high speeds, making it a must-have for any backcountry shredder. [12]"} +{"question": "What are the details provided about the Powder Pro Snowboard's price, brand, and type?", "truth": "The Powder Pro Snowboard by Zephyr is priced at 399.0 and belongs to the brand Zephyr. It falls under the type Ski/boarding. [12]"} +{"question": "What are some key features of the Trailblaze hiking backpack?", "truth": "The Trailblaze hiking backpack features an ergonomic design, adjustable straps for a comfortable fit, ample storage space, and gear protection. It is priced at $89.99 and belongs to the Daybird brand under the category of Bags. [13]"} +{"question": "What are the key features of the Stellar Duffle Bag from Gravitator?", "truth": "The Stellar Duffle Bag from Gravitator is made from waterproof nylon, available in sleek black, and features multiple internal pockets and a separate shoe compartment to keep belongings organized. It also has an adjustable shoulder strap and reinforced handles for functionality and style. [14]"} +{"question": "How much does the Stellar Duffle Bag from Gravitator cost and what type of product is it?", "truth": "The Stellar Duffle Bag from Gravitator is priced at $59.99 and it falls under the category of Bags. [14]"} +{"question": "What are the key features of the Summit Pro Insulated Jacket by Raptor Elite?", "truth": "The Summit Pro Insulated Jacket by Raptor Elite is designed to keep you warm and dry in extreme conditions with its waterproof and breathable construction, heat-sealed seams, insulation made from recycled materials, and eco-friendly yet high-performance design. It also includes a removable hood, plenty of storage pockets, and is available in vibrant red. [15]"} +{"question": "What is the price of the Summit Pro Insulated Jacket by Raptor Elite?", "truth": "The price of the Summit Pro Insulated Jacket by Raptor Elite is $249.99. [15]"} +{"question": "What features do the Solstix Expedition 2022 Goggles offer?", "truth": "The Solstix Expedition 2022 Goggles provide clear vision and optimal protection on the slopes with features like an anti-fog lens, UV protection, and a comfortable foam lining. They ensure a great fit and unrestricted vision even in challenging conditions. [16]"} +{"question": "What is the price and brand of the Expedition 2022 Goggles?", "truth": "The price of the Expedition 2022 Goggles is 89.0 and the brand is Solstix. [16]"} +{"question": "What are the key features of the Apex Climbing Harness by Legend?", "truth": "The Apex Climbing Harness by Legend is a lightweight and durable harness designed for maximum comfort and safety. It features adjustable leg loops, a contoured waistbelt, and a secure buckle system for a secure fit during climbing sessions. [17]"} +{"question": "What is the price of the Apex Climbing Harness by Legend and what color is it available in?", "truth": "The price of the Apex Climbing Harness by Legend is 89.99. It is available in bold orange and also includes gear loops for easy access to climbing equipment. [17]"} +{"question": "What are the key features of the Alpine Tech Crampons by Grolltex?", "truth": "The Alpine Tech Crampons by Grolltex are made from strong and lightweight stainless steel, providing excellent traction and stability on icy mountain terrains. They have a simple adjustment system for easy fitting and quick attachment to most hiking boots. [18]"} +{"question": "What is the price and brand of the Alpine Tech Crampons?", "truth": "The price of the Alpine Tech Crampons is 149.0 and they are manufactured by Grolltex. [18]"} +{"question": "What are the key features of the EcoTrail Running Shoes?", "truth": "The EcoTrail Running Shoes are made from recycled materials, offering a lightweight, breathable, and flexible design in an earthy green color. They also feature a durable Vibram outsole and cushioned midsole for optimal comfort and grip on any trail. [19]"} +{"question": "What is the price of the EcoTrail Running Shoes and who is the brand behind them?", "truth": "The price of the EcoTrail Running Shoes is 119.99, and they are produced by the brand Green Equipment. [19]"} +{"question": "What are the key features of the Explorer Biking Computer by B&R?", "truth": "The Explorer Biking Computer by B&R is equipped with an intuitive touchscreen display, GPS capabilities for tracking routes, monitoring performance metrics, and receiving turn-by-turn directions. Its sleek black design and waterproof construction make it a reliable companion for cycling adventures. [20]"} +{"question": "What is the price and brand of the Explorer Biking Computer?", "truth": "The Explorer Biking Computer by B&R is priced at 199.99 and belongs to the brand B&R. [20]"} +{"question": "What are the key features of the Legend Trailblazer Black Hiking Shoes?", "truth": "The Legend Trailblazer Black Hiking Shoes feature a waterproof GORE-TEX lining, Vibram rubber outsole for enhanced traction, and a reinforced toe cap for added protection. [21]"} +{"question": "What is the price of the Legend Trailblazer Black Hiking Shoes?", "truth": "The price of the Legend Trailblazer Black Hiking Shoes is $129.99. [21]"} +{"question": "What are some key features of the Raptor Elite Venture 2022 Snowboard?", "truth": "The Raptor Elite Venture 2022 Snowboard features a responsive camber profile, carbon fiber laminates for enhanced stability, and a sintered base for maximum speed. It is designed as a true all-mountain performer suitable for riders of all levels. [22]"} +{"question": "What is the price and brand of the Venture 2022 Snowboard?", "truth": "The price of the Venture 2022 Snowboard is 499.0 and the brand is Raptor Elite. [22]"} +{"question": "What are the key features of the Zephyr Summit Pro Climbing Harness?", "truth": "The Zephyr Summit Pro Climbing Harness features 30kN rated webbing, speed-adjust buckles, multiple gear loops for organization, a lightweight design for comfort, and freedom of movement. [23]"} +{"question": "Who is the target audience for the Zephyr Summit Pro Climbing Harness?", "truth": "The Zephyr Summit Pro Climbing Harness is designed for professional climbers who demand reliability and performance. [23]"} +{"question": "What is the description of the Ridgevent Stealth Hiking Backpack?", "truth": "The WildRunner Ridgevent Stealth Hiking Backpack is described as the ultimate companion for outdoor adventures, combining style with functionality. It is made from durable nylon, features multiple compartments, and offers ample storage space for essentials. [24]"} +{"question": "What is the price and brand of the Ridgevent Stealth Hiking Backpack?", "truth": "The price of the Ridgevent Stealth Hiking Backpack is 69.99, and the brand is WildRunner. [24]"} +{"question": "What are the key features of the Daybird Stealth Lite Bike Helmet?", "truth": "The Daybird Stealth Lite Bike Helmet features a lightweight in-mold construction, adjustable retention system, and multiple ventilation channels for optimal airflow. [25]"} +{"question": "What is the price of the Daybird Stealth Lite Bike Helmet?", "truth": "The price of the Daybird Stealth Lite Bike Helmet is $89.99. [25]"} +{"question": "What is the description of the Gravity Beam Climbing Rope?", "truth": "The Gravitator Gravity Beam Climbing Rope is a high-quality climbing rope with kernmantle construction, offering excellent strength and durability. It is vibrant yellow in color, highly visible, and suitable for tackling steep rock faces or frozen waterfalls. [26]"} +{"question": "What are some key details about the Gravity Beam Climbing Rope?", "truth": "The Gravity Beam Climbing Rope is priced at $179.99, branded as Gravitator, and classified as a climbing rope. [26]"} +{"question": "What are the key features of the EcoLodge 45L Travel Backpack?", "truth": "The EcoLodge 45L Travel Backpack is a sustainable and versatile option made from recycled materials. It features multiple compartments, a padded laptop sleeve, and durable zippers. [27]"} +{"question": "What is the price of the Green Equipment EcoLodge 45L Travel Backpack?", "truth": "The price of the Green Equipment EcoLodge 45L Travel Backpack is $129.0. [27]"} +{"question": "What are the key features of the Solstix Alpine Peak Down Jacket?", "truth": "The Solstix Alpine Peak Down Jacket is crafted for extreme cold conditions with high-quality goose down insulation for exceptional warmth and comfort. It features a bold red color, sleek design, removable hood, adjustable cuffs, and multiple zippered pockets for storage. [28]"} +{"question": "What is the price of the Solstix Alpine Peak Down Jacket?", "truth": "The price of the Solstix Alpine Peak Down Jacket is $249.99. [28]"} +{"question": "What are the key features of the B&R Pulse Recon Tactical GPS Watch?", "truth": "The B&R Pulse Recon Tactical GPS Watch features a built-in GPS, altimeter, compass, and multiple sports modes. It is designed for outdoor enthusiasts and comes in a military green color with a durable construction. [29]"} +{"question": "What is the price of the B&R Pulse Recon Tactical GPS Watch?", "truth": "The price of the B&R Pulse Recon Tactical GPS Watch is $169.0. [29]"} +{"question": "What features do the Gravitator Zero Gravity Ski Goggles offer?", "truth": "The Gravitator Zero Gravity Ski Goggles offer an anti-fog coating, 100% UV protection, and an adjustable strap for a secure fit. [30]"} +{"question": "What is the price of the Zero Gravity Ski Goggles and what brand manufactures them?", "truth": "The price of the Zero Gravity Ski Goggles is $79.99, and they are manufactured by the brand Gravitator. [30]"} +{"question": "What are the features of the Guardian Blue Chalk Bag by Legend?", "truth": "The Guardian Blue Chalk Bag by Legend features a spacious compartment for chalk, a drawstring closure, and a waist belt for easy access while climbing. It also comes in a vibrant blue color for a stylish touch to your climbing gear. [31]"} +{"question": "What is the price of the Guardian Blue Chalk Bag by Legend?", "truth": "The price of the Guardian Blue Chalk Bag by Legend is $21.99. [31]"} +{"question": "What is the description of the Cosmic Purple Snowboard by Gravitator?", "truth": "The Cosmic Purple Snowboard by Gravitator is a freestyle board that offers a perfect balance of control and maneuverability. Its bright purple design is complemented by the Gravitator emblem, making it stand out on the mountain. [32]"} +{"question": "What is the price of the Cosmic Purple Snowboard and what brand manufactures it?", "truth": "The price of the Cosmic Purple Snowboard is 419.99 and it is manufactured by the brand Gravitator. [32]"} +{"question": "What is the description of the Venture Grey Trail Shoes by WildRunner?", "truth": "The Venture Grey Trail Shoes by WildRunner are constructed with breathable mesh and a rugged outsole, providing excellent traction and long-lasting durability. They are designed for hitting the trails in style and comfort. [33]"} +{"question": "What is the price of the Venture Grey Trail Shoes and who is the brand?", "truth": "The price of the Venture Grey Trail Shoes by WildRunner is $79.99. The brand of these shoes is WildRunner. [33]"} +{"question": "What are the key features of the Velocity Red Bike Helmet by AirStrider?", "truth": "The Velocity Red Bike Helmet by AirStrider is a lightweight helmet with a streamlined design, adjustable straps, and ventilation channels for optimal airflow. [34]"} +{"question": "What is the price of the Velocity Red Bike Helmet and what type of product is it?", "truth": "The price of the Velocity Red Bike Helmet is $54.99, and it is a cycling helmet from the brand AirStrider. [34]"} +{"question": "What is the description of the Carbon Fiber Trekking Poles?", "truth": "The Raptor Elite Carbon Fiber Trekking Poles are lightweight and durable, providing stability on various terrains and reducing strain on joints. They feature an ergonomic grip and adjustable length, making them essential for outdoor adventures. [35]"} +{"question": "What is the price of the Carbon Fiber Trekking Poles and what brand manufactures them?", "truth": "The price of the Carbon Fiber Trekking Poles is $99.0, and they are manufactured by the brand Raptor Elite. [35]"} +{"question": "What is the description of the Excursion 20L Daypack by B&R?", "truth": "The Excursion 20L Daypack by B&R is a spacious pack made from durable waterproof nylon, featuring multiple pockets, adjustable straps, and a padded back for enhanced comfort. It is described as the perfect companion for hiking or camping trips. [36]"} +{"question": "What are some key features of the Excursion 20L Daypack by B&R?", "truth": "Some key features of the Excursion 20L Daypack by B&R include its durable waterproof nylon material, multiple pockets for storage, adjustable straps for comfort, a padded back, and a sleek design in a versatile white color. [36]"} +{"question": "What is the description of the Stormbreaker Waterproof Jacket by Zephyr?", "truth": "The Stormbreaker Waterproof Jacket by Zephyr offers superior protection with its fully waterproof and windproof design. It comes in a bold red color with the Zephyr logo, adding a stylish touch to your outdoor look. [37]"} +{"question": "What is the price of the Stormbreaker Waterproof Jacket by Zephyr?", "truth": "The price of the Stormbreaker Waterproof Jacket by Zephyr is $139.99. [37]"} +{"question": "What are the key features of the Pathfinder Portable GPS by Solstix?", "truth": "The Pathfinder Portable GPS by Solstix is a compact and reliable navigation device with a color display, preloaded maps, and advanced tracking capabilities. It also offers an intuitive interface and long battery life for confident exploration. [38]"} +{"question": "How much does the Pathfinder Portable GPS by Solstix cost and what brand manufactures it?", "truth": "The price of the Pathfinder Portable GPS by Solstix is 199.0. It is manufactured by the brand Solstix and falls under the type of Navigation device. [38]"} +{"question": "What are the features of the Midnight Blue Goggles by Daybird?", "truth": "The Midnight Blue Goggles by Daybird offer a wide field of vision, anti-fog coating, UV protection, and a blue tinted lens for style. [39]"} +{"question": "What is the price of the Midnight Blue Goggles by Daybird?", "truth": "The price of the Midnight Blue Goggles by Daybird is 89.99. [39]"} +{"question": "What are the key features of the EcoTrek Trail Running Shoes by Green Equipment?", "truth": "The EcoTrek Trail Running Shoes by Green Equipment are designed with eco-friendly materials, offer a comfortable fit, responsive cushioning, and a durable outsole for optimal grip on rugged terrains. The forest green color adds a refreshing touch to the outdoor look. [40]"} +{"question": "What is the price and brand of the EcoTrek Trail Running Shoes?", "truth": "The EcoTrek Trail Running Shoes by Green Equipment are priced at $99.99 and are part of the Green Equipment brand. [40]"} +{"question": "What are the key features of the Trekker Clear Hiking Shoes from WildRunner?", "truth": "The Trekker Clear Hiking Shoes from WildRunner are designed with a waterproof and breathable upper fabric, a rugged carbon-infused sole for excellent traction, and a shock-absorbing midsole for enhanced comfort on long hikes. [41]"} +{"question": "What is the price and brand of the Trekker Clear Hiking Shoes?", "truth": "The Trekker Clear Hiking Shoes are priced at 84.99 and are from the brand WildRunner. [41]"} +{"question": "What are the key features of the Gravity 5000 All-Mountain Skis?", "truth": "The Gravity 5000 All-Mountain Skis by Gravitator feature a versatile design that excels in all conditions, a lightweight wood core, carbon inserts for responsiveness, and ABS sidewalls for added durability. They come in a striking blue color. [42]"} +{"question": "What is the price and brand of the Gravity 5000 All-Mountain Skis?", "truth": "The price of the Gravity 5000 All-Mountain Skis is 699.0 and they are manufactured by the brand Gravitator. [42]"} +{"question": "What are the key features of the Glacier Frost Snowboard from Legend?", "truth": "The Glacier Frost Snowboard from Legend features a hybrid camber profile for excellent edge control and maneuverability, a carbon fiber composite core for lightweight strength and optimal flex, and it is available in a cool white color with a vibrant frost graphic. [43]"} +{"question": "What is the price of the Glacier Frost Snowboard and what type of product is it?", "truth": "The price of the Glacier Frost Snowboard from Legend is 419.99. It is a high-performance snowboard designed for skiing and snowboarding. [43]"} +{"question": "What are the key features of the Summit Pro Climbing Harness by Raptor Elite?", "truth": "The Summit Pro Climbing Harness by Raptor Elite features a lightweight and breathable construction, adjustable leg loops for a personalized fit, reinforced tie-in points for safety and durability, and a vivid green color for style. [44]"} +{"question": "What is the price of the Summit Pro Climbing Harness by Raptor Elite?", "truth": "The price of the Summit Pro Climbing Harness by Raptor Elite is $109.99. [44]"} +{"question": "What are the key features of the Elemental 3-Season Tent by Solstix?", "truth": "The Elemental 3-Season Tent by Solstix is a lightweight and compact tent perfect for backpacking adventures. It offers ample space for two people, features a durable waterproof fabric, sturdy aluminum poles, ventilation panels for optimal airflow, and a vibrant green color for visibility. [45]"} +{"question": "What is the price and brand of the Elemental 3-Season Tent?", "truth": "The price of the Elemental 3-Season Tent by Solstix is 189.99, and the brand is Solstix. [45]"} +{"question": "What are the key features of the Zenith Cycling Jersey by B&R?", "truth": "The Zenith Cycling Jersey by B&R is a high-performance jersey made from moisture-wicking fabric to keep cyclists cool and dry. It features a full-length zipper, three rear pockets for storage, and reflective accents for increased visibility in low-light conditions. [46]"} +{"question": "What is the price of the Zenith Cycling Jersey by B&R and in what color is it available?", "truth": "The Zenith Cycling Jersey by B&R is priced at 64.99 and is available in a vibrant red color. [46]"} +{"question": "What are the key features of the Edge Pro Ice Axe from Grolltex?", "truth": "The Edge Pro Ice Axe from Grolltex features a lightweight aluminum shaft, a durable stainless steel pick, and a comfortable hand grip for maximum control. It is designed for tackling steep ice walls and mixed alpine terrain. [47]"} +{"question": "What is the price and brand of the Edge Pro Ice Axe?", "truth": "The Edge Pro Ice Axe is priced at 129.0 and is from the brand Grolltex. [47]"} +{"question": "What are the key features of the Trailblazer 45L Backpack by Zephyr?", "truth": "The Trailblazer 45L Backpack by Zephyr features multiple compartments for easy organization, adjustable shoulder straps and hip belt for a customized fit, and durable waterproof construction for ultimate protection against the elements. It comes in a classic yellow color that is timeless and versatile. [48]"} +{"question": "How much does the Trailblazer 45L Backpack by Zephyr cost and what brand manufactures it?", "truth": "The Trailblazer 45L Backpack by Zephyr is priced at $124.99 and is manufactured by the brand Zephyr. [48]"} +{"question": "What are the key features of the Arctic Shield Insulated Jacket by Daybird?", "truth": "The Arctic Shield Insulated Jacket by Daybird features a water-resistant outer shell, insulated fill for exceptional warmth, and a detachable hood for added versatility. It comes in a sleek pink color, perfect for any outdoor occasion. [49]"} +{"question": "How much does the Arctic Shield Insulated Jacket by Daybird cost?", "truth": "The price of the Arctic Shield Insulated Jacket by Daybird is $169.99. [49]"} +{"question": "What are the key features of the Astro GPS Navigator by AirStrider?", "truth": "The Astro GPS Navigator by AirStrider comes loaded with topographic maps, GPS tracking, waypoint storage, and a long-lasting battery. It is equipped with a high-resolution color display for easy navigation in any lighting condition. [50]"} +{"question": "What is the price and color option available for the Astro GPS Navigator by AirStrider?", "truth": "The Astro GPS Navigator by AirStrider is available in a sleek gray color and is priced at 249.99. [50]"} +{"question": "What is the description of the SummitStone Chalk Bag?", "truth": "The SummitStone Chalk Bag in forest green is a durable and compact bag designed to keep climbers' hands dry and provide easy access to chalk. It features a drawstring closure, adjustable waist strap, and a Loop-Slider buckle for easy attachment to harnesses. [51]"} +{"question": "What is the price of the SummitStone Chalk Bag?", "truth": "The price of the SummitStone Chalk Bag is $29.99. [51]"} +{"question": "What are the features of the TrailHug 50L Backpack?", "truth": "The TrailHug 50L Backpack in navy blue is made from lightweight and water-resistant nylon. It features adjustable padded straps, multiple compartments for organized storage, a breathable back panel for added comfort, and a built-in rain cover for unexpected showers. [52]"} +{"question": "What is the price of the TrailHug 50L Backpack and what brand does it belong to?", "truth": "The TrailHug 50L Backpack is priced at 129.99 and belongs to the Legend brand. [52]"} +{"question": "What are the key features of the Raven Swift Snowboard?", "truth": "The Raven Swift Snowboard features a camber profile for stability and pop, a medium flex for smooth turns and responsive control. It is designed with a striking white color and black logo, suitable for riders of all skill levels. [53]"} +{"question": "What is the price and brand of the Raven Swift Snowboard?", "truth": "The Raven Swift Snowboard is priced at 349.0 and belongs to the brand Daybird. [53]"} +{"question": "What are the features of the Nebula Pro Headlamp?", "truth": "The Nebula Pro Headlamp has 500 lumens of bright white light, adjustable brightness modes, a rechargeable battery, and a sleek design with water-resistant construction in neon color. It is suitable for night hikes, camping trips, and emergencies. [54]"} +{"question": "What is the price of the Nebula Pro Headlamp and what brand manufactures it?", "truth": "The Nebula Pro Headlamp is priced at 59.99 and is manufactured by the brand Gravitator. [54]"} +{"question": "What are the key features of the Vigor 2.0 Insulated Jacket?", "truth": "The Vigor 2.0 Insulated Jacket is waterproof and breathable, made with a 2-layer technical shell. It features a detachable hood, adjustable cuffs, and multiple pockets for storage, offering both style and functionality for winter activities. [55]"} +{"question": "What is the price and brand of the Vigor 2.0 Insulated Jacket?", "truth": "The price of the Vigor 2.0 Insulated Jacket is 189.99, and it is from the brand Solstix. [55]"} +{"question": "What are the features of the Traveler's Companion Duffel Bag?", "truth": "The Traveler's Companion Duffel Bag is made from durable waxed canvas in earthy brown and offers multiple carry options such as shoulder straps and handles. It includes multiple compartments for organized packing and a padded laptop sleeve, making it suitable for both adventure and work. [56]"} +{"question": "What is the price of the Traveler's Companion Duffel Bag and what brand does it belong to?", "truth": "The price of the Traveler's Companion Duffel Bag is $79.99, and it belongs to the brand B&R under the type Bags. [56]"} +{"question": "What are the key features of the Ascend XT Trail Running Shoes?", "truth": "The Ascend XT Trail Running Shoes in charcoal gray are lightweight yet rugged, offering excellent grip and support with a durable rubber outsole and advanced cushioning technology. The breathable mesh upper keeps feet cool during outdoor adventures. [57]"} +{"question": "What is the price and brand of the Ascend XT Trail Running Shoes?", "truth": "The Ascend XT Trail Running Shoes are priced at 109.99 and are from the brand Zephyr. [57]"} +{"question": "What are the features of the VelociX 2000 Bike Helmet?", "truth": "The VelociX 2000 Bike Helmet in glossy black features an adjustable fit system, detachable visor, and 14 ventilation channels to keep the rider cool during intense rides. It is aerodynamic, sleek in design, and lightweight, making it suitable for road cycling, mountain biking, and various other activities. [58]"} +{"question": "What is the price and brand of the VelociX 2000 Bike Helmet?", "truth": "The VelociX 2000 Bike Helmet is priced at 79.99 and belongs to the Raptor Elite brand. [58]"} +{"question": "What features does the TrailSeeker GPS Watch offer?", "truth": "The TrailSeeker GPS Watch is packed with features like GPS navigation, heart rate monitoring, and activity tracking. It also offers a long battery life for uninterrupted usage. [59]"} +{"question": "What type of users is the TrailSeeker GPS Watch designed for?", "truth": "The TrailSeeker GPS Watch is designed for outdoor enthusiasts who love to explore, offering them a durable design in stealth black and features like GPS navigation, heart rate monitoring, and activity tracking. [59]"} +{"question": "What are the key features of the SummitRider Snowboard Boots?", "truth": "The SummitRider Snowboard Boots in matte black offer a heat-moldable liner, dual-zone lacing system, and impact-resistant outsole for a precise and snug fit, enhancing control on any terrain. [60]"} +{"question": "What is the price and brand of the SummitRider Snowboard Boots?", "truth": "The SummitRider Snowboard Boots are priced at 249.0 and belong to the WildRunner brand in the Ski/boarding category. [60]"} +{"question": "What are the key features of the Trailblaze Steel-Blue Hiking Shoes by WildRunner?", "truth": "The Trailblaze Steel-Blue Hiking Shoes by WildRunner feature a steel-blue color, a waterproof membrane, a high-traction rubber outsole for superior grip, a breathable upper for comfort, and a reinforced toe cap for extra protection. [61]"} +{"question": "What activities are the Trailblaze Steel-Blue Hiking Shoes suitable for?", "truth": "The Trailblaze Steel-Blue Hiking Shoes are perfect for hiking, camping, and other outdoor adventures. [61]"} +{"question": "What are the key features of the Daybird Shadow Black Snowboard?", "truth": "The Daybird Shadow Black Snowboard features a black colorway, a camber profile for stability, a medium-flex rating for responsive turns, a lightweight construction for maneuverability, and a sintered base for speed and durability. [62]"} +{"question": "What type of performance is the Daybird Shadow Black Snowboard designed to deliver?", "truth": "The Daybird Shadow Black Snowboard is designed to deliver peak performance whether you're shredding on groomed runs or exploring the backcountry. [62]"} +{"question": "What are the key features of the Raptor Elite Razor Climbing Harness?", "truth": "The Raptor Elite Razor Climbing Harness is lightweight, breathable, and designed for maximum comfort and performance. It has adjustable waist and leg loops for a secure and customized fit, razor-shaped webbing for style, and durable construction with reinforced tie-in points. [63]"} +{"question": "What is the price of the Raptor Elite Razor Climbing Harness?", "truth": "The price of the Raptor Elite Razor Climbing Harness is $94.99. [63]"} +{"question": "What are the key features of the EcoVenture Olive Green Backpack?", "truth": "The EcoVenture Olive Green Backpack is made from recycled materials, has a spacious main compartment, multiple pockets for organizing gear, and adjustable padded shoulder straps for comfortable carrying. The backpack is eco-friendly, functional, and comes in an olive green color. [64]"} +{"question": "What is the price of the EcoVenture Olive Green Backpack and what brand manufactures it?", "truth": "The price of the EcoVenture Olive Green Backpack is $69.99, and it is manufactured by the brand Green Equipment. [64]"} +{"question": "What material is the Solstix Sprint PRO Carbon Cycling Helmet made of?", "truth": "The Solstix Sprint PRO Carbon Cycling Helmet is crafted from carbon fiber for optimal impact protection and durability. [65]"} +{"question": "What features does the Solstix Sprint PRO Carbon Cycling Helmet offer?", "truth": "The Solstix Sprint PRO Carbon Cycling Helmet features an aerodynamic design, adjustable fit system, ventilation channels for cooling, and a stylish rainbow color with the Solstix emblem. [65]"} +{"question": "What are the key features of the Compass Pro A-320 Professional Compass?", "truth": "The Compass Pro A-320 Professional Compass is designed for outdoor enthusiasts and professionals, featuring a liquid-filled housing for accurate readings, a rotating bezel for easy navigation, and a lanyard for convenient carrying. Its gunmetal color with white markings ensures clarity and visibility even in low-light conditions. [66]"} +{"question": "What is the price of the Compass Pro A-320 Professional Compass and what brand manufactures it?", "truth": "The Compass Pro A-320 Professional Compass is priced at $59.99 and is manufactured by B&R. [66]"} +{"question": "What are the key features of the Venture 2.0 40L Waterproof Duffel Bag?", "truth": "The Venture 2.0 40L Waterproof Duffel Bag is made from waterproof nylon material with taped seams for protection against the elements. It includes a spacious main compartment, external zippered pockets, and adjustable shoulder straps for easy carrying. The vibrant orange color adds excitement to outdoor adventures. [67]"} +{"question": "What is the price of the Venture 2.0 40L Waterproof Duffel Bag and what brand manufactures it?", "truth": "The price of the Venture 2.0 40L Waterproof Duffel Bag is $79.99. It is manufactured by Quester and falls under the category of Bags. [67]"} +{"question": "What are the key features of the Mens Horizon 80s Softshell Jacket?", "truth": "The Mens Horizon 80s Softshell Jacket is made from a water-resistant and breathable fabric in retro 1980s style. It features multiple colors, a detachable hood, adjustable cuffs, and multiple pockets for storing essentials. [68]"} +{"question": "What activities is the Mens Horizon 80s Softshell Jacket suitable for?", "truth": "The Mens Horizon 80s Softshell Jacket is suitable for activities like hiking, skiing, or exploring the city, as it combines style and functionality to keep you dry and comfortable in any weather. [68]"} +{"question": "What are the key features of the Gravitator Expedition 200 GPS Navigator?", "truth": "The Gravitator Expedition 200 GPS Navigator features a built-in GPS antenna for accurate positioning, preloaded maps, a user-friendly interface with intuitive controls, a sleek design in black color with the Gravitator logo, long battery life, and durable construction. [69]"} +{"question": "What is the price and brand of the Expedition 200 GPS Navigator?", "truth": "The price of the Expedition 200 GPS Navigator is 299.0 and the brand is Gravitator. [69]"} +{"question": "What is the description of the GripTrek Hiking Poles by WildRunner?", "truth": "The GripTrek hiking poles by WildRunner are described as a must-have for adventurers due to their durable aluminum construction and anti-slip handles, providing stability and support on any terrain. They are available in sleek yellow color, perfect for tackling steep inclines and rough trails. [70]"} +{"question": "What is the price and brand of the GripTrek Hiking Poles?", "truth": "The price of the GripTrek Hiking Poles is $79.99, and the brand is WildRunner. [70]"} +{"question": "What is the description of the Explorer Frost Boots by Daybird?", "truth": "The Explorer Frost Boots by Daybird are premium boots designed with a waterproof and insulated shell to keep your feet warm and protected in icy conditions. The sleek black design with blue accents adds a touch of style to your outdoor gear. [71]"} +{"question": "What is the price of the Explorer Frost Boots by Daybird?", "truth": "The price of the Explorer Frost Boots by Daybird is 149.99. [71]"} +{"question": "What is the description of the GravityZone All-Mountain Skis by Gravitator?", "truth": "The GravityZone All-Mountain Skis by Gravitator are high-performance skis designed for precision and control on all types of terrain. They feature a sleek metallic blue design and are constructed with carbon fiber for lightweight durability. [72]"} +{"question": "What is the price of the GravityZone All-Mountain Skis by Gravitator?", "truth": "The price of the GravityZone All-Mountain Skis by Gravitator is 699.0. [72]"} +{"question": "What is the description of the Omni-Snow Dual Snowboard by WildRunner?", "truth": "The Omni-Snow Dual Snowboard by WildRunner combines the maneuverability of a skateboard with the speed and stability of a snowboard. It is designed to unleash snowboarding skills with its innovative design. [73]"} +{"question": "What is the price and brand of the Omni-Snow Dual Snowboard?", "truth": "The price of the Omni-Snow Dual Snowboard is 289.99 and it is manufactured by WildRunner. [73]"} +{"question": "What is the description of the Apex Climbing Harness by Raptor Elite?", "truth": "The Apex Climbing Harness by Raptor Elite is constructed with high-strength nylon webbing and features adjustable leg loops for a secure and comfortable fit. It has a sleek white design with a vibrant orange emblem. [74]"} +{"question": "What is the price of the Apex Climbing Harness by Raptor Elite?", "truth": "The price of the Apex Climbing Harness by Raptor Elite is $99.99. [74]"} +{"question": "What features do the TrailTracker Hiking Shoes have?", "truth": "The TrailTracker Hiking Shoes by AirStrider are lightweight and breathable with a rugged rubber sole for excellent traction and stability. They come in a cool gray color with green accents for added style. [75]"} +{"question": "What material is the Fusion Carbon Cycling Helmet made of?", "truth": "The Fusion Carbon Cycling Helmet is made from lightweight carbon fiber. [76]"} +{"question": "What features does the Fusion Carbon Cycling Helmet have?", "truth": "The Fusion Carbon Cycling Helmet features an aerodynamic design for maximum speed and a colorful finish with a bold blue stripe to make the wearer stand out on the road. [76]"} +{"question": "What is the name of the tent and who is the manufacturer?", "truth": "The name of the tent is Survivor 2-Person Tent by XE. [77]"} +{"question": "What are some features of the Survivor 2-Person Tent?", "truth": "The Survivor 2-Person Tent is made from durable ripstop nylon with a waterproof coating, ensuring dryness in any weather. It also comes in a vibrant orange color for high visibility in the wild. [77]"} +{"question": "What is the description of the Basecamp Duffle Bag by Solstix?", "truth": "The Basecamp Duffle Bag by Solstix is described as the ultimate adventure companion made from durable nylon with multiple compartments for optimal organization. Its sleek red design with gray accents combines style and functionality. [78]"} +{"question": "What is the price and brand of the Basecamp Duffle Bag?", "truth": "The price of the Basecamp Duffle Bag is $129.0, and the brand is Solstix. [78]"} +{"question": "What is the description of the Everest Insulated Jacket by Legend?", "truth": "The Everest Insulated Jacket by Legend is designed to conquer the cold with its insulated design and sleek grey color. It combines warmth and style, featuring a water-resistant shell to keep you dry during unexpected showers and a cozy fleece lining for extra comfort. [79]"} +{"question": "What is the price of the Everest Insulated Jacket by Legend?", "truth": "The price of the Everest Insulated Jacket by Legend is $179.99. [79]"} +{"question": "What features does the Pathfinder GPS Watch by XE include?", "truth": "The Pathfinder GPS Watch by XE includes GPS tracking, altimeter, barometer, and compass functions to guide you on your outdoor adventures. [80]"} +{"question": "What is the design of the Pathfinder GPS Watch by XE?", "truth": "The Pathfinder GPS Watch by XE has a sleek pink design with a vibrant green dial, adding a sporty touch to your wrist. [80]"} +{"question": "What is the description of the Trail Breeze Hiking Shoes by AirStrider?", "truth": "The Trail Breeze hiking shoes by AirStrider offer ultimate comfort and stability with a lightweight design featuring a breathable mesh upper in vivid blue, ideal for hot summer hikes. The durable rubber outsole provides exceptional grip on various terrains. [81]"} +{"question": "How much do the Trail Breeze Hiking Shoes by AirStrider cost?", "truth": "The Trail Breeze Hiking Shoes by AirStrider are priced at 109.99. [81]"} +{"question": "What are the key features of the Maverick Pro ski goggles by WildRunner?", "truth": "The Maverick Pro ski goggles by WildRunner feature a sleek black frame, mirrored polarized lenses that reduce glare, and a comfortable foam lining with an adjustable strap for a secure fit. [82]"} +{"question": "What is the price of the Maverick Pro ski goggles by WildRunner?", "truth": "The price of the Maverick Pro ski goggles by WildRunner is $139.99. [82]"} +{"question": "What are the key features of the Blizzard Freestyle Snowboard from Zephyr?", "truth": "The Blizzard snowboard from Zephyr features a vibrant orange and black design, a durable bamboo core, and carbon fiber reinforcement. It offers riders an optimal blend of flexibility and responsiveness, making it perfect for those who crave speed and control. [83]"} +{"question": "What is the price and brand of the Blizzard Freestyle Snowboard?", "truth": "The price of the Blizzard Freestyle Snowboard is 379.0 and it is from the brand Zephyr. [83]"} +{"question": "What is the description of the Gravity Harness?", "truth": "The Gravity Harness is described as a tool to reach new heights with its innovative design and sturdy construction. It ensures safety while climbing, offers maximum comfort and freedom of movement, and features a sleek black and red color scheme for style. [84]"} +{"question": "What is the price of the Gravity Harness?", "truth": "The price of the Gravity Harness is $89.99. [84]"} +{"question": "What are the key features of the LumenHead headlamp by Daybird?", "truth": "The LumenHead headlamp by Daybird is compact yet powerful, featuring a bright LED light in a vibrant green housing. It offers multiple lighting modes, including a red light for preserving night vision, providing exceptional visibility in any conditions. [85]"} +{"question": "How much does the LumenHead headlamp by Daybird cost and what type of outdoor activity is it suitable for?", "truth": "The LumenHead headlamp by Daybird is priced at $49.99 and is designed for trekking. [85]"} +{"question": "What are the key features of the ProVent Bike Helmet?", "truth": "The ProVent Bike Helmet features a matte black finish with striking red accents, ProVent technology for optimal airflow, an adjustable fit system, and a removable visor. It is suitable for both casual and professional riders. [86]"} +{"question": "What is the price of the ProVent Bike Helmet and what brand manufactures it?", "truth": "The price of the ProVent Bike Helmet is $79.99, and it is manufactured by the brand Raptor Elite. [86]"} +{"question": "What are the key features of the Nomad 2-Person Tent by XE?", "truth": "The Nomad 2-Person Tent by XE is designed for rugged outdoor conditions with a durable waterproof fabric in earthy tones. It offers a spacious interior, easy-to-use setup, and an innovative ventilation system for a comfortable camping experience. [87]"} +{"question": "How much does the Nomad 2-Person Tent by XE cost and what type of tent is it?", "truth": "The Nomad 2-Person Tent by XE is priced at 229.0 and is categorized as a Trekking tent. [87]"} +{"question": "What is the description of the AlpinePack Backpack by Green Equipment?", "truth": "The AlpinePack backpack by Green Equipment is a versatile and durable backpack with a sleek navy design, reinforced straps, a capacity of 45 liters, multiple compartments, and a hydration pack sleeve. It offers ample storage and organization, with an ergonomic back panel for maximum comfort on challenging treks. [88]"} +{"question": "What are the key features of the AlpinePack Backpack by Green Equipment?", "truth": "The key features of the AlpinePack backpack by Green Equipment include a capacity of 45 liters, multiple compartments, a hydration pack sleeve, reinforced straps, a sleek navy design, and an ergonomic back panel for maximum comfort. [88]"} +{"question": "What is the description of the Summit Pro Down Jacket?", "truth": "The Summit Pro Down Jacket is a high-performance jacket filled with premium down insulation for exceptional warmth. It features a sleek design in deep navy blue with contrasting silver zippers and emblems, along with weather-resistant fabric and a removable hood. [89]"} +{"question": "What is the price of the Summit Pro Down Jacket?", "truth": "The price of the Summit Pro Down Jacket is $239.99. [89]"} +{"question": "What are the key features of the TrailTracker GPS Watch?", "truth": "The TrailTracker GPS Watch features a built-in GPS that accurately tracks location, speed, and distance. It has a sleek camo design with a vivid orange strap, long battery life, and water-resistant construction. [90]"} +{"question": "What is the price of the TrailTracker GPS Watch and what brand manufactures it?", "truth": "The price of the TrailTracker GPS Watch is 199.0 and it is manufactured by B&R. [90]"} +{"question": "What are the key features of the Trailblazer Trail Running Shoes by WildRunner?", "truth": "The Trailblazer Trail Running Shoes by WildRunner are lightweight shoes that come in vibrant blue. They feature a rugged outsole for excellent traction, a breathable mesh upper for maximum comfort, and quick-drying materials to keep you dry on your adventures. [91]"} +{"question": "What is the price and brand of the Trailblazer Trail Running Shoes?", "truth": "The Trailblazer Trail Running Shoes by WildRunner are priced at 89.99 and the brand is WildRunner. [91]"} +{"question": "What are the key features of the Blizzard Snowboard by Daybird?", "truth": "The Blizzard Snowboard by Daybird features a sleek design in icy white, a durable wood core, a versatile medium flex, and a precision base that allows for smooth rides and easy turns. [92]"} +{"question": "What is the price of the Blizzard Snowboard by Daybird?", "truth": "The price of the Blizzard Snowboard by Daybird is $449.99. [92]"} +{"question": "What is the description of the Summit Climbing Harness?", "truth": "The Summit Climbing Harness is described as a durable and lightweight harness available in bold red, providing maximum comfort and safety while scaling tricky routes. It features adjustable waistband and leg loops for a snug fit, along with gear loops for easy access to equipment. [93]"} +{"question": "What is the price of the Summit Climbing Harness?", "truth": "The price of the Summit Climbing Harness is $109.99. [93]"} +{"question": "What is the description of the Gravity Hiking Backpack?", "truth": "The Gravity Hiking Backpack is described as a backpack designed for unforgettable hikes, available in tiger stripes. It offers a spacious main compartment, multiple pockets, and a hydration system compatible design. The backpack is lightweight and durable for maximum comfort on the trails. [94]"} +{"question": "What is the price and brand of the Gravity Hiking Backpack?", "truth": "The Gravity Hiking Backpack is priced at $79.99 and is branded as Gravitator. [94]"} +{"question": "What are the features of the AirStrider AeroLite Cycling Helmet?", "truth": "The AirStrider AeroLite Cycling Helmet in glossy grey features a lightweight design, adjustable straps, excellent ventilation, and an aerodynamic shape that reduces air resistance. [95]"} +{"question": "What is the price of the AirStrider AeroLite Cycling Helmet?", "truth": "The price of the AirStrider AeroLite Cycling Helmet is $129.99. [95]"} +{"question": "What is the description of the B&R Explorer Camping Tent?", "truth": "The B&R Explorer Camping Tent is a spacious tent available in forest green that comfortably fits up to six people with a separate sleeping area and a generous living space. Its sturdy construction and weather-resistant materials ensure durability and protection from the elements. [96]"} +{"question": "What is the price and brand of the Explorer Camping Tent?", "truth": "The price of the B&R Explorer Camping Tent is 279.99 and the brand is B&R. [96]"} +{"question": "What are the features of the Quester Gravity Waterproof Dry Bag?", "truth": "The Quester Gravity Waterproof Dry Bag features a roll-top closure system, adjustable shoulder straps, and durable PVC-coated fabric to withstand water, sand, and dirt. It is described as versatile and ideal for adventures on land or water. [97]"} +{"question": "What is the price of the Gravity Waterproof Dry Bag by Quester?", "truth": "The price of the Gravity Waterproof Dry Bag by Quester is $49.99. [97]"} +{"question": "What is the description of the Legend Element Outdoor Jacket?", "truth": "The Legend Element Outdoor Jacket is available in charcoal gray and offers ultimate protection with its waterproof and windproof shell. The breathable fabric and adjustable cuffs ensure comfort, allowing you to explore in any weather condition. [98]"} +{"question": "What is the price of the Legend Element Outdoor Jacket?", "truth": "The price of the Legend Element Outdoor Jacket is $179.99. [98]"} +{"question": "What features does the Solstix Adventurer GPS Watch offer?", "truth": "The Solstix Adventurer GPS Watch features a built-in GPS, altimeter, and compass, allowing users to track routes and monitor progress. It also has multiple sport modes, making it an ideal companion for outdoor enthusiasts. [99]"} +{"question": "What is the price of the Solstix Adventurer GPS Watch?", "truth": "The price of the Solstix Adventurer GPS Watch is 199.99. [99]"} +{"question": "What are the features of the Green Equipment EcoLite Trekking Poles?", "truth": "The Green Equipment EcoLite Trekking Poles are lightweight with adjustable height, shock-absorbing capabilities, and ergonomic cork handles for a comfortable grip. They provide stability and support for challenging trails. [100]"} +{"question": "What is the price of the EcoLite Trekking Poles and what brand manufactures them?", "truth": "The EcoLite Trekking Poles are priced at 79.99 and are manufactured by Green Equipment. [100]"} +{"question": "What features do the Trek Xtreme hiking shoes offer?", "truth": "The Trek Xtreme hiking shoes by Raptor Elite are built with a durable leather upper and rugged rubber sole, providing excellent traction and protection on any trail. They come in a timeless brown color, adding style to outdoor adventures. [101]"} diff --git a/evals/results/baseline/config.json b/evals/results/baseline/config.json new file mode 100644 index 00000000..82653d21 --- /dev/null +++ b/evals/results/baseline/config.json @@ -0,0 +1,16 @@ +{ + "testdata_path": "ground_truth.jsonl", + "results_dir": "results/experiment", + "requested_metrics": ["gpt_groundedness", "gpt_relevance", "answer_length", "latency", "citation_match"], + "target_url": "http://host.docker.internal:8000/chat", + "target_parameters": { + "overrides": { + "use_advanced_flow": true, + "top": 3, + "retrieval_mode": "hybrid", + "temperature": 0.3 + } + }, + "target_response_answer_jmespath": "message.content", + "target_response_context_jmespath": "context.data_points" +} diff --git a/evals/results/baseline/eval_results.jsonl b/evals/results/baseline/eval_results.jsonl new file mode 100644 index 00000000..978afacf --- /dev/null +++ b/evals/results/baseline/eval_results.jsonl @@ -0,0 +1,20 @@ +{"question": "What is the description of the Wanderer Black Hiking Boots?", "truth": "The Wanderer Black Hiking Boots by Daybird are made with a waterproof leather upper and a durable rubber sole for superior traction. They feature a cushioned insole and padded collar for all-day comfort during outdoor adventures. [1]", "answer": "The Wanderer Black Hiking Boots by Daybird are made with a waterproof leather upper and a durable rubber sole for superior traction. They feature a cushioned insole and padded collar for all-day comfort during your outdoor adventures. The price for the Wanderer Black Hiking Boots is $109.99. [1]", "context": "{\"1\": {\"id\": 1, \"type\": \"Footwear\", \"brand\": \"Daybird\", \"name\": \"Wanderer Black Hiking Boots\", \"description\": \"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.\", \"price\": 109.99}, \"21\": {\"id\": 21, \"type\": \"Footwear\", \"brand\": \"Legend\", \"name\": \"Trailblazer Black Hiking Shoes\", \"description\": \"The Legend Trailblazer is a versatile hiking shoe designed to provide unparalleled durability and comfort on any adventure. With its black color, these shoes offer a sleek and minimalist style. The shoes feature a waterproof GORE-TEX lining, Vibram rubber outsole for enhanced traction, and a reinforced toe cap for added protection. Conquer any trail with confidence in the Legend Trailblazer Black Hiking Shoes.\", \"price\": 129.99}, \"61\": {\"id\": 61, \"type\": \"Footwear\", \"brand\": \"WildRunner\", \"name\": \"Trailblaze Steel-Blue Hiking Shoes\", \"description\": \"Explore the great outdoors with the Trailblaze Steel-Blue Hiking Shoes by WildRunner. These rugged and durable shoes feature a steel-blue color, a waterproof membrane, and a high-traction rubber outsole for superior grip on any terrain. The breathable upper keeps your feet cool and comfortable, while the reinforced toe cap adds extra protection. Perfect for hiking, camping, and other outdoor adventures.\", \"price\": 129.99}}", "latency": 1.848982, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 296, "citation_match": true} +{"question": "What is the price and brand of the Wanderer Black Hiking Boots?", "truth": "The Wanderer Black Hiking Boots are priced at 109.99 and are from the brand Daybird. [1]", "answer": "The Wanderer Black Hiking Boots are priced at $109.99 and are made by the brand Daybird. [1]", "context": "{\"1\": {\"id\": 1, \"type\": \"Footwear\", \"brand\": \"Daybird\", \"name\": \"Wanderer Black Hiking Boots\", \"description\": \"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.\", \"price\": 109.99}, \"21\": {\"id\": 21, \"type\": \"Footwear\", \"brand\": \"Legend\", \"name\": \"Trailblazer Black Hiking Shoes\", \"description\": \"The Legend Trailblazer is a versatile hiking shoe designed to provide unparalleled durability and comfort on any adventure. With its black color, these shoes offer a sleek and minimalist style. The shoes feature a waterproof GORE-TEX lining, Vibram rubber outsole for enhanced traction, and a reinforced toe cap for added protection. Conquer any trail with confidence in the Legend Trailblazer Black Hiking Shoes.\", \"price\": 129.99}, \"61\": {\"id\": 61, \"type\": \"Footwear\", \"brand\": \"WildRunner\", \"name\": \"Trailblaze Steel-Blue Hiking Shoes\", \"description\": \"Explore the great outdoors with the Trailblaze Steel-Blue Hiking Shoes by WildRunner. These rugged and durable shoes feature a steel-blue color, a waterproof membrane, and a high-traction rubber outsole for superior grip on any terrain. The breathable upper keeps your feet cool and comfortable, while the reinforced toe cap adds extra protection. Perfect for hiking, camping, and other outdoor adventures.\", \"price\": 129.99}}", "latency": 2.047678, "gpt_groundedness": 5.0, "gpt_relevance": 1.0, "answer_length": 92, "citation_match": true} +{"question": "What are the key features of the Summit Pro Harness by Gravitator?", "truth": "The Summit Pro Harness by Gravitator is a lightweight and durable climbing harness with adjustable leg loops and waist belt for a customized fit. It also features a reinforced tie-in point and strong webbing loops for safety. [2]", "answer": "The Summit Pro Harness by Gravitator [2] is a lightweight and durable climbing harness with adjustable leg loops and waist belt for a customized fit. It features a vibrant blue color for a stylish look while climbing. The harness also has a reinforced tie-in point and strong webbing loops to prioritize safety.", "context": "{\"2\": {\"id\": 2, \"type\": \"Climbing\", \"brand\": \"Gravitator\", \"name\": \"Summit Pro Harness\", \"description\": \"Conquer new heights with the Summit Pro Harness by Gravitator. This lightweight and durable climbing harness features adjustable leg loops and waist belt for a customized fit. With its vibrant blue color, you'll look stylish while maneuvering difficult routes. Safety is a top priority with a reinforced tie-in point and strong webbing loops.\", \"price\": 89.99}, \"84\": {\"id\": 84, \"type\": \"Climbing\", \"brand\": \"Gravitator\", \"name\": \"Gravity Harness\", \"description\": \"Reach new heights with the Gravitator Gravity harness. With its innovative design and sturdy construction, this harness ensures your safety while climbing. The sleek black and red color scheme adds a touch of style. It offers maximum comfort and freedom of movement, giving you the confidence to conquer any climbing challenge.\", \"price\": 89.99}, \"44\": {\"id\": 44, \"type\": \"Climbing\", \"brand\": \"Raptor Elite\", \"name\": \"Summit Pro Climbing Harness\", \"description\": \"Conquer the highest peaks with the Summit Pro Climbing Harness by Raptor Elite. This harness features a lightweight and breathable construction, complete with adjustable leg loops for a personalized fit. It has reinforced tie-in points for maximum safety and durability. The vivid green color adds a touch of style to your climbing gear.\", \"price\": 109.99}}", "latency": 4.400196, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 311, "citation_match": true} +{"question": "What is the price of the Summit Pro Harness by Gravitator and what type of product is it?", "truth": "The price of the Summit Pro Harness by Gravitator is 89.99. It is a climbing harness designed for conquering new heights with style and safety in mind. [2]", "answer": "The Summit Pro Harness by Gravitator [2] is priced at $89.99 and it is a climbing harness.", "context": "{\"2\": {\"id\": 2, \"type\": \"Climbing\", \"brand\": \"Gravitator\", \"name\": \"Summit Pro Harness\", \"description\": \"Conquer new heights with the Summit Pro Harness by Gravitator. This lightweight and durable climbing harness features adjustable leg loops and waist belt for a customized fit. With its vibrant blue color, you'll look stylish while maneuvering difficult routes. Safety is a top priority with a reinforced tie-in point and strong webbing loops.\", \"price\": 89.99}, \"84\": {\"id\": 84, \"type\": \"Climbing\", \"brand\": \"Gravitator\", \"name\": \"Gravity Harness\", \"description\": \"Reach new heights with the Gravitator Gravity harness. With its innovative design and sturdy construction, this harness ensures your safety while climbing. The sleek black and red color scheme adds a touch of style. It offers maximum comfort and freedom of movement, giving you the confidence to conquer any climbing challenge.\", \"price\": 89.99}, \"94\": {\"id\": 94, \"type\": \"Bags\", \"brand\": \"Gravitator\", \"name\": \"Gravity Hiking Backpack\", \"description\": \"Embark on unforgettable hikes with the Gravitator Gravity Hiking Backpack. Available in tiger stripes, this backpack offers a spacious main compartment, multiple pockets, and a hydration system compatible design. The lightweight and durable construction ensures maximum comfort on the trails.\", \"price\": 79.99}}", "latency": 3.497859, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 90, "citation_match": true} +{"question": "What are the key features of the Alpine Fusion Goggles from WildRunner?", "truth": "The Alpine Fusion Goggles from WildRunner offer full UV protection, anti-fog lenses, a stylish silver frame with orange lenses, adjustable straps for a secure fit, and soft foam padding for comfort during skiing or snowboarding. [3]", "answer": "The Alpine Fusion Goggles from WildRunner offer full UV protection and anti-fog lenses to keep your vision clear on the slopes. They have a stylish silver frame with orange lenses, adjustable straps for a secure fit, and soft foam padding for all-day comfort. The price for these goggles is $79.99 [3].", "context": "{\"3\": {\"id\": 3, \"type\": \"Ski/boarding\", \"brand\": \"WildRunner\", \"name\": \"Alpine Fusion Goggles\", \"description\": \"Enhance your skiing experience with the Alpine Fusion Goggles from WildRunner. These goggles offer full UV protection and anti-fog lenses to keep your vision clear on the slopes. With their stylish silver frame and orange lenses, you'll stand out from the crowd. Adjustable straps ensure a secure fit, while the soft foam padding provides comfort all day long.\", \"price\": 79.99}, \"82\": {\"id\": 82, \"type\": \"Ski/boarding\", \"brand\": \"WildRunner\", \"name\": \"Maverick Pro Ski Goggles\", \"description\": \"Conquer the slopes in style with the Maverick Pro ski goggles by WildRunner. Designed for maximum performance, these goggles feature a sleek black frame and mirrored, polarized lenses that reduce glare, enhancing your visibility. With a comfortable foam lining and adjustable strap, these goggles provide a secure and snug fit.\", \"price\": 139.99}, \"16\": {\"id\": 16, \"type\": \"Ski/boarding\", \"brand\": \"Solstix\", \"name\": \"Expedition 2022 Goggles\", \"description\": \"Solstix Expedition 2022 Goggles provide clear vision and optimal protection on the slopes. With an anti-fog lens, UV protection, and a comfortable foam lining, these goggles ensure a great fit and unrestricted vision even in challenging conditions. The matte black frame gives them a sleek and modern look.\", \"price\": 89.0}}", "latency": 3.576792, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 302, "citation_match": true} +{"question": "What is the price and brand of the Alpine Fusion Goggles?", "truth": "The Alpine Fusion Goggles from WildRunner are priced at 79.99 and are part of the WildRunner brand. [3]", "answer": "The Alpine Fusion Goggles are priced at $79.99 and are made by WildRunner. [3]", "context": "{\"3\": {\"id\": 3, \"type\": \"Ski/boarding\", \"brand\": \"WildRunner\", \"name\": \"Alpine Fusion Goggles\", \"description\": \"Enhance your skiing experience with the Alpine Fusion Goggles from WildRunner. These goggles offer full UV protection and anti-fog lenses to keep your vision clear on the slopes. With their stylish silver frame and orange lenses, you'll stand out from the crowd. Adjustable straps ensure a secure fit, while the soft foam padding provides comfort all day long.\", \"price\": 79.99}, \"82\": {\"id\": 82, \"type\": \"Ski/boarding\", \"brand\": \"WildRunner\", \"name\": \"Maverick Pro Ski Goggles\", \"description\": \"Conquer the slopes in style with the Maverick Pro ski goggles by WildRunner. Designed for maximum performance, these goggles feature a sleek black frame and mirrored, polarized lenses that reduce glare, enhancing your visibility. With a comfortable foam lining and adjustable strap, these goggles provide a secure and snug fit.\", \"price\": 139.99}, \"16\": {\"id\": 16, \"type\": \"Ski/boarding\", \"brand\": \"Solstix\", \"name\": \"Expedition 2022 Goggles\", \"description\": \"Solstix Expedition 2022 Goggles provide clear vision and optimal protection on the slopes. With an anti-fog lens, UV protection, and a comfortable foam lining, these goggles ensure a great fit and unrestricted vision even in challenging conditions. The matte black frame gives them a sleek and modern look.\", \"price\": 89.0}}", "latency": 1.402853, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 78, "citation_match": true} +{"question": "What is the description of the Expedition Backpack by Quester?", "truth": "The Expedition Backpack by Quester is described as a must-have for every outdoor enthusiast with its spacious interior, multiple pockets, and durable nylon fabric that can withstand tough conditions. It also features orange accents for added style. [4]", "answer": "The Expedition Backpack by Quester is a must-have for every outdoor enthusiast. It is made with durable nylon fabric, has a spacious interior, multiple pockets for easy organization, and orange accents for a touch of style. The price is $129.99. [4]", "context": "{\"4\": {\"id\": 4, \"type\": \"Bags\", \"brand\": \"Quester\", \"name\": \"Expedition Backpack\", \"description\": \"The Expedition Backpack by Quester is a must-have for every outdoor enthusiast. With its spacious interior and multiple pockets, you can easily carry all your gear and essentials. Made with durable nylon fabric, this backpack is built to withstand the toughest conditions. The orange accents add a touch of style to this functional backpack.\", \"price\": 129.99}, \"67\": {\"id\": 67, \"type\": \"Bags\", \"brand\": \"Quester\", \"name\": \"Venture 2.0 40L Waterproof Duffel Bag\", \"description\": \"Pack your gear in the Quester Venture 2.0 40L Waterproof Duffel Bag. This versatile duffel bag is made from waterproof nylon material and features taped seams to keep your belongings safe from the elements. It offers a spacious main compartment, external zippered pockets, and adjustable shoulder straps for easy carrying. The vibrant orange color adds a pop of excitement to your outdoor adventures.\", \"price\": 79.99}, \"7\": {\"id\": 7, \"type\": \"Bags\", \"brand\": \"Solstix\", \"name\": \"Explorer 45L Backpack\", \"description\": \"The Explorer 45L Backpack by Solstix is perfect for your next outdoor expedition. Made with waterproof and tear-resistant materials, this backpack can withstand even the harshest weather conditions. With its spacious main compartment and multiple pockets, you can easily organize your gear. The green and black color scheme adds a rugged and adventurous edge.\", \"price\": 149.99}}", "latency": 3.85642, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 249, "citation_match": true} +{"question": "What is the price of the Expedition Backpack by Quester?", "truth": "The price of the Expedition Backpack by Quester is $129.99. [4]", "answer": "The price of the Expedition Backpack by Quester is $129.99 [4].", "context": "{\"4\": {\"id\": 4, \"type\": \"Bags\", \"brand\": \"Quester\", \"name\": \"Expedition Backpack\", \"description\": \"The Expedition Backpack by Quester is a must-have for every outdoor enthusiast. With its spacious interior and multiple pockets, you can easily carry all your gear and essentials. Made with durable nylon fabric, this backpack is built to withstand the toughest conditions. The orange accents add a touch of style to this functional backpack.\", \"price\": 129.99}, \"67\": {\"id\": 67, \"type\": \"Bags\", \"brand\": \"Quester\", \"name\": \"Venture 2.0 40L Waterproof Duffel Bag\", \"description\": \"Pack your gear in the Quester Venture 2.0 40L Waterproof Duffel Bag. This versatile duffel bag is made from waterproof nylon material and features taped seams to keep your belongings safe from the elements. It offers a spacious main compartment, external zippered pockets, and adjustable shoulder straps for easy carrying. The vibrant orange color adds a pop of excitement to your outdoor adventures.\", \"price\": 79.99}, \"97\": {\"id\": 97, \"type\": \"Bags\", \"brand\": \"Quester\", \"name\": \"Gravity Waterproof Dry Bag\", \"description\": \"Keep your essentials dry and secure with the Quester Gravity Waterproof Dry Bag. This versatile bag, in vibrant orange, features a roll-top closure system, adjustable shoulder straps, and durable PVC-coated fabric to withstand water, sand, and dirt. Ideal for adventures on land or water.\", \"price\": 49.99}}", "latency": 3.238277, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 63, "citation_match": true} +{"question": "What is the description of the Blizzard Rider Snowboard by B&R?", "truth": "The Blizzard Rider Snowboard by B&R is a versatile snowboard perfect for riders of all levels, featuring a medium flex and twin shape. It has a sleek black and blue color scheme and is suitable for carving turns or hitting the terrain park. [5]", "answer": "The Blizzard Rider Snowboard by B&R is a versatile snowboard perfect for riders of all levels. It has a medium flex, a twin shape, and a sleek black and blue color scheme. Whether you're carving turns or hitting the terrain park, this snowboard will help you shred with confidence. It is priced at $299.99. [5]", "context": "{\"5\": {\"id\": 5, \"type\": \"Ski/boarding\", \"brand\": \"B&R\", \"name\": \"Blizzard Rider Snowboard\", \"description\": \"Get ready to ride the slopes with the Blizzard Rider Snowboard by B&R. This versatile snowboard is perfect for riders of all levels with its medium flex and twin shape. Its black and blue color scheme gives it a sleek and cool look. Whether you're carving turns or hitting the terrain park, this snowboard will help you shred with confidence.\", \"price\": 299.99}, \"92\": {\"id\": 92, \"type\": \"Ski/boarding\", \"brand\": \"Daybird\", \"name\": \"Blizzard Snowboard\", \"description\": \"Take on the slopes with the Daybird Blizzard Snowboard. This powerful board features a sleek design in icy white, with a durable wood core, a versatile medium flex, and a precision base that allows for smooth rides and easy turns. Strap on and carve your way to glory.\", \"price\": 449.99}, \"83\": {\"id\": 83, \"type\": \"Ski/boarding\", \"brand\": \"Zephyr\", \"name\": \"Blizzard Freestyle Snowboard\", \"description\": \"Unleash your freestyle skills on the slopes with the Blizzard snowboard from Zephyr. Featuring a vibrant orange and black design, this snowboard is perfect for riders who crave speed and control. Constructed with a durable bamboo core and carbon fiber reinforcement, the Blizzard offers an optimal blend of flexibility and responsiveness.\", \"price\": 379.0}}", "latency": 2.263607, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 310, "citation_match": true} +{"question": "What is the price and brand of the Blizzard Rider Snowboard?", "truth": "The price of the Blizzard Rider Snowboard is 299.99, and it is manufactured by the brand B&R under the type Ski/boarding. [5]", "answer": "The price of the Blizzard Rider Snowboard is $299.99 and it is made by B&R. [5]", "context": "{\"5\": {\"id\": 5, \"type\": \"Ski/boarding\", \"brand\": \"B&R\", \"name\": \"Blizzard Rider Snowboard\", \"description\": \"Get ready to ride the slopes with the Blizzard Rider Snowboard by B&R. This versatile snowboard is perfect for riders of all levels with its medium flex and twin shape. Its black and blue color scheme gives it a sleek and cool look. Whether you're carving turns or hitting the terrain park, this snowboard will help you shred with confidence.\", \"price\": 299.99}, \"83\": {\"id\": 83, \"type\": \"Ski/boarding\", \"brand\": \"Zephyr\", \"name\": \"Blizzard Freestyle Snowboard\", \"description\": \"Unleash your freestyle skills on the slopes with the Blizzard snowboard from Zephyr. Featuring a vibrant orange and black design, this snowboard is perfect for riders who crave speed and control. Constructed with a durable bamboo core and carbon fiber reinforcement, the Blizzard offers an optimal blend of flexibility and responsiveness.\", \"price\": 379.0}, \"92\": {\"id\": 92, \"type\": \"Ski/boarding\", \"brand\": \"Daybird\", \"name\": \"Blizzard Snowboard\", \"description\": \"Take on the slopes with the Daybird Blizzard Snowboard. This powerful board features a sleek design in icy white, with a durable wood core, a versatile medium flex, and a precision base that allows for smooth rides and easy turns. Strap on and carve your way to glory.\", \"price\": 449.99}}", "latency": 3.283513, "gpt_groundedness": 1.0, "gpt_relevance": 1.0, "answer_length": 79, "citation_match": true} +{"question": "What are the key features of the Carbon Fiber Trekking Poles by Raptor Elite?", "truth": "The Carbon Fiber Trekking Poles by Raptor Elite are designed with lightweight carbon fiber shafts for excellent support and durability. They have comfortable and adjustable cork grips for a secure hold, and blue accents for a stylish touch. Additionally, they are compact and collapsible for easy transport and storage. [6]", "answer": "The Carbon Fiber Trekking Poles by Raptor Elite are designed with lightweight carbon fiber shafts for excellent support and durability. They have comfortable and adjustable cork grips for a secure hold, and stylish blue accents. These poles are compact, collapsible, and easy to transport and store. The price is $69.99 [6].\n\nThe Raptor Elite Carbon Fiber Trekking Poles are lightweight and durable, providing stability on various terrains and reducing strain on your joints. They feature an ergonomic grip and adjustable length, making them essential for outdoor adventures. The price is $99.00 [35].", "context": "{\"6\": {\"id\": 6, \"type\": \"Trekking\", \"brand\": \"Raptor Elite\", \"name\": \"Carbon Fiber Trekking Poles\", \"description\": \"The Carbon Fiber Trekking Poles by Raptor Elite are the ultimate companion for your hiking adventures. Designed with lightweight carbon fiber shafts, these poles provide excellent support and durability. The comfortable and adjustable cork grips ensure a secure hold, while the blue accents add a stylish touch. Compact and collapsible, these trekking poles are easy to transport and store.\", \"price\": 69.99}, \"35\": {\"id\": 35, \"type\": \"Trekking\", \"brand\": \"Raptor Elite\", \"name\": \"Carbon Fiber Trekking Poles\", \"description\": \"Hike with confidence using the Raptor Elite Carbon Fiber Trekking Poles. These lightweight and durable poles provide stability on various terrains and reduce strain on your joints. With an ergonomic grip and adjustable length, these poles are a must-have for your outdoor adventures.\", \"price\": 99.0}, \"100\": {\"id\": 100, \"type\": \"Trekking\", \"brand\": \"Green Equipment\", \"name\": \"EcoLite Trekking Poles\", \"description\": \"Tackle challenging trails with the Green Equipment EcoLite Trekking Poles. These lightweight poles, in vibrant green, feature adjustable height, shock-absorbing capabilities, and ergonomic cork handles for a comfortable grip. Whether ascending or descending, these poles provide stability and support.\", \"price\": 79.99}}", "latency": 7.618347, "gpt_groundedness": 1.0, "gpt_relevance": 4.0, "answer_length": 601, "citation_match": true} +{"question": "What is the price of the Carbon Fiber Trekking Poles by Raptor Elite?", "truth": "The price of the Carbon Fiber Trekking Poles by Raptor Elite is $69.99. [6]", "answer": "The price of the Carbon Fiber Trekking Poles by Raptor Elite is $69.99 [6].", "context": "{\"35\": {\"id\": 35, \"type\": \"Trekking\", \"brand\": \"Raptor Elite\", \"name\": \"Carbon Fiber Trekking Poles\", \"description\": \"Hike with confidence using the Raptor Elite Carbon Fiber Trekking Poles. These lightweight and durable poles provide stability on various terrains and reduce strain on your joints. With an ergonomic grip and adjustable length, these poles are a must-have for your outdoor adventures.\", \"price\": 99.0}, \"6\": {\"id\": 6, \"type\": \"Trekking\", \"brand\": \"Raptor Elite\", \"name\": \"Carbon Fiber Trekking Poles\", \"description\": \"The Carbon Fiber Trekking Poles by Raptor Elite are the ultimate companion for your hiking adventures. Designed with lightweight carbon fiber shafts, these poles provide excellent support and durability. The comfortable and adjustable cork grips ensure a secure hold, while the blue accents add a stylish touch. Compact and collapsible, these trekking poles are easy to transport and store.\", \"price\": 69.99}, \"93\": {\"id\": 93, \"type\": \"Climbing\", \"brand\": \"Raptor Elite\", \"name\": \"Summit Climbing Harness\", \"description\": \"Conquer the highest peaks with the Raptor Elite Summit Climbing Harness. This durable and lightweight harness, available in bold red, provides maximum comfort and safety while scaling tricky routes. Its adjustable waistband and leg loops ensure a snug fit, while the gear loops provide easy access to your equipment.\", \"price\": 109.99}}", "latency": 1.953346, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 75, "citation_match": true} +{"question": "What are the key features of the Explorer 45L Backpack by Solstix?", "truth": "The Explorer 45L Backpack by Solstix is made with waterproof and tear-resistant materials, making it suitable for harsh weather conditions. It has a spacious main compartment, multiple pockets for organization, and a green and black color scheme for a rugged look. [7]", "answer": "The Explorer 45L Backpack by Solstix is made with waterproof and tear-resistant materials, making it durable for harsh weather conditions. It has a spacious main compartment and multiple pockets for easy gear organization. The green and black color scheme adds a rugged and adventurous edge. The price is $149.99. [7]", "context": "{\"7\": {\"id\": 7, \"type\": \"Bags\", \"brand\": \"Solstix\", \"name\": \"Explorer 45L Backpack\", \"description\": \"The Explorer 45L Backpack by Solstix is perfect for your next outdoor expedition. Made with waterproof and tear-resistant materials, this backpack can withstand even the harshest weather conditions. With its spacious main compartment and multiple pockets, you can easily organize your gear. The green and black color scheme adds a rugged and adventurous edge.\", \"price\": 149.99}, \"48\": {\"id\": 48, \"type\": \"Bags\", \"brand\": \"Zephyr\", \"name\": \"Trailblazer 45L Backpack\", \"description\": \"Take everything you need for your next adventure with the Trailblazer 45L Backpack by Zephyr. This spacious backpack features multiple compartments for easy organization, adjustable shoulder straps and hip belt for a customized fit, and durable waterproof construction for ultimate protection against the elements. The classic yellow color is timeless and versatile.\", \"price\": 124.99}, \"78\": {\"id\": 78, \"type\": \"Bags\", \"brand\": \"Solstix\", \"name\": \"Basecamp Duffle Bag\", \"description\": \"The Basecamp Duffle Bag by Solstix is the ultimate adventure companion. This spacious bag is made from durable nylon and features multiple compartments for optimal organization. Its sleek red design with gray accents exudes both style and functionality.\", \"price\": 129.0}}", "latency": 24.611204, "gpt_groundedness": 5.0, "gpt_relevance": 4.0, "answer_length": 317, "citation_match": true} +{"question": "How much does the Explorer 45L Backpack by Solstix cost and what brand manufactures it?", "truth": "The Explorer 45L Backpack by Solstix is priced at 149.99 and is manufactured by the brand Solstix. [7]", "answer": "Response from target http://host.docker.internal:8000/chat is not valid JSON:\n\nInternal Server Error \nMake sure that your configuration points at a chat endpoint that returns a single JSON object.\n", "context": "Response from target http://host.docker.internal:8000/chat is not valid JSON:\n\nInternal Server Error \nMake sure that your configuration points at a chat endpoint that returns a single JSON object.\n", "latency": -1, "gpt_groundedness": 1.0, "gpt_relevance": 1.0, "answer_length": 197, "citation_match": true} +{"question": "What are the key features of the Frostbite Insulated Jacket by Grolltex?", "truth": "The Frostbite Insulated Jacket by Grolltex features a water-resistant outer shell, lightweight insulation, adjustable hood, multiple pockets, black and gray color combination, and the Grolltex logo for a touch of sophistication. [8]", "answer": "The Frostbite Insulated Jacket by Grolltex features a water-resistant outer shell, lightweight insulation, adjustable hood, multiple pockets, black and gray color combination, and a Grolltex logo. It is priced at $179.99 [8].", "context": "{\"8\": {\"id\": 8, \"type\": \"Jackets\", \"brand\": \"Grolltex\", \"name\": \"Frostbite Insulated Jacket\", \"description\": \"Stay warm and stylish with the Frostbite Insulated Jacket by Grolltex. Featuring a water-resistant outer shell and lightweight insulation, this jacket is perfect for cold weather adventures. The black and gray color combination and Grolltex logo add a touch of sophistication. With its adjustable hood and multiple pockets, this jacket offers both style and functionality.\", \"price\": 179.99}, \"68\": {\"id\": 68, \"type\": \"Jackets\", \"brand\": \"Grolltex\", \"name\": \"Mens Horizon 80s Softshell Jacket\", \"description\": \"Stay protected from the elements in the Grolltex Mens Horizon 80s Softshell Jacket. Made from a water-resistant and breathable fabric in retro 1980s style, this jacket keeps you dry and comfortable in any weather. It features multiple colors, a detachable hood, adjustable cuffs, and multiple pockets for storing your essentials. Whether you're hiking, skiing, or exploring the city, this jacket combines style and functionality.\", \"price\": 169.99}, \"55\": {\"id\": 55, \"type\": \"Jackets\", \"brand\": \"Solstix\", \"name\": \"Vigor 2.0 Insulated Jacket\", \"description\": \"Stay warm and stylish on the slopes with the Vigor 2.0 Insulated Jacket in vibrant red. This waterproof and breathable jacket is made with a 2-layer technical shell and features a detachable hood, adjustable cuffs, and multiple pockets for storage. With its modern design and ergonomic fit, it's the perfect outer layer for your winter adventures.\", \"price\": 189.99}}", "latency": 3.226864, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 225, "citation_match": true} +{"question": "What is the price of the Frostbite Insulated Jacket and which brand manufactures it?", "truth": "The Frostbite Insulated Jacket is priced at 179.99 and is manufactured by Grolltex. [8]", "answer": "The Frostbite Insulated Jacket is priced at $179.99 and is manufactured by Grolltex [8].", "context": "{\"8\": {\"id\": 8, \"type\": \"Jackets\", \"brand\": \"Grolltex\", \"name\": \"Frostbite Insulated Jacket\", \"description\": \"Stay warm and stylish with the Frostbite Insulated Jacket by Grolltex. Featuring a water-resistant outer shell and lightweight insulation, this jacket is perfect for cold weather adventures. The black and gray color combination and Grolltex logo add a touch of sophistication. With its adjustable hood and multiple pockets, this jacket offers both style and functionality.\", \"price\": 179.99}, \"49\": {\"id\": 49, \"type\": \"Jackets\", \"brand\": \"Daybird\", \"name\": \"Arctic Shield Insulated Jacket\", \"description\": \"Stay warm and stylish in the Arctic Shield Insulated Jacket by Daybird. This jacket features a water-resistant outer shell, insulated fill for exceptional warmth, and a detachable hood for added versatility. The sleek pink color is perfect for any outdoor occasion.\", \"price\": 169.99}, \"79\": {\"id\": 79, \"type\": \"Jackets\", \"brand\": \"Legend\", \"name\": \"Everest Insulated Jacket\", \"description\": \"Conquer the cold with the Everest Insulated Jacket by Legend. This jacket combines warmth and style with its insulated design and sleek grey color. The water-resistant shell will keep you dry during unexpected showers while the cozy fleece lining adds extra comfort.\", \"price\": 179.99}}", "latency": 2.705667, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 88, "citation_match": true} +{"question": "What features does the VenturePro GPS Watch by AirStrider offer?", "truth": "The VenturePro GPS Watch by AirStrider features a built-in GPS, altimeter, and compass, allowing users to navigate with confidence in any terrain. [9]", "answer": "The VenturePro GPS Watch by AirStrider offers a built-in GPS, altimeter, and compass for navigation. It has a sleek black design with an easy-to-read display, making it stylish and practical for adventurers. [9]", "context": "{\"9\": {\"id\": 9, \"type\": \"Navigation\", \"brand\": \"AirStrider\", \"name\": \"VenturePro GPS Watch\", \"description\": \"Navigate with confidence using the VenturePro GPS Watch by AirStrider. This rugged and durable watch features a built-in GPS, altimeter, and compass, allowing you to track your progress and find your way in any terrain. With its sleek black design and easy-to-read display, this watch is both stylish and practical. The VenturePro GPS Watch is a must-have for every adventurer.\", \"price\": 199.99}, \"50\": {\"id\": 50, \"type\": \"Navigation\", \"brand\": \"AirStrider\", \"name\": \"Astro GPS Navigator\", \"description\": \"Never get lost on your outdoor adventures with the Astro GPS Navigator by AirStrider. This compact and rugged device comes loaded with topographic maps, GPS tracking, waypoint storage, and a long-lasting battery. It is equipped with a high-resolution color display for easy navigation in any lighting condition. Available in a sleek gray color.\", \"price\": 249.99}, \"59\": {\"id\": 59, \"type\": \"Navigation\", \"brand\": \"Quester\", \"name\": \"TrailSeeker GPS Watch\", \"description\": \"Stay on track and explore new trails with the TrailSeeker GPS Watch. With its durable design in stealth black, this watch is packed with features like GPS navigation, heart rate monitoring, and activity tracking. It also offers a long battery life, so you can keep going without worrying about recharging. The TrailSeeker is the ultimate companion for outdoor enthusiasts who love to explore.\", \"price\": 149.99}}", "latency": 1.828663, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 211, "citation_match": true} +{"question": "What makes the VenturePro GPS Watch a must-have for adventurers?", "truth": "The VenturePro GPS Watch is described as rugged, durable, stylish, and practical with its sleek black design, easy-to-read display, and essential navigation features like GPS, altimeter, and compass. [9]", "answer": "The VenturePro GPS Watch by AirStrider is a must-have for adventurers because it is a rugged and durable watch that features a built-in GPS, altimeter, and compass. This allows you to track your progress and find your way in any terrain. With its sleek black design and easy-to-read display, the VenturePro GPS Watch is both stylish and practical, making it an essential companion for every adventurer. [9]", "context": "{\"9\": {\"id\": 9, \"type\": \"Navigation\", \"brand\": \"AirStrider\", \"name\": \"VenturePro GPS Watch\", \"description\": \"Navigate with confidence using the VenturePro GPS Watch by AirStrider. This rugged and durable watch features a built-in GPS, altimeter, and compass, allowing you to track your progress and find your way in any terrain. With its sleek black design and easy-to-read display, this watch is both stylish and practical. The VenturePro GPS Watch is a must-have for every adventurer.\", \"price\": 199.99}, \"99\": {\"id\": 99, \"type\": \"Navigation\", \"brand\": \"Solstix\", \"name\": \"Adventurer GPS Watch\", \"description\": \"Take navigation to the next level with the Solstix Adventurer GPS Watch. This sleek and durable watch, in midnight blue, features a built-in GPS, altimeter, and compass, allowing you to track your routes and monitor your progress. With multiple sport modes, it's the ideal companion for outdoor enthusiasts.\", \"price\": 199.99}, \"59\": {\"id\": 59, \"type\": \"Navigation\", \"brand\": \"Quester\", \"name\": \"TrailSeeker GPS Watch\", \"description\": \"Stay on track and explore new trails with the TrailSeeker GPS Watch. With its durable design in stealth black, this watch is packed with features like GPS navigation, heart rate monitoring, and activity tracking. It also offers a long battery life, so you can keep going without worrying about recharging. The TrailSeeker is the ultimate companion for outdoor enthusiasts who love to explore.\", \"price\": 149.99}}", "latency": 3.806653, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 406, "citation_match": true} +{"question": "What are the key features of the Trailblazer Bike Helmet by Green Equipment?", "truth": "The Trailblazer Bike Helmet by Green Equipment is lightweight and durable, with an adjustable fit system and ventilation for added comfort. It also features a vibrant green color and sleek design. [10]", "answer": "The Trailblazer Bike Helmet by Green Equipment features an adjustable fit system, ventilation for added comfort, a vibrant green color, and a sleek design. It is lightweight and durable, suitable for all types of cycling from mountain biking to road cycling. The price is $59.99 [10].", "context": "{\"10\": {\"id\": 10, \"type\": \"Cycling\", \"brand\": \"Green Equipment\", \"name\": \"Trailblazer Bike Helmet\", \"description\": \"Stay safe on your cycling adventures with the Trailblazer Bike Helmet by Green Equipment. This lightweight and durable helmet features an adjustable fit system and ventilation for added comfort. With its vibrant green color and sleek design, you'll stand out on the road. The Trailblazer Bike Helmet is perfect for all types of cycling, from mountain biking to road cycling.\", \"price\": 59.99}, \"34\": {\"id\": 34, \"type\": \"Cycling\", \"brand\": \"AirStrider\", \"name\": \"Velocity Red Bike Helmet\", \"description\": \"Protect yourself while cycling in style with the Velocity Red Bike Helmet by AirStrider. This lightweight helmet features a streamlined design, adjustable straps, and ventilation channels for optimal airflow. Stay safe on the road or trails with this vibrant red helmet.\", \"price\": 54.99}, \"25\": {\"id\": 25, \"type\": \"Cycling\", \"brand\": \"Daybird\", \"name\": \"Stealth Lite Bike Helmet\", \"description\": \"The Daybird Stealth Lite Bike Helmet is designed for cyclists who value both safety and style. With its sleek matte silver color, this helmet will make you stand out on the road. The helmet features a lightweight in-mold construction, adjustable retention system, and multiple ventilation channels for optimal airflow. Stay protected and look cool with the Daybird Stealth Lite Bike Helmet.\", \"price\": 89.99}}", "latency": 3.137111, "gpt_groundedness": 5.0, "gpt_relevance": 5.0, "answer_length": 284, "citation_match": true} +{"question": "What types of cycling is the Trailblazer Bike Helmet suitable for?", "truth": "The Trailblazer Bike Helmet is perfect for all types of cycling, from mountain biking to road cycling. [10]", "answer": "The Trailblazer Bike Helmet [10] is suitable for all types of cycling, from mountain biking to road cycling.", "context": "{\"10\": {\"id\": 10, \"type\": \"Cycling\", \"brand\": \"Green Equipment\", \"name\": \"Trailblazer Bike Helmet\", \"description\": \"Stay safe on your cycling adventures with the Trailblazer Bike Helmet by Green Equipment. This lightweight and durable helmet features an adjustable fit system and ventilation for added comfort. With its vibrant green color and sleek design, you'll stand out on the road. The Trailblazer Bike Helmet is perfect for all types of cycling, from mountain biking to road cycling.\", \"price\": 59.99}, \"34\": {\"id\": 34, \"type\": \"Cycling\", \"brand\": \"AirStrider\", \"name\": \"Velocity Red Bike Helmet\", \"description\": \"Protect yourself while cycling in style with the Velocity Red Bike Helmet by AirStrider. This lightweight helmet features a streamlined design, adjustable straps, and ventilation channels for optimal airflow. Stay safe on the road or trails with this vibrant red helmet.\", \"price\": 54.99}, \"58\": {\"id\": 58, \"type\": \"Cycling\", \"brand\": \"Raptor Elite\", \"name\": \"VelociX 2000 Bike Helmet\", \"description\": \"Protect your head in style with the VelociX 2000 Bike Helmet in glossy black. This aerodynamic helmet features an adjustable fit system, detachable visor, and 14 ventilation channels to keep you cool during intense rides. With its sleek design and lightweight construction, it's the perfect choice for road cycling, mountain biking, and everything in between.\", \"price\": 79.99}}", "latency": 1.530218, "gpt_groundedness": 5.0, "gpt_relevance": 4.0, "answer_length": 108, "citation_match": true} diff --git a/evals/results/baseline/evaluate_parameters.json b/evals/results/baseline/evaluate_parameters.json new file mode 100644 index 00000000..8d94fb06 --- /dev/null +++ b/evals/results/baseline/evaluate_parameters.json @@ -0,0 +1,15 @@ +{ + "evaluation_gpt_model": "chat", + "evaluation_timestamp": 1725037099, + "testdata_path": "/workspace/evals/ground_truth.jsonl", + "target_url": "http://host.docker.internal:8000/chat", + "target_parameters": { + "overrides": { + "use_advanced_flow": true, + "top": 3, + "retrieval_mode": "hybrid", + "temperature": 0.3 + } + }, + "num_questions": 20 +} diff --git a/evals/results/baseline/summary.json b/evals/results/baseline/summary.json new file mode 100644 index 00000000..2c6bfb70 --- /dev/null +++ b/evals/results/baseline/summary.json @@ -0,0 +1,26 @@ +{ + "gpt_groundedness": { + "pass_count": 17, + "pass_rate": 0.85, + "mean_rating": 4.4 + }, + "gpt_relevance": { + "pass_count": 17, + "pass_rate": 0.85, + "mean_rating": 4.25 + }, + "answer_length": { + "mean": 219.1, + "max": 601, + "min": 63 + }, + "latency": { + "mean": 3.94, + "max": 24.611204, + "min": -1.0 + }, + "citation_match": { + "total": 20, + "rate": 1.0 + } +} diff --git a/infra/main.bicep b/infra/main.bicep index 76e43aca..d8b19355 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -17,12 +17,31 @@ param principalId string = '' @minLength(1) @description('Location for the OpenAI resource') -// Look for gpt-35-turbo 0125 on the availability table: -// https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#standard-deployment-model-availability +// Look for desired models on the availability table: +// https://learn.microsoft.com/azure/ai-services/openai/concepts/models#global-standard-model-availability @allowed([ + 'australiaeast' + 'brazilsouth' 'canadaeast' + 'eastus' + 'eastus2' + 'francecentral' + 'germanywestcentral' + 'japaneast' + 'koreacentral' 'northcentralus' + 'norwayeast' + 'polandcentral' + 'spaincentral' + 'southafricanorth' 'southcentralus' + 'southindia' + 'swedencentral' + 'switzerlandnorth' + 'uksouth' + 'westeurope' + 'westus' + 'westus3' ]) @metadata({ azd: { @@ -52,16 +71,6 @@ param openAIEmbedHost string = 'azure' @secure() param openAIComKey string = '' -@description('Name of the GPT model to deploy') -param chatModelName string = '' -@description('Name of the model deployment') -param chatDeploymentName string = '' - -@description('Version of the GPT model to deploy') -// See version availability in this table: -// https://learn.microsoft.com/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-preview-models -param chatDeploymentVersion string = '' - param azureOpenAIAPIVersion string = '2024-03-01-preview' @secure() param azureOpenAIKey string = '' @@ -69,30 +78,70 @@ param azureOpenAIKey string = '' @description('Azure OpenAI endpoint to use, if not using the one deployed here.') param azureOpenAIEndpoint string = '' -@description('Capacity of the GPT deployment') +// Chat completion model +@description('Name of the chat model to deploy') +param chatModelName string // Set in main.parameters.json +@description('Name of the model deployment') +param chatDeploymentName string // Set in main.parameters.json + +@description('Version of the chat model to deploy') +// See version availability in this table: +// https://learn.microsoft.com/azure/ai-services/openai/concepts/models#global-standard-model-availability +param chatDeploymentVersion string // Set in main.parameters.json + +@description('Sku of the chat deployment') +param chatDeploymentSku string // Set in main.parameters.json + +@description('Capacity of the chat deployment') // You can increase this, but capacity is limited per model/region, so you will get errors if you go over // https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits -param chatDeploymentCapacity int = 0 -var chatConfig = { - modelName: !empty(chatModelName) ? chatModelName : (openAIChatHost == 'azure' ? 'gpt-35-turbo' : 'gpt-3.5-turbo') - deploymentName: !empty(chatDeploymentName) ? chatDeploymentName : 'gpt-35-turbo' - deploymentVersion: !empty(chatDeploymentVersion) ? chatDeploymentVersion : '0125' - deploymentCapacity: chatDeploymentCapacity != 0 ? chatDeploymentCapacity : 30 -} +param chatDeploymentCapacity int // Set in main.parameters.json + +@description('Whether to deploy the evaluation model') +param deployEvalModel bool // Set in main.parameters.json + +// Chat completion model used for evaluations (use most powerful model) +@description('Name of the chat model to use for evaluations') +param evalModelName string // Set in main.parameters.json +@description('Name of the model deployment for the evaluation model') +param evalDeploymentName string // Set in main.parameters.json + +@description('Version of the chat model to deploy for evaluations') +// See version availability in this table: +// https://learn.microsoft.com/azure/ai-services/openai/concepts/models#global-standard-model-availability +param evalDeploymentVersion string // Set in main.parameters.json + +@description('Sku of the model deployment for evaluations') +param evalDeploymentSku string // Set in main.parameters.json + +@description('Capacity of the chat deployment for evaluations (Go as high as possible)') +// You can increase this, but capacity is limited per model/region, so you will get errors if you go over +// https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits +param evalDeploymentCapacity string // Set in main.parameters.json + + +// Embedding model +@description('Name of the embedding model to deploy') +param embedModelName string // Set in main.parameters.json +@description('Name of the embedding model deployment') +param embedDeploymentName string // Set in main.parameters.json + +@description('Version of the embedding model to deploy') +// See version availability in this table: +// https://learn.microsoft.com/azure/ai-services/openai/concepts/models#embeddings-models +param embedDeploymentVersion string // Set in main.parameters.json + +@description('Sku of the embeddings model deployment') +param embedDeploymentSku string // Set in main.parameters.json + +@description('Capacity of the embedding deployment') +// You can increase this, but capacity is limited per model/region, so you will get errors if you go over +// https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits +param embedDeploymentCapacity int // Set in main.parameters.json + +@description('Dimensions of the embedding model') +param embedDimensions int // Set in main.parameters.json -param embedModelName string = '' -param embedDeploymentName string = '' -param embedDeploymentVersion string = '' -param embedDeploymentCapacity int = 0 -param embedDimensions int = 0 - -var embedConfig = { - modelName: !empty(embedModelName) ? embedModelName : 'text-embedding-ada-002' - deploymentName: !empty(embedDeploymentName) ? embedDeploymentName : 'text-embedding-ada-002' - deploymentVersion: !empty(embedDeploymentVersion) ? embedDeploymentVersion : '2' - deploymentCapacity: embedDeploymentCapacity != 0 ? embedDeploymentCapacity : 30 - dimensions: embedDimensions != 0 ? embedDimensions : 1536 -} param webAppExists bool = false @@ -236,11 +285,11 @@ var webAppEnv = union(azureOpenAIKeyEnv, openAIComKeyEnv, [ } { name: 'AZURE_OPENAI_CHAT_DEPLOYMENT' - value: openAIChatHost == 'azure' ? chatConfig.deploymentName : '' + value: openAIChatHost == 'azure' ? chatDeploymentName : '' } { name: 'AZURE_OPENAI_CHAT_MODEL' - value: openAIChatHost == 'azure' ? chatConfig.modelName : '' + value: openAIChatHost == 'azure' ? chatModelName : '' } { name: 'OPENAICOM_CHAT_MODEL' @@ -260,15 +309,15 @@ var webAppEnv = union(azureOpenAIKeyEnv, openAIComKeyEnv, [ } { name: 'AZURE_OPENAI_EMBED_MODEL' - value: openAIEmbedHost == 'azure' ? embedConfig.modelName : '' + value: openAIEmbedHost == 'azure' ? embedModelName : '' } { name: 'AZURE_OPENAI_EMBED_DEPLOYMENT' - value: openAIEmbedHost == 'azure' ? embedConfig.deploymentName : '' + value: openAIEmbedHost == 'azure' ? embedDeploymentName : '' } { - name: 'AZURE_OPENAI_EMBED_MODEL_DIMENSIONS' - value: openAIEmbedHost == 'azure' ? string(embedConfig.dimensions) : '' + name: 'AZURE_OPENAI_EMBED_DIMENSIONS' + value: openAIEmbedHost == 'azure' ? string(embedDimensions) : '' } { name: 'AZURE_OPENAI_ENDPOINT' @@ -276,7 +325,7 @@ var webAppEnv = union(azureOpenAIKeyEnv, openAIComKeyEnv, [ } { name: 'AZURE_OPENAI_VERSION' - value: openAIEmbedHost == 'azure' ? azureOpenAIAPIVersion : '' + value: openAIChatHost == 'azure' ? azureOpenAIAPIVersion : '' } ]) @@ -300,6 +349,48 @@ resource openAIResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' exi name: !empty(openAIResourceGroupName) ? openAIResourceGroupName : resourceGroup.name } +var defaultDeployments = [ + { + name: chatDeploymentName + model: { + format: 'OpenAI' + name: chatModelName + version: chatDeploymentVersion + } + sku: { + name: chatDeploymentSku + capacity: chatDeploymentCapacity + } +} +{ + name: embedDeploymentName + model: { + format: 'OpenAI' + name: embedModelName + version: embedDeploymentVersion + } + sku: { + name: embedDeploymentSku + capacity: embedDeploymentCapacity + } +}] + +var evalDeployment = { + name: evalDeploymentName + model: { + format: 'OpenAI' + name: evalModelName + version: evalDeploymentVersion + } + sku: { + name: evalDeploymentSku + capacity: evalDeploymentCapacity + } +} + +var openAiDeployments = deployEvalModel ? union([evalDeployment], defaultDeployments) : defaultDeployments + + module openAI 'core/ai/cognitiveservices.bicep' = if (deployAzureOpenAI) { name: 'openai' scope: openAIResourceGroup @@ -311,43 +402,18 @@ module openAI 'core/ai/cognitiveservices.bicep' = if (deployAzureOpenAI) { name: 'S0' } disableLocalAuth: true - deployments: [ - { - name: chatConfig.deploymentName - model: { - format: 'OpenAI' - name: chatConfig.modelName - version: chatConfig.deploymentVersion - } - sku: { - name: 'Standard' - capacity: chatConfig.deploymentCapacity - } - } - { - name: embedConfig.deploymentName - model: { - format: 'OpenAI' - name: embedConfig.modelName - version: embedConfig.deploymentVersion - } - sku: { - name: 'Standard' - capacity: embedConfig.deploymentCapacity - } - } - ] + deployments: openAiDeployments } } // USER ROLES -module openAIRoleUser 'core/security/role.bicep' = if (empty(runningOnGh)) { +module openAIRoleUser 'core/security/role.bicep' = { scope: openAIResourceGroup name: 'openai-role-user' params: { principalId: principalId roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' - principalType: 'User' + principalType: empty(runningOnGh) ? 'User' : 'ServicePrincipal' } } @@ -363,6 +429,7 @@ module openAIRoleBackend 'core/security/role.bicep' = { } output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId output APPLICATIONINSIGHTS_NAME string = monitoring.outputs.applicationInsightsName output AZURE_CONTAINER_ENVIRONMENT_NAME string = containerApps.outputs.environmentName @@ -375,11 +442,29 @@ output SERVICE_WEB_NAME string = web.outputs.SERVICE_WEB_NAME output SERVICE_WEB_URI string = web.outputs.SERVICE_WEB_URI output SERVICE_WEB_IMAGE_NAME string = web.outputs.SERVICE_WEB_IMAGE_NAME +output OPENAI_CHAT_HOST string = openAIChatHost +output OPENAI_EMBED_HOST string = openAIEmbedHost output AZURE_OPENAI_ENDPOINT string = !empty(azureOpenAIEndpoint) ? azureOpenAIEndpoint : (deployAzureOpenAI ? openAI.outputs.endpoint : '') -output AZURE_OPENAI_CHAT_DEPLOYMENT string = deployAzureOpenAI ? chatConfig.deploymentName : '' -output AZURE_OPENAI_EMBED_DEPLOYMENT string = deployAzureOpenAI ? embedConfig.deploymentName : '' +output AZURE_OPENAI_VERSION string = openAIEmbedHost == 'chat' ? azureOpenAIAPIVersion : '' +output AZURE_OPENAI_CHAT_DEPLOYMENT string = deployAzureOpenAI ? chatDeploymentName : '' +output AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION string = deployAzureOpenAI ? chatDeploymentVersion : '' +output AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY int = deployAzureOpenAI ? chatDeploymentCapacity : 0 +output AZURE_OPENAI_CHAT_DEPLOYMENT_SKU string = deployAzureOpenAI ? chatDeploymentSku : '' +output AZURE_OPENAI_CHAT_MODEL string = deployAzureOpenAI ? chatModelName : '' +output AZURE_OPENAI_EMBED_DEPLOYMENT string = deployAzureOpenAI ? embedDeploymentName : '' +output AZURE_OPENAI_EMBED_DEPLOYMENT_VERSION string = deployAzureOpenAI ? embedDeploymentVersion : '' +output AZURE_OPENAI_EMBED_DEPLOYMENT_CAPACITY int = deployAzureOpenAI ? embedDeploymentCapacity : 0 +output AZURE_OPENAI_EMBED_DEPLOYMENT_SKU string = deployAzureOpenAI ? embedDeploymentSku : '' +output AZURE_OPENAI_EMBED_MODEL string = deployAzureOpenAI ? embedModelName : '' +output AZURE_OPENAI_EMBED_DIMENSIONS string = deployAzureOpenAI ? string(embedDimensions) : '' + +output AZURE_OPENAI_EVAL_DEPLOYMENT string = deployAzureOpenAI ? evalDeploymentName : '' +output AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION string = deployAzureOpenAI ? evalDeploymentVersion : '' +output AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY string = deployAzureOpenAI ? evalDeploymentCapacity : '' +output AZURE_OPENAI_EVAL_DEPLOYMENT_SKU string = deployAzureOpenAI ? evalDeploymentSku : '' +output AZURE_OPENAI_EVAL_MODEL string = deployAzureOpenAI ? evalModelName : '' output POSTGRES_HOST string = postgresServer.outputs.POSTGRES_DOMAIN_NAME output POSTGRES_USERNAME string = postgresEntraAdministratorName diff --git a/infra/main.parameters.json b/infra/main.parameters.json index b67e6682..5899c214 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -33,34 +33,58 @@ "value": "${OPENAI_CHAT_HOST=azure}" }, "chatModelName": { - "value": "${AZURE_OPENAI_CHAT_MODEL}" + "value": "${AZURE_OPENAI_CHAT_MODEL=gpt-4o-mini}" }, "chatDeploymentName": { - "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT}" + "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o-mini}" }, "chatDeploymentVersion":{ - "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION}" + "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION=2024-07-18}" + }, + "chatDeploymentSku": { + "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_SKU=GlobalStandard}" }, "chatDeploymentCapacity":{ - "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY}" + "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY=30}" + }, + "deployEvalModel": { + "value": "${DEPLOY_EVAL_MODEL=false}" + }, + "evalModelName": { + "value": "${AZURE_OPENAI_EVAL_MODEL=gpt-4}" + }, + "evalDeploymentName": { + "value": "${AZURE_OPENAI_EVAL_DEPLOYMENT=gpt-4}" + }, + "evalDeploymentVersion":{ + "value": "${AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION=turbo-2024-04-09}" + }, + "evalDeploymentSku": { + "value": "${AZURE_OPENAI_EVAL_DEPLOYMENT_SKU=GlobalStandard}" + }, + "evalDeploymentCapacity":{ + "value": "${AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY=80}" }, "openAIEmbedHost": { "value": "${OPENAI_EMBED_HOST=azure}" }, "embedModelName":{ - "value": "${AZURE_OPENAI_EMBED_MODEL}" + "value": "${AZURE_OPENAI_EMBED_MODEL=text-embedding-ada-002}" }, "embedDeploymentName": { - "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT}" + "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-ada-002}" }, "embedDeploymentVersion":{ - "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_VERSION}" + "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_VERSION=2}" + }, + "embedDeploymentSku": { + "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_SKU=Standard}" }, "embedDeploymentCapacity":{ - "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_CAPACITY}" + "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_CAPACITY=30}" }, "embedDimensions": { - "value": "${AZURE_OPENAI_EMBED_DIMENSIONS}" + "value": "${AZURE_OPENAI_EMBED_DIMENSIONS=1536}" }, "openAIComKey": { "value": "${OPENAICOM_KEY}" diff --git a/pyproject.toml b/pyproject.toml index 1f168be0..00ca09ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ filterwarnings = ["ignore::DeprecationWarning"] [[tool.mypy.overrides]] module = [ "pgvector.*", + "evaltools.*" ] ignore_missing_imports = true diff --git a/requirements-dev.txt b/requirements-dev.txt index 08ce71aa..1169d8a2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,3 +12,5 @@ pytest-cov pytest-playwright pytest-snapshot locust +git+https://github.com/Azure-Samples/ai-rag-chat-evaluator/@installable +psycopg2 diff --git a/src/backend/fastapi_app/__init__.py b/src/backend/fastapi_app/__init__.py index 318eab97..e4cf5be1 100644 --- a/src/backend/fastapi_app/__init__.py +++ b/src/backend/fastapi_app/__init__.py @@ -16,7 +16,7 @@ FastAPIAppContext, common_parameters, create_async_sessionmaker, - get_azure_credentials, + get_azure_credential, ) from fastapi_app.openai_clients import create_openai_chat_client, create_openai_embed_client from fastapi_app.postgres_engine import create_postgres_engine_from_env @@ -34,7 +34,7 @@ class State(TypedDict): @asynccontextmanager async def lifespan(app: fastapi.FastAPI) -> AsyncIterator[State]: context = await common_parameters() - azure_credential = await get_azure_credentials() + azure_credential = await get_azure_credential() engine = await create_postgres_engine_from_env(azure_credential) sessionmaker = await create_async_sessionmaker(engine) chat_client = await create_openai_chat_client(azure_credential) diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index 616275c2..e1c71b93 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -52,6 +52,10 @@ class RAGContext(BaseModel): followup_questions: list[str] | None = None +class ErrorResponse(BaseModel): + error: str + + class RetrievalResponse(BaseModel): message: Message context: RAGContext diff --git a/src/backend/fastapi_app/dependencies.py b/src/backend/fastapi_app/dependencies.py index c7cc666f..d3d0177c 100644 --- a/src/backend/fastapi_app/dependencies.py +++ b/src/backend/fastapi_app/dependencies.py @@ -56,8 +56,8 @@ async def common_parameters(): openai_embed_dimensions = int(os.getenv("OPENAICOM_EMBED_DIMENSIONS", 1536)) embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN", "embedding_ada002") if OPENAI_CHAT_HOST == "azure": - openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT", "gpt-35-turbo") - openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL", "gpt-35-turbo") + openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT", "gpt-4o-mini") + openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL", "gpt-4o-mini") elif OPENAI_CHAT_HOST == "ollama": openai_chat_deployment = None openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL", "phi3:3.8b") @@ -75,8 +75,10 @@ async def common_parameters(): ) -async def get_azure_credentials() -> azure.identity.DefaultAzureCredential | azure.identity.ManagedIdentityCredential: - azure_credential: azure.identity.DefaultAzureCredential | azure.identity.ManagedIdentityCredential +async def get_azure_credential() -> ( + azure.identity.AzureDeveloperCliCredential | azure.identity.ManagedIdentityCredential +): + azure_credential: azure.identity.AzureDeveloperCliCredential | azure.identity.ManagedIdentityCredential try: if client_id := os.getenv("APP_IDENTITY_ID"): # Authenticate using a user-assigned managed identity on Azure @@ -87,7 +89,12 @@ async def get_azure_credentials() -> azure.identity.DefaultAzureCredential | azu ) azure_credential = azure.identity.ManagedIdentityCredential(client_id=client_id) else: - azure_credential = azure.identity.DefaultAzureCredential() + if tenant_id := os.getenv("AZURE_TENANT_ID"): + logger.info("Authenticating to Azure using Azure Developer CLI Credential for tenant %s", tenant_id) + azure_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=tenant_id, process_timeout=60) + else: + logger.info("Authenticating to Azure using Azure Developer CLI Credential") + azure_credential = azure.identity.AzureDeveloperCliCredential(process_timeout=60) return azure_credential except Exception as e: logger.warning("Failed to authenticate to Azure: %s", e) diff --git a/src/backend/fastapi_app/openai_clients.py b/src/backend/fastapi_app/openai_clients.py index 562118e3..0610aece 100644 --- a/src/backend/fastapi_app/openai_clients.py +++ b/src/backend/fastapi_app/openai_clients.py @@ -8,16 +8,20 @@ async def create_openai_chat_client( - azure_credential: azure.identity.DefaultAzureCredential | azure.identity.ManagedIdentityCredential, + azure_credential: azure.identity.AzureDeveloperCliCredential | azure.identity.ManagedIdentityCredential, ) -> openai.AsyncAzureOpenAI | openai.AsyncOpenAI: openai_chat_client: openai.AsyncAzureOpenAI | openai.AsyncOpenAI OPENAI_CHAT_HOST = os.getenv("OPENAI_CHAT_HOST") if OPENAI_CHAT_HOST == "azure": - api_version = os.environ["AZURE_OPENAI_VERSION"] + api_version = os.environ["AZURE_OPENAI_VERSION"] or "2024-03-01-preview" azure_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] azure_deployment = os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"] if api_key := os.getenv("AZURE_OPENAI_KEY"): - logger.info("Authenticating to Azure OpenAI using API key...") + logger.info( + "Setting up Azure OpenAI client for chat completions using API key, endpoint %s, deployment %s", + azure_endpoint, + azure_deployment, + ) openai_chat_client = openai.AsyncAzureOpenAI( api_version=api_version, azure_endpoint=azure_endpoint, @@ -25,7 +29,11 @@ async def create_openai_chat_client( api_key=api_key, ) else: - logger.info("Authenticating to Azure OpenAI Chat using Azure Identity...") + logger.info( + "Setting up Azure OpenAI client for chat completions using Azure Identity, endpoint %s, deployment %s", + azure_endpoint, + azure_deployment, + ) token_provider = azure.identity.get_bearer_token_provider( azure_credential, "https://cognitiveservices.azure.com/.default" ) @@ -36,20 +44,20 @@ async def create_openai_chat_client( azure_ad_token_provider=token_provider, ) elif OPENAI_CHAT_HOST == "ollama": - logger.info("Authenticating to OpenAI using Ollama...") + logger.info("Setting up OpenAI client for chat completions using Ollama") openai_chat_client = openai.AsyncOpenAI( base_url=os.getenv("OLLAMA_ENDPOINT"), api_key="nokeyneeded", ) else: - logger.info("Authenticating to OpenAI using OpenAI.com API key...") + logger.info("Setting up OpenAI client for chat completions using OpenAI.com API key") openai_chat_client = openai.AsyncOpenAI(api_key=os.getenv("OPENAICOM_KEY")) return openai_chat_client async def create_openai_embed_client( - azure_credential: azure.identity.DefaultAzureCredential | azure.identity.ManagedIdentityCredential, + azure_credential: azure.identity.AzureDeveloperCliCredential | azure.identity.ManagedIdentityCredential, ) -> openai.AsyncAzureOpenAI | openai.AsyncOpenAI: openai_embed_client: openai.AsyncAzureOpenAI | openai.AsyncOpenAI OPENAI_EMBED_HOST = os.getenv("OPENAI_EMBED_HOST") @@ -58,7 +66,11 @@ async def create_openai_embed_client( azure_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] azure_deployment = os.environ["AZURE_OPENAI_EMBED_DEPLOYMENT"] if api_key := os.getenv("AZURE_OPENAI_KEY"): - logger.info("Authenticating to Azure OpenAI using API key...") + logger.info( + "Setting up Azure OpenAI client for embeddings using API key, endpoint %s, deployment %s", + azure_endpoint, + azure_deployment, + ) openai_embed_client = openai.AsyncAzureOpenAI( api_version=api_version, azure_endpoint=azure_endpoint, @@ -66,7 +78,11 @@ async def create_openai_embed_client( api_key=api_key, ) else: - logger.info("Authenticating to Azure OpenAI Embedding using Azure Identity...") + logger.info( + "Setting up Azure OpenAI client for embeddings using Azure Identity, endpoint %s, deployment %s", + azure_endpoint, + azure_deployment, + ) token_provider = azure.identity.get_bearer_token_provider( azure_credential, "https://cognitiveservices.azure.com/.default" ) @@ -77,12 +93,12 @@ async def create_openai_embed_client( azure_ad_token_provider=token_provider, ) elif OPENAI_EMBED_HOST == "ollama": - logger.info("Authenticating to OpenAI using Ollama...") + logger.info("Setting up OpenAI client for embeddings using Ollama") openai_embed_client = openai.AsyncOpenAI( base_url=os.getenv("OLLAMA_ENDPOINT"), api_key="nokeyneeded", ) else: - logger.info("Authenticating to OpenAI using OpenAI.com API key...") + logger.info("Setting up OpenAI client for embeddings using OpenAI.com API key") openai_embed_client = openai.AsyncOpenAI(api_key=os.getenv("OPENAICOM_KEY")) return openai_embed_client diff --git a/src/backend/fastapi_app/postgres_engine.py b/src/backend/fastapi_app/postgres_engine.py index 0dcaf814..364dc3e9 100644 --- a/src/backend/fastapi_app/postgres_engine.py +++ b/src/backend/fastapi_app/postgres_engine.py @@ -1,10 +1,12 @@ import logging import os -from azure.identity import DefaultAzureCredential +from azure.identity import AzureDeveloperCliCredential from sqlalchemy import event from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from fastapi_app.dependencies import get_azure_credential + logger = logging.getLogger("ragapp") @@ -44,7 +46,7 @@ def update_password_token(dialect, conn_rec, cargs, cparams): async def create_postgres_engine_from_env(azure_credential=None) -> AsyncEngine: if azure_credential is None and os.environ["POSTGRES_HOST"].endswith(".database.azure.com"): - azure_credential = DefaultAzureCredential() + azure_credential = get_azure_credential() return await create_postgres_engine( host=os.environ["POSTGRES_HOST"], @@ -58,7 +60,7 @@ async def create_postgres_engine_from_env(azure_credential=None) -> AsyncEngine: async def create_postgres_engine_from_args(args, azure_credential=None) -> AsyncEngine: if azure_credential is None and args.host.endswith(".database.azure.com"): - azure_credential = DefaultAzureCredential() + azure_credential = AzureDeveloperCliCredential(process_timeout=60) return await create_postgres_engine( host=args.host, diff --git a/src/backend/fastapi_app/routes/api_routes.py b/src/backend/fastapi_app/routes/api_routes.py index adc988b0..57cbe3a1 100644 --- a/src/backend/fastapi_app/routes/api_routes.py +++ b/src/backend/fastapi_app/routes/api_routes.py @@ -9,6 +9,7 @@ from fastapi_app.api_models import ( ChatRequest, + ErrorResponse, ItemPublic, ItemWithDistance, RetrievalResponse, @@ -89,7 +90,7 @@ async def search_handler( return [ItemPublic.model_validate(item.to_dict()) for item in results] -@router.post("/chat", response_model=RetrievalResponse) +@router.post("/chat", response_model=RetrievalResponse | ErrorResponse) async def chat_handler( context: CommonDeps, database_session: DBSession, @@ -97,37 +98,40 @@ async def chat_handler( openai_chat: ChatClient, chat_request: ChatRequest, ): - searcher = PostgresSearcher( - db_session=database_session, - openai_embed_client=openai_embed.client, - embed_deployment=context.openai_embed_deployment, - embed_model=context.openai_embed_model, - embed_dimensions=context.openai_embed_dimensions, - embedding_column=context.embedding_column, - ) - rag_flow: SimpleRAGChat | AdvancedRAGChat - if chat_request.context.overrides.use_advanced_flow: - rag_flow = AdvancedRAGChat( - searcher=searcher, - openai_chat_client=openai_chat.client, - chat_model=context.openai_chat_model, - chat_deployment=context.openai_chat_deployment, + try: + searcher = PostgresSearcher( + db_session=database_session, + openai_embed_client=openai_embed.client, + embed_deployment=context.openai_embed_deployment, + embed_model=context.openai_embed_model, + embed_dimensions=context.openai_embed_dimensions, + embedding_column=context.embedding_column, ) - else: - rag_flow = SimpleRAGChat( - searcher=searcher, - openai_chat_client=openai_chat.client, - chat_model=context.openai_chat_model, - chat_deployment=context.openai_chat_deployment, + rag_flow: SimpleRAGChat | AdvancedRAGChat + if chat_request.context.overrides.use_advanced_flow: + rag_flow = AdvancedRAGChat( + searcher=searcher, + openai_chat_client=openai_chat.client, + chat_model=context.openai_chat_model, + chat_deployment=context.openai_chat_deployment, + ) + else: + rag_flow = SimpleRAGChat( + searcher=searcher, + openai_chat_client=openai_chat.client, + chat_model=context.openai_chat_model, + chat_deployment=context.openai_chat_deployment, + ) + + chat_params = rag_flow.get_params(chat_request.messages, chat_request.context.overrides) + + contextual_messages, results, thoughts = await rag_flow.prepare_context(chat_params) + response = await rag_flow.answer( + chat_params=chat_params, contextual_messages=contextual_messages, results=results, earlier_thoughts=thoughts ) - - chat_params = rag_flow.get_params(chat_request.messages, chat_request.context.overrides) - - contextual_messages, results, thoughts = await rag_flow.prepare_context(chat_params) - response = await rag_flow.answer( - chat_params=chat_params, contextual_messages=contextual_messages, results=results, earlier_thoughts=thoughts - ) - return response + return response + except Exception as e: + return {"error": str(e)} @router.post("/chat/stream") diff --git a/src/backend/fastapi_app/update_embeddings.py b/src/backend/fastapi_app/update_embeddings.py index 15165751..da8df55c 100644 --- a/src/backend/fastapi_app/update_embeddings.py +++ b/src/backend/fastapi_app/update_embeddings.py @@ -8,7 +8,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import async_sessionmaker -from fastapi_app.dependencies import common_parameters, get_azure_credentials +from fastapi_app.dependencies import common_parameters, get_azure_credential from fastapi_app.embeddings import compute_text_embedding from fastapi_app.openai_clients import create_openai_embed_client from fastapi_app.postgres_engine import create_postgres_engine_from_env @@ -18,7 +18,7 @@ async def update_embeddings(in_seed_data=False): - azure_credential = await get_azure_credentials() + azure_credential = await get_azure_credential() engine = await create_postgres_engine_from_env(azure_credential) openai_embed_client = await create_openai_embed_client(azure_credential) common_params = await common_parameters() diff --git a/tests/conftest.py b/tests/conftest.py index aaa4bc48..87a1eb22 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,11 +61,11 @@ def mock_session_env(monkeypatch_session): monkeypatch_session.setenv("OPENAI_EMBED_HOST", "azure") monkeypatch_session.setenv("AZURE_OPENAI_ENDPOINT", "https://api.openai.com") monkeypatch_session.setenv("AZURE_OPENAI_VERSION", "2024-03-01-preview") - monkeypatch_session.setenv("AZURE_OPENAI_CHAT_DEPLOYMENT", "gpt-35-turbo") - monkeypatch_session.setenv("AZURE_OPENAI_CHAT_MODEL", "gpt-35-turbo") + monkeypatch_session.setenv("AZURE_OPENAI_CHAT_DEPLOYMENT", "gpt-4o-mini") + monkeypatch_session.setenv("AZURE_OPENAI_CHAT_MODEL", "gpt-4o-mini") monkeypatch_session.setenv("AZURE_OPENAI_EMBED_DEPLOYMENT", "text-embedding-ada-002") monkeypatch_session.setenv("AZURE_OPENAI_EMBED_MODEL", "text-embedding-ada-002") - monkeypatch_session.setenv("AZURE_OPENAI_EMBED_MODEL_DIMENSIONS", "1536") + monkeypatch_session.setenv("AZURE_OPENAI_EMBED_DIMENSIONS", "1536") monkeypatch_session.setenv("AZURE_OPENAI_KEY", "fakekey") yield @@ -170,7 +170,7 @@ def mock_openai_chatcompletion(monkeypatch_session): class AsyncChatCompletionIterator: def __init__(self, answer: str): chunk_id = "test-id" - model = "gpt-35-turbo" + model = "gpt-4o-mini" self.responses = [ {"object": "chat.completion.chunk", "choices": [], "id": chunk_id, "model": model, "created": 1}, { @@ -274,22 +274,22 @@ async def mock_acreate(*args, **kwargs): @pytest.fixture(scope="function") -def mock_default_azure_credential(mock_session_env): +def mock_azure_credential(mock_session_env): """Mock the Azure credential for testing.""" - with mock.patch("azure.identity.DefaultAzureCredential") as mock_default_azure_credential: - mock_default_azure_credential.return_value = MockAzureCredential() - yield mock_default_azure_credential + with mock.patch("azure.identity.AzureDeveloperCliCredential") as mock_azure_credential: + mock_azure_credential.return_value = MockAzureCredential() + yield mock_azure_credential @pytest_asyncio.fixture(scope="function") -async def test_client(app, mock_default_azure_credential, mock_openai_embedding, mock_openai_chatcompletion): +async def test_client(app, mock_azure_credential, mock_openai_embedding, mock_openai_chatcompletion): """Create a test client.""" with TestClient(app) as test_client: yield test_client @pytest_asyncio.fixture(scope="function") -async def db_session(mock_session_env, mock_default_azure_credential): +async def db_session(mock_session_env, mock_azure_credential): """Create a new database session with a rollback at the end of the test.""" engine = await create_postgres_engine_from_env() async_sesion = async_sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -302,10 +302,10 @@ async def db_session(mock_session_env, mock_default_azure_credential): @pytest_asyncio.fixture(scope="function") -async def postgres_searcher(mock_session_env, mock_default_azure_credential, db_session, mock_openai_embedding): +async def postgres_searcher(mock_session_env, mock_azure_credential, db_session, mock_openai_embedding): from fastapi_app.postgres_searcher import PostgresSearcher - openai_embed_client = await create_openai_embed_client(mock_default_azure_credential) + openai_embed_client = await create_openai_embed_client(mock_azure_credential) yield PostgresSearcher( db_session=db_session, diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index d903a132..7ad46375 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -72,8 +72,8 @@ } ], "props": { - "model": "gpt-35-turbo", - "deployment": "gpt-35-turbo" + "model": "gpt-4o-mini", + "deployment": "gpt-4o-mini" } }, { @@ -113,8 +113,8 @@ } ], "props": { - "model": "gpt-35-turbo", - "deployment": "gpt-35-turbo" + "model": "gpt-4o-mini", + "deployment": "gpt-4o-mini" } } ], diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index cb54ff3a..3615937f 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"role":"system","content":"Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query."},{"role":"user","content":"good options for climbing gear that can be used outside?"},{"role":"assistant","tool_calls":[{"id":"call_abc123","type":"function","function":{"arguments":"{\"search_query\":\"climbing gear outside\"}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc123","content":"Search results for climbing gear that can be used outside: ..."},{"role":"user","content":"are there any shoes less than $50?"},{"role":"assistant","tool_calls":[{"id":"call_abc456","type":"function","function":{"arguments":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc456","content":"Search results for shoes cheaper than 50: ..."},{"role":"user","content":"What is the capital of France?"}],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"role":"system","content":"Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query."},{"role":"user","content":"good options for climbing gear that can be used outside?"},{"role":"assistant","tool_calls":[{"id":"call_abc123","type":"function","function":{"arguments":"{\"search_query\":\"climbing gear outside\"}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc123","content":"Search results for climbing gear that can be used outside: ..."},{"role":"user","content":"are there any shoes less than $50?"},{"role":"assistant","tool_calls":[{"id":"call_abc456","type":"function","function":{"arguments":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc456","content":"Search results for shoes cheaper than 50: ..."},{"role":"user","content":"What is the capital of France?"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null} {"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json index 0a145b99..ca9bc1bb 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json @@ -51,8 +51,8 @@ } ], "props": { - "model": "gpt-35-turbo", - "deployment": "gpt-35-turbo" + "model": "gpt-4o-mini", + "deployment": "gpt-4o-mini" } } ], diff --git a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines index bcc59765..e79e5461 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null} {"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 732bec88..66f7495e 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,15 +1,15 @@ import pytest -from fastapi_app.dependencies import common_parameters, get_azure_credentials +from fastapi_app.dependencies import common_parameters, get_azure_credential @pytest.mark.asyncio async def test_get_common_parameters(mock_session_env): result = await common_parameters() - assert result.openai_chat_model == "gpt-35-turbo" + assert result.openai_chat_model == "gpt-4o-mini" assert result.openai_embed_model == "text-embedding-ada-002" assert result.openai_embed_dimensions == 1536 - assert result.openai_chat_deployment == "gpt-35-turbo" + assert result.openai_chat_deployment == "gpt-4o-mini" assert result.openai_embed_deployment == "text-embedding-ada-002" @@ -34,8 +34,8 @@ async def test_get_common_parameters_openai(mock_session_env_openai): @pytest.mark.asyncio -async def test_get_azure_credentials(mock_session_env, mock_default_azure_credential): - result = await get_azure_credentials() +async def test_get_azure_credential(mock_session_env, mock_azure_credential): + result = await get_azure_credential() token = result.get_token("https://vault.azure.net") assert token.expires_on == 9999999999 assert token.token == "" diff --git a/tests/test_embeddings.py b/tests/test_embeddings.py index c9a5e8e2..9ecf3637 100644 --- a/tests/test_embeddings.py +++ b/tests/test_embeddings.py @@ -6,8 +6,8 @@ @pytest.mark.asyncio -async def test_compute_text_embedding(mock_default_azure_credential, mock_openai_embedding): - openai_embed_client = await create_openai_embed_client(mock_default_azure_credential) +async def test_compute_text_embedding(mock_azure_credential, mock_openai_embedding): + openai_embed_client = await create_openai_embed_client(mock_azure_credential) result = await compute_text_embedding( q="test", openai_client=openai_embed_client, @@ -18,8 +18,8 @@ async def test_compute_text_embedding(mock_default_azure_credential, mock_openai @pytest.mark.asyncio -async def test_compute_text_embedding_dimensions(mock_default_azure_credential, mock_openai_embedding): - openai_embed_client = await create_openai_embed_client(mock_default_azure_credential) +async def test_compute_text_embedding_dimensions(mock_azure_credential, mock_openai_embedding): + openai_embed_client = await create_openai_embed_client(mock_azure_credential) result = await compute_text_embedding( q="test", openai_client=openai_embed_client, diff --git a/tests/test_openai_clients.py b/tests/test_openai_clients.py index 722445a5..3ff00264 100644 --- a/tests/test_openai_clients.py +++ b/tests/test_openai_clients.py @@ -5,8 +5,8 @@ @pytest.mark.asyncio -async def test_create_openai_embed_client(mock_default_azure_credential, mock_openai_embedding): - openai_embed_client = await create_openai_embed_client(mock_default_azure_credential) +async def test_create_openai_embed_client(mock_azure_credential, mock_openai_embedding): + openai_embed_client = await create_openai_embed_client(mock_azure_credential) assert openai_embed_client.embeddings.create is not None embeddings = await openai_embed_client.embeddings.create( model="text-embedding-ada-002", input="test", dimensions=1536 @@ -15,10 +15,10 @@ async def test_create_openai_embed_client(mock_default_azure_credential, mock_op @pytest.mark.asyncio -async def test_create_openai_chat_client(mock_default_azure_credential, mock_openai_chatcompletion): - openai_chat_client = await create_openai_chat_client(mock_default_azure_credential) +async def test_create_openai_chat_client(mock_azure_credential, mock_openai_chatcompletion): + openai_chat_client = await create_openai_chat_client(mock_azure_credential) assert openai_chat_client.chat.completions.create is not None response = await openai_chat_client.chat.completions.create( - model="gpt-35-turbo", messages=[{"content": "test", "role": "user"}] + model="gpt-4o-mini", messages=[{"content": "test", "role": "user"}] ) assert response.choices[0].message.content == "The capital of France is Paris. [Benefit_Options-2.pdf]." diff --git a/tests/test_postgres_engine.py b/tests/test_postgres_engine.py index 098110b9..2bef4d6a 100644 --- a/tests/test_postgres_engine.py +++ b/tests/test_postgres_engine.py @@ -11,14 +11,14 @@ @pytest.mark.asyncio -async def test_create_postgres_engine(mock_session_env, mock_default_azure_credential): +async def test_create_postgres_engine(mock_session_env, mock_azure_credential): engine = await create_postgres_engine( host=os.environ["POSTGRES_HOST"], username=os.environ["POSTGRES_USERNAME"], database=os.environ["POSTGRES_DATABASE"], password=os.environ.get("POSTGRES_PASSWORD"), sslmode=os.environ.get("POSTGRES_SSL"), - azure_credential=mock_default_azure_credential, + azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" assert engine.url.username == "admin" @@ -28,9 +28,9 @@ async def test_create_postgres_engine(mock_session_env, mock_default_azure_crede @pytest.mark.asyncio -async def test_create_postgres_engine_from_env(mock_session_env, mock_default_azure_credential): +async def test_create_postgres_engine_from_env(mock_session_env, mock_azure_credential): engine = await create_postgres_engine_from_env( - azure_credential=mock_default_azure_credential, + azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" assert engine.url.username == "admin" @@ -40,7 +40,7 @@ async def test_create_postgres_engine_from_env(mock_session_env, mock_default_az @pytest.mark.asyncio -async def test_create_postgres_engine_from_args(mock_default_azure_credential): +async def test_create_postgres_engine_from_args(mock_azure_credential): args = type( "Args", (), @@ -54,7 +54,7 @@ async def test_create_postgres_engine_from_args(mock_default_azure_credential): ) engine = await create_postgres_engine_from_args( args=args, - azure_credential=mock_default_azure_credential, + azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" assert engine.url.username == "admin"