Test evaluate

.github/workflows/app-tests.yaml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13"]
+        os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13", "windows-latest"]
         python_version: ["3.10", "3.11", "3.12"]
         exclude:
           - os: macos-latest-xlarge
@@ -53,18 +53,27 @@ jobs:
             psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'"
             psql -d postgres -c 'CREATE EXTENSION vector'
 
-        - name: (Windows) Install pgvector using install-pgvector.bat
-          if: matrix.os == 'windows-latest'
-          shell: cmd
-          run: .github\workflows\install-pgvector.bat
+        # - name: (Windows) Start postgreSQL
+        #   if: matrix.os == 'windows-latest'
+        #   uses: ikalnytskyi/action-setup-postgres@v6
+        #   with:
+        #     username: ${{ env.POSTGRES_USERNAME }}
+        #     password: ${{ env.POSTGRES_PASSWORD }}
+        #     database: ${{ env.POSTGRES_DATABASE }}
 
-        - name: (Windows) Start postgreSQL
+        - name: Install pgvector
           if: matrix.os == 'windows-latest'
-          uses: ikalnytskyi/action-setup-postgres@v6
-          with:
-            username: ${{ env.POSTGRES_USERNAME }}
-            password: ${{ env.POSTGRES_PASSWORD }}
-            database: ${{ env.POSTGRES_DATABASE }}
+          shell: cmd
+          run: |
+              call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+              cd %TEMP%
+              git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
+              cd pgvector
+              nmake /NOLOGO /F Makefile.win
+              nmake /NOLOGO /F Makefile.win install
+              sc config postgresql-x64-14 start=auto
+              net start postgresql-x64-14
+              "%PGBIN%/psql" -d postgres -c "CREATE EXTENSION vector"
 
         - name: (Linux) Install pgvector and set password
           if: matrix.os == 'ubuntu-latest'

.github/workflows/evaluate.yaml

@@ -1,4 +1,4 @@
-name: Evaluate
+name: Evaluate RAG answer flow
 
 on:
   issue_comment:
@@ -133,7 +133,7 @@ jobs:
 
       - name: Evaluate local RAG flow
         run: |
-          python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr${{ github.event.issue.number }}
+          python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=evals/results/pr${{ github.event.issue.number }}
 
       - name: Upload server logs as build artifact
         uses: actions/upload-artifact@v4
@@ -163,10 +163,12 @@ jobs:
         uses: actions/github-script@v7
         with:
           script: |
-            const { GITHUB_STEP_SUMMARY } = process.env;
+            const fs = require('fs');
+            const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+            const summary = fs.readFileSync(summaryPath, 'utf8');
             github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: GITHUB_STEP_SUMMARY
+              body: summary
             })

src/backend/fastapi_app/__init__.py

@@ -60,7 +60,7 @@ def create_app(testing: bool = False):
     if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"):
         logger.info("Configuring Azure Monitor")
         configure_azure_monitor(logger_name="ragapp")
-        # OpenAI SDK requests use httpx and are thus not auto-instrumented:
+        # OpenAI SDK requests use httpx, so are thus not auto-instrumented:
         OpenAIInstrumentor().instrument()
 
     app = fastapi.FastAPI(docs_url="/docs", lifespan=lifespan)