From 3370bf45cd836c773f710ebb5904f51da619ba21 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Oct 2024 19:40:56 +0000 Subject: [PATCH 1/3] Test eval --- evals/generate_prompt.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/evals/generate_prompt.txt b/evals/generate_prompt.txt index c73a1909..b464a270 100644 --- a/evals/generate_prompt.txt +++ b/evals/generate_prompt.txt @@ -5,5 +5,6 @@ For example, 'What climbing gear do you have?' with answer: 'We have a variety of climbing gear, including ropes, harnesses, and carabiners. [1][2]'. + Remember that customers probably don't know the names of specific brands, so your questions should be more general questions from someone who is shopping for these types of products. From 6292e7ce293ea4a5fcc47f4097cbe80b04c000f2 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Oct 2024 20:45:51 +0000 Subject: [PATCH 2/3] Check out the head --- .github/workflows/evaluate.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 607dad10..a12cd2da 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -57,7 +57,10 @@ jobs: body: "Starting evaluation! Check the Actions tab for progress, or wait for a comment with the results." }) - - uses: actions/checkout@v4 + - name: Checkout pull request 🏁 + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head - name: Install pgvector run: | From d39bed29d5333eb4389a3838f4c8169c0568d888 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Oct 2024 20:46:55 +0000 Subject: [PATCH 3/3] Revert prompt change --- evals/generate_prompt.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/evals/generate_prompt.txt b/evals/generate_prompt.txt index b464a270..c73a1909 100644 --- a/evals/generate_prompt.txt +++ b/evals/generate_prompt.txt @@ -5,6 +5,5 @@ For example, 'What climbing gear do you have?' with answer: 'We have a variety of climbing gear, including ropes, harnesses, and carabiners. [1][2]'. - Remember that customers probably don't know the names of specific brands, so your questions should be more general questions from someone who is shopping for these types of products.