fix multi observe resolution, move dom settling to a method, eval passes

evals/vanta.eval.ts

@@ -8,28 +8,38 @@ const exactMatch = (args: { input; output; expected? }) => {
   };
 };
 
-
 Eval("Vanta", {
   data: () => {
     return [
       {
-        input: {text: "find the request demo button", desired: `body > div.page-wrapper > div.nav_component > div.nav_element.w-nav > div.padding-global > div > div > nav > div.nav_cta-wrapper.is-new > a.nav_cta-button-desktop.is-smaller.w-button`},
+        input: {
+          text: "find the request demo button",
+          desired: `body > div.page-wrapper > div.nav_component > div.nav_element.w-nav > div.padding-global > div > div > nav > div.nav_cta-wrapper.is-new > a.nav_cta-button-desktop.is-smaller.w-button`,
+        },
         expected: true,
       },
     ];
   },
   task: async (input) => {
     const stageHand = new Stagehand({ env: "LOCAL" });
     await stageHand.init();
-  
+
     await stageHand.page.goto("https://www.vanta.com/");
-
+    await stageHand.waitForSettledDom();
+
     const observation = await stageHand.observe(input.text);
-    const observationResult = await stageHand.page.locator(stageHand.observations[observation].result).first().innerHTML();
-    const desiredResult = await stageHand.page.locator(input.desired).first().innerHTML();
+    const observationResult = await stageHand.page
+      .locator(stageHand.observations[observation].result)
+      .first()
+      .innerHTML();
+    const desiredResult = await stageHand.page
+      .locator(input.desired)
+      .first()
+      .innerHTML();
+
+    await stageHand.browser.close();
 
     return observationResult == desiredResult;
   },
   scores: [exactMatch],
-
-});
+});

examples/index.ts

@@ -1,6 +1,5 @@
 #!/usr/bin/env -S pnpm tsx
 import { Stagehand } from "../lib/playwright";
-import { expect } from "@playwright/test";
 
 async function example() {
   const stageHand = new Stagehand({ env: "LOCAL" });

lib/playwright/index.ts

@@ -113,9 +113,10 @@ export class Stagehand {
       "../lib/playwright/preload.js"
     );
     await this.page.addInitScript({ path: currentPath });
-    await this.page.on("domcontentloaded", async () => {
-      return this.page.evaluate(() => window.waitForDomSettle());
-    });
+  }
+
+  async waitForSettledDom() {
+    return this.page.evaluate(() => window.waitForDomSettle());
   }
 
   async cleanDOM(parent: Locator) {
@@ -148,11 +149,19 @@ export class Stagehand {
 
   async observe(observation: string): Promise<string> {
     const key = getCacheKey(observation);
-    if (this.observations[key]) {
+    const observationLocatorStr = this.observations[key].result;
+    if (observationLocatorStr) {
       console.log("cache hit!");
       console.log(`using ${JSON.stringify(this.observations[key])}`);
 
-      expect(this.page.locator(this.observations[key].result)).toBeAttached();
+      // the locator string found by the LLM might resolve to multiple places in the DOM
+      const firstLocator = await this.page
+        .locator(observationLocatorStr)
+        .first();
+
+      await expect(firstLocator).toBeAttached();
+
+      console.log("done observing");
 
       return key;
     }
@@ -233,25 +242,21 @@ export class Stagehand {
         presence_penalty: 0,
       });
 
-      if (!selectorResponse.choices[0].message.content) {
+      const locatorStr = selectorResponse.choices[0].message.content;
+
+      if (!locatorStr) {
         throw new Error("no response when finding a selector");
       }
 
-      if (selectorResponse.choices[0].message.content === "NONE") {
+      if (locatorStr === "NONE") {
         continue;
       }
 
-      expect(
-        this.page.locator(selectorResponse.choices[0].message.content)
-      ).toBeAttached();
-
-      console.log(
-        this.page.locator(selectorResponse.choices[0].message.content)
-      );
-      const key = await this.cacheObservation(
-        observation,
-        selectorResponse.choices[0].message.content
-      );
+      // the locator string found by the LLM might resolve to multiple places in the DOM
+      const firstLocator = this.page.locator(locatorStr).first();
+
+      await expect(firstLocator).toBeAttached();
+      const key = await this.cacheObservation(observation, locatorStr);
 
       return key;
     }