chore(har): allow replaying from zip har (#14962)

pavelfeldman · web-flow · commit 030e7d211cd0 · 2022-06-17T16:11:22.000-07:00
diff --git a/packages/playwright-core/src/client/harRouter.ts b/packages/playwright-core/src/client/harRouter.ts
@@ -18,48 +18,89 @@ import fs from 'fs';
 import type { HAREntry, HARFile, HARResponse } from '../../types/types';
 import { debugLogger } from '../common/debugLogger';
 import { rewriteErrorMessage } from '../utils/stackTrace';
+import { ZipFile } from '../utils/zipFile';
 import type { BrowserContext } from './browserContext';
+import { Events } from './events';
 import type { Route } from './network';
 import type { BrowserContextOptions } from './types';
 
 type HarOptions = NonNullable<BrowserContextOptions['har']>;
 
 export class HarRouter {
   private _pattern: string | RegExp;
-  private _handler: (route: Route) => Promise<void>;
+  private _harFile: HARFile;
+  private _zipFile: ZipFile | null;
+  private _options: HarOptions | undefined;
 
   static async create(options: HarOptions): Promise<HarRouter> {
+    if (options.path.endsWith('.zip')) {
+      const zipFile = new ZipFile(options.path);
+      const har = await zipFile.read('har.har');
+      const harFile = JSON.parse(har.toString()) as HARFile;
+      return new HarRouter(harFile, zipFile, options);
+    }
     const harFile = JSON.parse(await fs.promises.readFile(options.path, 'utf-8')) as HARFile;
-    return new HarRouter(harFile, options);
+    return new HarRouter(harFile, null, options);
   }
 
-  constructor(harFile: HARFile, options?: HarOptions) {
+  constructor(harFile: HARFile, zipFile: ZipFile | null, options?: HarOptions) {
+    this._harFile = harFile;
+    this._zipFile = zipFile;
     this._pattern = options?.urlFilter ?? /.*/;
-    this._handler = async (route: Route) => {
-      let response;
-      try {
-        response = harFindResponse(harFile, {
-          url: route.request().url(),
-          method: route.request().method()
+    this._options = options;
+  }
+
+  private async _handle(route: Route) {
+    let response;
+    try {
+      response = harFindResponse(this._harFile, {
+        url: route.request().url(),
+        method: route.request().method()
+      });
+    } catch (e) {
+      rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`);
+      debugLogger.log('api', e);
+    }
+
+    if (response) {
+      debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`);
+      const sha1 = (response.content as any)._sha1;
+
+      if (this._zipFile && sha1) {
+        const body = await this._zipFile.read(sha1).catch(() => {
+          debugLogger.log('api', `payload ${sha1} for request ${route.request().url()} is not found in archive`);
+          return null;
         });
-      } catch (e) {
-        rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`);
-        debugLogger.log('api', e);
+        if (body) {
+          await route.fulfill({
+            status: response.status,
+            headers: Object.fromEntries(response.headers.map(h => [h.name, h.value])),
+            body
+          });
+          return;
+        }
       }
-      if (response) {
-        debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`);
-        await route.fulfill({ response });
-      } else if (options?.fallback === 'continue') {
-        await route.fallback();
-      } else {
-        debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`);
-        await route.abort();
-      }
-    };
+
+      await route.fulfill({ response });
+      return;
+    }
+
+    if (this._options?.fallback === 'continue') {
+      await route.fallback();
+      return;
+    }
+
+    debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`);
+    await route.abort();
   }
 
   async addRoute(context: BrowserContext) {
-    await context.route(this._pattern, this._handler);
+    await context.route(this._pattern, route => this._handle(route));
+    context.once(Events.BrowserContext.Close, () => this.dispose());
+  }
+
+  dispose() {
+    this._zipFile?.close();
   }
 }
 
diff --git a/packages/playwright-core/src/utils/DEPS.list b/packages/playwright-core/src/utils/DEPS.list
@@ -3,3 +3,4 @@
 ../third_party/diff_match_patch
 ../third_party/pixelmatch
 ../utilsBundle.ts
+../zipBundle.ts
diff --git a/packages/playwright-core/src/utils/zipFile.ts b/packages/playwright-core/src/utils/zipFile.ts
@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { yauzl } from '../zipBundle';
+import type { UnzipFile, Entry } from '../zipBundle';
+
+export class ZipFile {
+  private _fileName: string;
+  private _zipFile: UnzipFile | undefined;
+  private _entries = new Map<string, Entry>();
+  private _openedPromise: Promise<void>;
+
+  constructor(fileName: string) {
+    this._fileName = fileName;
+    this._openedPromise = this._open();
+  }
+
+  private async _open() {
+    await new Promise<UnzipFile>((fulfill, reject) => {
+      yauzl.open(this._fileName, { autoClose: false }, (e, z) => {
+        if (e) {
+          reject(e);
+          return;
+        }
+        this._zipFile = z;
+        this._zipFile!.on('entry', (entry: Entry) => {
+          this._entries.set(entry.fileName, entry);
+        });
+        this._zipFile!.on('end', fulfill);
+      });
+    });
+  }
+
+  async entries(): Promise<string[]> {
+    await this._openedPromise;
+    return [...this._entries.keys()];
+  }
+
+  async read(entryPath: string): Promise<Buffer> {
+    await this._openedPromise;
+    const entry = this._entries.get(entryPath)!;
+    if (!entry)
+      throw new Error(`${entryPath} not found in file ${this._fileName}`);
+
+    return new Promise((resolve, reject) => {
+      this._zipFile!.openReadStream(entry, (error, readStream) => {
+        if (error || !readStream) {
+          reject(error || 'Entry not found');
+          return;
+        }
+
+        const buffers: Buffer[] = [];
+        readStream.on('data', data => buffers.push(data));
+        readStream.on('end', () => resolve(Buffer.concat(buffers)));
+      });
+    });
+  }
+
+  close() {
+    this._zipFile?.close();
+  }
+}
diff --git a/packages/playwright-core/src/zipBundle.ts b/packages/playwright-core/src/zipBundle.ts
@@ -17,4 +17,5 @@
 export const yazl: typeof import('../bundles/zip/node_modules/@types/yazl') = require('./zipBundleImpl').yazl;
 export type { ZipFile } from '../bundles/zip/node_modules/@types/yazl';
 export const yauzl: typeof import('../bundles/zip/node_modules/@types/yauzl') = require('./zipBundleImpl').yauzl;
+export type { ZipFile as UnzipFile, Entry } from '../bundles/zip/node_modules/@types/yauzl';
 export const extract: typeof import('../bundles/zip/node_modules/extract-zip') = require('./zipBundleImpl').extract;
diff --git a/tests/config/utils.ts b/tests/config/utils.ts
@@ -16,7 +16,7 @@
 
 import { expect } from '@playwright/test';
 import type { Frame, Page } from 'playwright-core';
-import { ZipFileSystem } from './vfs';
+import { ZipFile } from '../../packages/playwright-core/lib/utils/zipFile';
 
 export async function attachFrame(page: Page, frameId: string, url: string): Promise<Frame> {
   const handle = await page.evaluateHandle(async ({ frameId, url }) => {
@@ -91,7 +91,7 @@ export function suppressCertificateWarning() {
 }
 
 export async function parseTrace(file: string): Promise<{ events: any[], resources: Map<string, Buffer> }> {
-  const zipFS = new ZipFileSystem(file);
+  const zipFS = new ZipFile(file);
   const resources = new Map<string, Buffer>();
   for (const entry of await zipFS.entries())
     resources.set(entry, await zipFS.read(entry));
@@ -113,7 +113,7 @@ export async function parseTrace(file: string): Promise<{ events: any[], resourc
 }
 
 export async function parseHar(file: string): Promise<Map<string, Buffer>> {
-  const zipFS = new ZipFileSystem(file);
+  const zipFS = new ZipFile(file);
   const resources = new Map<string, Buffer>();
   for (const entry of await zipFS.entries())
     resources.set(entry, await zipFS.read(entry));
diff --git a/tests/config/vfs.ts b/tests/config/vfs.ts
diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts
@@ -15,7 +15,7 @@
  */
 
 import { test, expect } from './playwright-test-fixtures';
-import { ZipFileSystem } from '../config/vfs';
+import { ZipFile } from '../../packages/playwright-core/lib/utils/zipFile';
 import fs from 'fs';
 
 test('should stop tracing with trace: on-first-retry, when not retrying', async ({ runInlineTest }, testInfo) => {
@@ -243,7 +243,7 @@ test('should not override trace file in afterAll', async ({ runInlineTest, serve
 
 
 async function parseTrace(file: string): Promise<Map<string, Buffer>> {
-  const zipFS = new ZipFileSystem(file);
+  const zipFS = new ZipFile(file);
   const resources = new Map<string, Buffer>();
   for (const entry of await zipFS.entries())
     resources.set(entry, await zipFS.read(entry));
