feat(har): allow storing content as separate files (#14934)

Author: pavelfeldman

docs/src/api/params.md

@@ -591,8 +591,9 @@ Logger sink for Playwright logging.
 * langs: js
 - `recordHar` <[Object]>
   - `omitContent` ?<[boolean]> Optional setting to control whether to omit request content from the HAR. Defaults to
-    `false`.
-  - `path` <[path]> Path on the filesystem to write the HAR file to.
+    `false`. Deprecated, use `content` policy instead.
+  - `content` ?<[HarContentPolicy]<"omit"|"embed"|"attach">> Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach` is specified, resources are persistet as separate files and all of these files are archived along with the HAR file. Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
+  - `path` <[path]> Path on the filesystem to write the HAR file to. If the file name ends with `.zip`, the har file is archived. Content `attach` will also enforce `zip` compression.
   - `urlFilter` ?<[string]|[RegExp]> A glob or regex pattern to filter requests that are stored in the HAR. When a [`option: baseURL`] via the context options was provided and the passed URL is a path, it gets merged via the [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
 
 Enables [HAR](http://www.softwareishard.com/blog/har-12-spec) recording for all pages into `recordHar.path` file. If not

packages/playwright-core/src/client/browserContext.ts

@@ -383,7 +383,7 @@ function prepareRecordHarOptions(options: BrowserContextOptions['recordHar']): c
     return;
   return {
     path: options.path,
-    omitContent: options.omitContent,
+    content: options.content || (options.omitContent ? 'omit' : 'embed'),
     urlGlob: isString(options.urlFilter) ? options.urlFilter : undefined,
     urlRegexSource: isRegExp(options.urlFilter) ? options.urlFilter.source : undefined,
     urlRegexFlags: isRegExp(options.urlFilter) ? options.urlFilter.flags : undefined,

packages/playwright-core/src/client/electron.ts

@@ -20,8 +20,7 @@ import type * as structs from '../../types/structs';
 import type * as api from '../../types/types';
 import type * as channels from '../protocol/channels';
 import { TimeoutSettings } from '../common/timeoutSettings';
-import { headersObjectToArray } from '../utils';
-import { BrowserContext } from './browserContext';
+import { BrowserContext, prepareBrowserContextParams } from './browserContext';
 import { ChannelOwner } from './channelOwner';
 import { envObjectToArray } from './clientHelper';
 import { Events } from './events';
@@ -31,10 +30,11 @@ import type { Env, WaitForEventOptions, Headers, BrowserContextOptions } from '.
 import { Waiter } from './waiter';
 import { HarRouter } from './harRouter';
 
-type ElectronOptions = Omit<channels.ElectronLaunchOptions, 'env'|'extraHTTPHeaders'> & {
+type ElectronOptions = Omit<channels.ElectronLaunchOptions, 'env'|'extraHTTPHeaders'|'recordHar'> & {
   env?: Env,
   extraHTTPHeaders?: Headers,
-  har?: BrowserContextOptions['har']
+  har?: BrowserContextOptions['har'],
+  recordHar?: BrowserContextOptions['recordHar'],
 };
 
 type ElectronAppType = typeof import('electron');
@@ -50,8 +50,7 @@ export class Electron extends ChannelOwner<channels.ElectronChannel> implements
 
   async launch(options: ElectronOptions = {}): Promise<ElectronApplication> {
     const params: channels.ElectronLaunchParams = {
-      ...options,
-      extraHTTPHeaders: options.extraHTTPHeaders && headersObjectToArray(options.extraHTTPHeaders),
+      ...await prepareBrowserContextParams(options),
       env: envObjectToArray(options.env ? options.env : process.env),
     };
     const harRouter = options.har ? await HarRouter.create(options.har) : null;

packages/playwright-core/src/client/types.ts

@@ -62,6 +62,7 @@ export type BrowserContextOptions = Omit<channels.BrowserNewContextOptions, 'vie
   recordHar?: {
     path: string,
     omitContent?: boolean,
+    content?: 'omit' | 'embed' | 'attach',
     urlFilter?: string | RegExp,
   },
 };

packages/playwright-core/src/protocol/channels.ts

@@ -264,8 +264,8 @@ export type SerializedError = {
 };
 
 export type RecordHarOptions = {
-  omitContent?: boolean,
   path: string,
+  content: 'embed' | 'attach' | 'omit',
   urlGlob?: string,
   urlRegexSource?: string,
   urlRegexFlags?: string,

packages/playwright-core/src/protocol/protocol.yml

@@ -224,8 +224,13 @@ SerializedError:
 RecordHarOptions:
   type: object
   properties:
-    omitContent: boolean?
     path: string
+    content:
+      type: enum
+      literals:
+      - embed
+      - attach
+      - omit
     urlGlob: string?
     urlRegexSource: string?
     urlRegexFlags: string?

packages/playwright-core/src/protocol/validator.ts

@@ -154,8 +154,8 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     value: tOptional(tType('SerializedValue')),
   });
   scheme.RecordHarOptions = tObject({
-    omitContent: tOptional(tBoolean),
     path: tString,
+    content: tEnum(['embed', 'attach', 'omit']),
     urlGlob: tOptional(tString),
     urlRegexSource: tOptional(tString),
     urlRegexFlags: tOptional(tString),

packages/playwright-core/src/server/browserContext.ts

@@ -17,7 +17,7 @@
 
 import * as os from 'os';
 import { TimeoutSettings } from '../common/timeoutSettings';
-import { debugMode, createGuid } from '../utils';
+import { debugMode } from '../utils';
 import { mkdirIfNeeded } from '../utils/fileUtils';
 import type { Browser, BrowserOptions } from './browser';
 import type { Download } from './download';
@@ -87,7 +87,7 @@ export abstract class BrowserContext extends SdkObject {
     this.fetchRequest = new BrowserContextAPIRequestContext(this);
 
     if (this._options.recordHar)
-      this._harRecorder = new HarRecorder(this, { ...this._options.recordHar, path: path.join(this._browser.options.artifactsDir, `${createGuid()}.har`) });
+      this._harRecorder = new HarRecorder(this, this._options.recordHar);
 
     this.tracing = new Tracing(this, browser.options.tracesDir);
   }

packages/playwright-core/src/server/har/harRecorder.ts

@@ -15,30 +15,35 @@
  */
 
 import fs from 'fs';
-import type { APIRequestContext } from '../fetch';
+import path from 'path';
 import { Artifact } from '../artifact';
 import type { BrowserContext } from '../browserContext';
 import type * as har from './har';
 import { HarTracer } from './harTracer';
 import type * as channels from '../../protocol/channels';
+import { yazl } from '../../zipBundle';
+import type { ZipFile } from '../../zipBundle';
+import { ManualPromise } from '../../utils/manualPromise';
+import type EventEmitter from 'events';
+import { createGuid } from '../../utils';
 
 export class HarRecorder {
   private _artifact: Artifact;
   private _isFlushed: boolean = false;
-  private _options: channels.RecordHarOptions;
   private _tracer: HarTracer;
   private _entries: har.Entry[] = [];
+  private _zipFile: ZipFile | null = null;
 
-  constructor(context: BrowserContext | APIRequestContext, options: channels.RecordHarOptions) {
-    this._artifact = new Artifact(context, options.path);
-    this._options = options;
+  constructor(context: BrowserContext, options: channels.RecordHarOptions) {
+    this._artifact = new Artifact(context, path.join(context._browser.options.artifactsDir, `${createGuid()}.har`));
     const urlFilterRe = options.urlRegexSource !== undefined && options.urlRegexFlags !== undefined ? new RegExp(options.urlRegexSource, options.urlRegexFlags) : undefined;
     this._tracer = new HarTracer(context, this, {
-      content: options.omitContent ? 'omit' : 'embedded',
+      content: options.content || 'embed',
       waitForContentOnStop: true,
       skipScripts: false,
       urlFilter: urlFilterRe ?? options.urlGlob,
     });
+    this._zipFile = options.content === 'attach' || options.path.endsWith('.zip') ? new yazl.ZipFile() : null;
     this._tracer.start();
   }
 
@@ -50,16 +55,33 @@ export class HarRecorder {
   }
 
   onContentBlob(sha1: string, buffer: Buffer) {
+    if (this._zipFile)
+      this._zipFile!.addBuffer(buffer, sha1);
   }
 
   async flush() {
     if (this._isFlushed)
       return;
     this._isFlushed = true;
     await this._tracer.flush();
+
     const log = this._tracer.stop();
     log.entries = this._entries;
-    await fs.promises.writeFile(this._options.path, JSON.stringify({ log }, undefined, 2));
+
+    const harFileContent = JSON.stringify({ log }, undefined, 2);
+
+    if (this._zipFile) {
+      const result = new ManualPromise<void>();
+      (this._zipFile as unknown as EventEmitter).on('error', error => result.reject(error));
+      this._zipFile.addBuffer(Buffer.from(harFileContent, 'utf-8'), 'har.har');
+      this._zipFile.end();
+      this._zipFile.outputStream.pipe(fs.createWriteStream(this._artifact.localPath())).on('close', () => {
+        result.resolve();
+      });
+      await result;
+    } else {
+      await fs.promises.writeFile(this._artifact.localPath(), harFileContent);
+    }
   }
 
   async export(): Promise<Artifact> {

packages/playwright-core/src/server/har/harTracer.ts

@@ -40,7 +40,7 @@ export interface HarTracerDelegate {
 }
 
 type HarTracerOptions = {
-  content: 'omit' | 'sha1' | 'embedded';
+  content: 'omit' | 'attach' | 'embed';
   skipScripts: boolean;
   waitForContentOnStop: boolean;
   urlFilter?: string | RegExp;
@@ -272,7 +272,7 @@ export class HarTracer {
       compressionCalculationBarrier.setDecodedBodySize(0);
     }).then(() => {
       const postData = response.request().postDataBuffer();
-      if (postData && harEntry.request.postData && this._options.content === 'sha1') {
+      if (postData && harEntry.request.postData && this._options.content === 'attach') {
         harEntry.request.postData._sha1 = calculateSha1(postData) + '.' + (mime.getExtension(harEntry.request.postData.mimeType) || 'dat');
         if (this._started)
           this._delegate.onContentBlob(harEntry.request.postData._sha1, postData);
@@ -308,7 +308,7 @@ export class HarTracer {
       return;
     }
     content.size = buffer.length;
-    if (this._options.content === 'embedded') {
+    if (this._options.content === 'embed') {
       // Sometimes, we can receive a font/media file with textual mime type. Browser
       // still interprets them correctly, but the 'content-type' header is obviously wrong.
       if (isTextualMimeType(content.mimeType) && resourceType !== 'font') {
@@ -317,7 +317,7 @@ export class HarTracer {
         content.text = buffer.toString('base64');
         content.encoding = 'base64';
       }
-    } else if (this._options.content === 'sha1') {
+    } else if (this._options.content === 'attach') {
       content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
       if (this._started)
         this._delegate.onContentBlob(content._sha1, buffer);
@@ -475,7 +475,7 @@ function createHarEntry(method: string, url: URL, requestref: string, frameref:
   return harEntry;
 }
 
-function postDataForRequest(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
+function postDataForRequest(request: network.Request, content: 'omit' | 'attach' | 'embed'): har.PostData | undefined {
   const postData = request.postDataBuffer();
   if (!postData)
     return;
@@ -484,7 +484,7 @@ function postDataForRequest(request: network.Request, content: 'omit' | 'sha1' |
   return postDataForBuffer(postData, contentType, content);
 }
 
-function postDataForBuffer(postData: Buffer | null, contentType: string | undefined, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
+function postDataForBuffer(postData: Buffer | null, contentType: string | undefined, content: 'omit' | 'attach' | 'embed'): har.PostData | undefined {
   if (!postData)
     return;
 
@@ -496,7 +496,7 @@ function postDataForBuffer(postData: Buffer | null, contentType: string | undefi
     params: []
   };
 
-  if (content === 'embedded' && contentType !== 'application/octet-stream')
+  if (content === 'embed' && contentType !== 'application/octet-stream')
     result.text = postData.toString();
 
   if (contentType === 'application/x-www-form-urlencoded') {