From 6569fdfefe7318840ee8a20e9ee954f855064460 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 30 Aug 2022 16:15:26 -0400 Subject: [PATCH 01/32] WIP initial ODP config and event manager --- .../lib/plugins/odp/odp_config.ts | 75 +++++++++++++++++++ .../lib/plugins/odp/odp_event_manager.ts | 27 +++++++ 2 files changed, 102 insertions(+) create mode 100644 packages/optimizely-sdk/lib/plugins/odp/odp_config.ts create mode 100644 packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts new file mode 100644 index 000000000..348cde962 --- /dev/null +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts @@ -0,0 +1,75 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +enum OdpConfigState { + notDetermined = 0, + integrated = 1, + notIntegrated = 2, +} + +class OdpConfig { + /** + * Fully-qualified URL for the ODP audience segments API (optional). + * @private + */ + private _apiHost?: string; + + /** + * Public API key for the ODP account from which the audience segments will be fetched (optional). + * @private + */ + private _apiKey?: string; + + /** + * All ODP segments used in the current datafile (associated with apiHost/apiKey). + * @private + */ + private _segmentsToCheck?: string[]; + + /** + * Indicates whether ODP is integrated for the project + * @private + */ + private _odpServiceIntegrated?: OdpConfigState; + + //let queue = DispatchQueue(label: "odpConfig"); + + constructor(apiKey?: string, apiHost?: string, segmentsToCheck?: string[]) { + this._apiKey = apiKey; + this._apiHost = apiHost; + this._segmentsToCheck = segmentsToCheck ?? [] as string[]; + this._odpServiceIntegrated = OdpConfigState.notDetermined; + } + + public update(apiKey?: string, apiHost?: string, segmentsToCheck?: string[]): boolean { + if (apiKey && apiHost) { + this._odpServiceIntegrated = OdpConfigState.integrated; + } else { + this._odpServiceIntegrated = OdpConfigState.notIntegrated; + } + + if (this._apiKey === apiKey && this._apiHost === apiHost && this._segmentsToCheck === segmentsToCheck) { + return false; + } else { + this._apiKey = apiKey; + this._apiHost = apiHost; + this._segmentsToCheck = segmentsToCheck; + + return true; + } + } +} diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts new file mode 100644 index 000000000..5e5050a5d --- /dev/null +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -0,0 +1,27 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface IOdpEventManager { + registerVUID(vuid: string): void; +} + +/** + * Manager for creating, persisting, and retrieving a Visitor Unique Identifier + */ +export class OdpEventManager implements IOdpEventManager { + registerVUID(vuid: string): void { + } +} From 3f1594ca86050d5a3b4f8accbdc51f6b96fdb804 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 13 Sep 2022 09:19:01 -0400 Subject: [PATCH 02/32] WIP skeleton of methods --- .../lib/plugins/odp/odp_event_manager.ts | 70 ++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 5e5050a5d..6cd5bdca3 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -14,14 +14,80 @@ * limitations under the License. */ +import { LogHandler } from '../../modules/logging'; +import { OdpClient } from './odp_client'; +import { OdpEvent } from './odp_event'; + export interface IOdpEventManager { + start(): void; + + sendEvent(event: OdpEvent): void; + + sendEvents(events: [OdpEvent]): void; + registerVUID(vuid: string): void; + + identifyUser(vuid: string, userId: string): void; } /** - * Manager for creating, persisting, and retrieving a Visitor Unique Identifier + * Manager for persisting events to the Optimizely Data Platform */ export class OdpEventManager implements IOdpEventManager { - registerVUID(vuid: string): void { + private static DEFAULT_BATCH_SIZE = 10; + private static DEFAULT_QUEUE_SIZE = 10000; + private static FLUSH_INTERVAL = 1000; + private static MAX_RETRIES = 3; + + private readonly _logger: LogHandler; + private readonly _batchSize: number; + private readonly _queueSize: number; + + private _odpConfig: OdpConfig; + private _isRunning = false; + private eventDispatcherThread: EventDispatcherThread; + + public constructor(odpConfig: OdpConfig, odpClient: OdpClient, logger: LogHandler, + batchSize = OdpEventManager.DEFAULT_BATCH_SIZE, + queueSize = OdpEventManager.DEFAULT_QUEUE_SIZE) { + this._odpConfig = odpConfig; + this._logger = logger; + this._batchSize = batchSize; + this._queueSize = queueSize; + } + + public start(): void { + this._isRunning = true; + eventDispatcherThread = new EventDispatcherThread(); + eventDispatcherThread.start(); + } + + public stop(): void { + logger.debug('Sending stop signal to ODP Event Dispatcher Thread'); + eventDispatcherThread.signalStop(); + } + + public updateSettings(odpConfig: OdpConfig): void { + this._odpConfig = odpConfig; + } + + public sendEvents(events: [OdpEvent]): void { + events.forEach(event => this.sendEvent(event)); + } + + public sendEvent(event: OdpEvent): void { + event.setData(augmentCommonData(event.getData())); + this.processEvent(event); + } + + private processEvent(event: OdpEvent): void { + + } + + public registerVUID(vuid: string): void { + + } + + public identifyUser(vuid: string, userId: string): void { } } From cecfe658ebdd0e3b8b2c21560dcb3f10767ea646 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Wed, 14 Sep 2022 17:35:40 -0400 Subject: [PATCH 03/32] WIP task queue --- .../lib/plugins/odp/odp_event_manager.ts | 14 +- .../lib/utils/task_queue/index.spec.ts | 42 ++++ .../lib/utils/task_queue/index.ts | 184 ++++++++++++++++++ 3 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts create mode 100644 packages/optimizely-sdk/lib/utils/task_queue/index.ts diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 6cd5bdca3..527e4fbc5 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -56,6 +56,13 @@ export class OdpEventManager implements IOdpEventManager { this._queueSize = queueSize; } + public registerVUID(vuid: string): void { + + } + + public identifyUser(vuid: string, userId: string): void { + } + public start(): void { this._isRunning = true; eventDispatcherThread = new EventDispatcherThread(); @@ -83,11 +90,4 @@ export class OdpEventManager implements IOdpEventManager { private processEvent(event: OdpEvent): void { } - - public registerVUID(vuid: string): void { - - } - - public identifyUser(vuid: string, userId: string): void { - } } diff --git a/packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts b/packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts new file mode 100644 index 000000000..56d6ca675 --- /dev/null +++ b/packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts @@ -0,0 +1,42 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ProcessMode, TaskQueue } from './index'; + +describe('Task Queue', () => { + it('(in node) should queue and execute function tasks in serial successfully', async () => { + const taskQueue = new TaskQueue({ + flushInterval: 1000, + maxRetries: 2, + maxQueueSize: 50, + batchSize: 3, + processMode: ProcessMode.SERIAL, + processAutomatically: false, // to control testing + }); + for (let taskId = 0; taskId <= 10; taskId = taskId + 1) { + const tasksRandomReturnTimeout = (Math.floor(Math.random() * 3) + 1) * 1000; + taskQueue.enqueue(() => setTimeout(() => { + console.log(`TaskId: ${taskId}`); + }, tasksRandomReturnTimeout)); + } + + await taskQueue.start(); + }); + + it('(in node) should queue and execute function tasks in parallel successfully', () => { + + }); +}); diff --git a/packages/optimizely-sdk/lib/utils/task_queue/index.ts b/packages/optimizely-sdk/lib/utils/task_queue/index.ts new file mode 100644 index 000000000..f5f419a30 --- /dev/null +++ b/packages/optimizely-sdk/lib/utils/task_queue/index.ts @@ -0,0 +1,184 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { v4 } from 'uuid'; + +// Inspired by +// https://github.com/Bartozzz/queue-promise +// https://gitlab.com/-/snippets/1775781 + +enum State { + IDLE = 0, + RUNNING = 1, + STOPPED = 2, +} + +export enum ProcessMode { + PARALLEL = 0, + SERIAL = 1, +} + +const DEFAULT_BATCH_SIZE = 10; +const DEFAULT_QUEUE_SIZE = 10000; +const DEFAULT_FLUSH_INTERVAL = 1000; +const DEFAULT_MAX_RETRIES = 3; + +export type TaskQueueConfiguration = { + processMode: ProcessMode, + batchSize: number, + maxQueueSize: number, + flushInterval: number, + maxRetries: number, + processAutomatically: boolean, +}; + +export class TaskQueue { + private tasks: Map any>; + private lastRan = 0; + private timeoutId: ReturnType | undefined; + private currentlyHandled = 0; + private state: State; + + private readonly options: TaskQueueConfiguration = { + processMode: ProcessMode.PARALLEL, + batchSize: DEFAULT_BATCH_SIZE, + maxQueueSize: DEFAULT_QUEUE_SIZE, + flushInterval: DEFAULT_FLUSH_INTERVAL, + maxRetries: DEFAULT_MAX_RETRIES, + processAutomatically: true, + }; + + public constructor(options: TaskQueueConfiguration) { + this.options = { ...this.options, ...options }; + this.options.flushInterval = parseInt(String(this.options.flushInterval), 10); + this.options.batchSize = parseInt(String(this.options.batchSize), 10); + + this.tasks = new Map any>(); + this.state = this.options.processAutomatically ? State.RUNNING : State.STOPPED; + } + + public start(): void { + if (this.state !== State.RUNNING && !this.isEmpty) { + this.state = State.RUNNING; + + (async () => { + while (this.shouldRun) { + await this.dequeue(); + } + })(); + } + } + + public stop(): void { + clearTimeout(this.timeoutId); + + this.state = State.STOPPED; + } + + public finalize(): void { + if (this.isEmpty) { + this.stop(); + this.state = State.IDLE; + } + } + + public async execute(): Promise { + const promises = new Array>(); + + this.tasks.forEach((fn, id) => { + if (this.currentlyHandled < this.options.batchSize) { + this.currentlyHandled++; + this.tasks.delete(id); + + promises.push(this.retry(fn)); + } + }); + + if (this.options.processMode === ProcessMode.PARALLEL) { + const results = await Promise.allSettled(promises); + return this.options.batchSize === 1 ? results[0] : results; + } else if (this.options.processMode === ProcessMode.SERIAL) { + promises.every(async promise => { + (await promise)(); + return (this.state !== State.STOPPED); + }); + } else { + throw new Error('ProcessMode not recognized'); + } + } + + private async retry(fn: () => Promise, + retriesLeft = this.options.maxRetries, + interval = this.options.flushInterval): Promise { + try { + const val = await fn(); + console.log('Resolved'); + return val; + } catch { + if (retriesLeft) { + await new Promise(r => setTimeout(r, interval)); + return this.retry(fn, retriesLeft - 1, interval); + } else { + throw new Error(`Max retries reached for function ${fn.name}`); + } + } finally { + this.finalize(); + } + } + + public dequeue(): Promise { + const { flushInterval } = this.options; + + return new Promise((resolve, reject) => { + const timeout = Math.max(0, flushInterval - (Date.now() - this.lastRan)); + + clearTimeout(this.timeoutId); + this.timeoutId = setTimeout(() => { + this.lastRan = Date.now(); + this.execute().then(resolve); + }, timeout); + }); + } + + public enqueue(tasks: () => any | Array<() => any>): void { + if (Array.isArray(tasks)) { + tasks.forEach((task) => this.enqueue(task)); + return; + } + + this.tasks.set(v4(), tasks); + + if (this.options.processAutomatically && this.state !== State.STOPPED) { + this.start(); + } + } + + public clear(): void { + this.tasks.clear(); + } + + get size(): number { + return this.tasks.size; + } + + get isEmpty(): boolean { + return this.size === 0; + } + + get shouldRun(): boolean { + return !this.isEmpty && this.state !== State.STOPPED; + } +} From f587acad6627b8193312e628f7ee27b53f1825e8 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Thu, 15 Sep 2022 16:23:53 -0400 Subject: [PATCH 04/32] Remove task queue work --- .../lib/utils/task_queue/index.spec.ts | 42 ---- .../lib/utils/task_queue/index.ts | 184 ------------------ 2 files changed, 226 deletions(-) delete mode 100644 packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts delete mode 100644 packages/optimizely-sdk/lib/utils/task_queue/index.ts diff --git a/packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts b/packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts deleted file mode 100644 index 56d6ca675..000000000 --- a/packages/optimizely-sdk/lib/utils/task_queue/index.spec.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2022, Optimizely - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ProcessMode, TaskQueue } from './index'; - -describe('Task Queue', () => { - it('(in node) should queue and execute function tasks in serial successfully', async () => { - const taskQueue = new TaskQueue({ - flushInterval: 1000, - maxRetries: 2, - maxQueueSize: 50, - batchSize: 3, - processMode: ProcessMode.SERIAL, - processAutomatically: false, // to control testing - }); - for (let taskId = 0; taskId <= 10; taskId = taskId + 1) { - const tasksRandomReturnTimeout = (Math.floor(Math.random() * 3) + 1) * 1000; - taskQueue.enqueue(() => setTimeout(() => { - console.log(`TaskId: ${taskId}`); - }, tasksRandomReturnTimeout)); - } - - await taskQueue.start(); - }); - - it('(in node) should queue and execute function tasks in parallel successfully', () => { - - }); -}); diff --git a/packages/optimizely-sdk/lib/utils/task_queue/index.ts b/packages/optimizely-sdk/lib/utils/task_queue/index.ts deleted file mode 100644 index f5f419a30..000000000 --- a/packages/optimizely-sdk/lib/utils/task_queue/index.ts +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Copyright 2022, Optimizely - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { v4 } from 'uuid'; - -// Inspired by -// https://github.com/Bartozzz/queue-promise -// https://gitlab.com/-/snippets/1775781 - -enum State { - IDLE = 0, - RUNNING = 1, - STOPPED = 2, -} - -export enum ProcessMode { - PARALLEL = 0, - SERIAL = 1, -} - -const DEFAULT_BATCH_SIZE = 10; -const DEFAULT_QUEUE_SIZE = 10000; -const DEFAULT_FLUSH_INTERVAL = 1000; -const DEFAULT_MAX_RETRIES = 3; - -export type TaskQueueConfiguration = { - processMode: ProcessMode, - batchSize: number, - maxQueueSize: number, - flushInterval: number, - maxRetries: number, - processAutomatically: boolean, -}; - -export class TaskQueue { - private tasks: Map any>; - private lastRan = 0; - private timeoutId: ReturnType | undefined; - private currentlyHandled = 0; - private state: State; - - private readonly options: TaskQueueConfiguration = { - processMode: ProcessMode.PARALLEL, - batchSize: DEFAULT_BATCH_SIZE, - maxQueueSize: DEFAULT_QUEUE_SIZE, - flushInterval: DEFAULT_FLUSH_INTERVAL, - maxRetries: DEFAULT_MAX_RETRIES, - processAutomatically: true, - }; - - public constructor(options: TaskQueueConfiguration) { - this.options = { ...this.options, ...options }; - this.options.flushInterval = parseInt(String(this.options.flushInterval), 10); - this.options.batchSize = parseInt(String(this.options.batchSize), 10); - - this.tasks = new Map any>(); - this.state = this.options.processAutomatically ? State.RUNNING : State.STOPPED; - } - - public start(): void { - if (this.state !== State.RUNNING && !this.isEmpty) { - this.state = State.RUNNING; - - (async () => { - while (this.shouldRun) { - await this.dequeue(); - } - })(); - } - } - - public stop(): void { - clearTimeout(this.timeoutId); - - this.state = State.STOPPED; - } - - public finalize(): void { - if (this.isEmpty) { - this.stop(); - this.state = State.IDLE; - } - } - - public async execute(): Promise { - const promises = new Array>(); - - this.tasks.forEach((fn, id) => { - if (this.currentlyHandled < this.options.batchSize) { - this.currentlyHandled++; - this.tasks.delete(id); - - promises.push(this.retry(fn)); - } - }); - - if (this.options.processMode === ProcessMode.PARALLEL) { - const results = await Promise.allSettled(promises); - return this.options.batchSize === 1 ? results[0] : results; - } else if (this.options.processMode === ProcessMode.SERIAL) { - promises.every(async promise => { - (await promise)(); - return (this.state !== State.STOPPED); - }); - } else { - throw new Error('ProcessMode not recognized'); - } - } - - private async retry(fn: () => Promise, - retriesLeft = this.options.maxRetries, - interval = this.options.flushInterval): Promise { - try { - const val = await fn(); - console.log('Resolved'); - return val; - } catch { - if (retriesLeft) { - await new Promise(r => setTimeout(r, interval)); - return this.retry(fn, retriesLeft - 1, interval); - } else { - throw new Error(`Max retries reached for function ${fn.name}`); - } - } finally { - this.finalize(); - } - } - - public dequeue(): Promise { - const { flushInterval } = this.options; - - return new Promise((resolve, reject) => { - const timeout = Math.max(0, flushInterval - (Date.now() - this.lastRan)); - - clearTimeout(this.timeoutId); - this.timeoutId = setTimeout(() => { - this.lastRan = Date.now(); - this.execute().then(resolve); - }, timeout); - }); - } - - public enqueue(tasks: () => any | Array<() => any>): void { - if (Array.isArray(tasks)) { - tasks.forEach((task) => this.enqueue(task)); - return; - } - - this.tasks.set(v4(), tasks); - - if (this.options.processAutomatically && this.state !== State.STOPPED) { - this.start(); - } - } - - public clear(): void { - this.tasks.clear(); - } - - get size(): number { - return this.tasks.size; - } - - get isEmpty(): boolean { - return this.size === 0; - } - - get shouldRun(): boolean { - return !this.isEmpty && this.state !== State.STOPPED; - } -} From 9cb9f3ad3b824067ff9816b8f8d43d4de663f6e5 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Mon, 19 Sep 2022 17:22:56 -0400 Subject: [PATCH 05/32] WIP event manager --- .../lib/plugins/odp/odp_event.ts | 34 +++++++ .../lib/plugins/odp/odp_event_dispatcher.ts | 93 +++++++++++++++++++ .../lib/plugins/odp/odp_event_manager.ts | 81 +++++++++++----- 3 files changed, 184 insertions(+), 24 deletions(-) create mode 100644 packages/optimizely-sdk/lib/plugins/odp/odp_event.ts create mode 100644 packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event.ts new file mode 100644 index 000000000..c1eef2903 --- /dev/null +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event.ts @@ -0,0 +1,34 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class OdpEvent { + public type: string; + + public action: string; + + public identifiers: Map; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public data: Map; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(type: string, action: string, identifiers?: Map, data?: Map) { + this.type = type; + this.action = action; + this.identifiers = identifiers ?? new Map(); + this.data = data ?? new Map(); + } +} diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts new file mode 100644 index 000000000..bf8c2f691 --- /dev/null +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -0,0 +1,93 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { OdpEvent } from './odp_event'; +import { LogHandler, LogLevel } from '../../modules/logging'; + +export class OdpEventDispatcher { + private shouldStop = false; + private currentBatch = new Array(); + private nextFlushTime: number = Date.now(); + + private readonly logger: LogHandler; + private readonly flushInterval: number; + private readonly batchSize: number; + + public constructor(logger: LogHandler, flushInterval: number, batchSize: number) { + this.logger = logger; + this.flushInterval = flushInterval; + this.batchSize = batchSize; + } + + public run(): void { + while (!this.shouldStop) { + try { + let nextEvent: OdpEvent; + + // If batch has events, set the timeout to remaining time for flush interval, + // otherwise wait for the new event indefinitely + if (this.currentBatch.length > 0) { + nextEvent = this.eventQueue.poll(this.nextFlushTime - new Date().getTime(), this.flushInterval); + } else { + nextEvent = this.eventQueue.poll(); + } + + if (nextEvent == null) { + // null means no new events received and flush interval is over, dispatch whatever is in the batch. + if (this.currentBatch.length > 0) { + this.flush(); + } + continue; + } + + if (this.currentBatch.length == 0) { + // Batch starting, create a new flush time + this.nextFlushTime = Date.now() + this.flushInterval; + } + + this.currentBatch.push(nextEvent); + + if (this.currentBatch.length >= this.batchSize) { + this.flush(); + } + } catch (e: any) { + this.logger.log(LogLevel.ERROR, e.toString()); + } + } + + this.logger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.'); + } + + private flush(): void { + if (odpConfig.isReady()) { + const payload = ODPJsonSerializerFactory.getSerializer().serializeEvents(this.currentBatch); + const endpoint = odpConfig.getApiHost() + EVENT_URL_PATH; + const statusCode: number; + let numAttempts = 0; + do { + statusCode = apiManager.sendEvents(odpConfig.getApiKey(), endpoint, payload); + numAttempts += 1; + } while (numAttempts < MAX_RETRIES && statusCode != null && (statusCode == 0 || statusCode >= 500)); + } else { + this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch'); + } + this.currentBatch = new Array(); + } + + public signalStop(): void { + this.shouldStop = true; + } +} diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 527e4fbc5..f207f9f9b 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -14,9 +14,17 @@ * limitations under the License. */ -import { LogHandler } from '../../modules/logging'; +import { LogHandler, LogLevel } from '../../modules/logging'; import { OdpClient } from './odp_client'; import { OdpEvent } from './odp_event'; +import { OdpEventDispatcher } from './odp_event_dispatcher'; +import { DEFAULT_FLUSH_INTERVAL } from '../../modules/event_processor'; +import { uuid } from '../../utils/fns'; + +const DEFAULT_BATCH_SIZE = 10; +const DEFAULT_QUEUE_SIZE = 10000; +const FLUSH_INTERVAL = 1000; +const MAX_RETRIES = 3; export interface IOdpEventManager { start(): void; @@ -34,26 +42,32 @@ export interface IOdpEventManager { * Manager for persisting events to the Optimizely Data Platform */ export class OdpEventManager implements IOdpEventManager { - private static DEFAULT_BATCH_SIZE = 10; - private static DEFAULT_QUEUE_SIZE = 10000; - private static FLUSH_INTERVAL = 1000; - private static MAX_RETRIES = 3; - private readonly _logger: LogHandler; - private readonly _batchSize: number; - private readonly _queueSize: number; + private readonly apiManager: OdpClient; + private readonly logger: LogHandler; + private readonly queueSize: number; + private readonly batchSize: number; + private readonly flushInterval: number; + + private isRunning = false; - private _odpConfig: OdpConfig; - private _isRunning = false; - private eventDispatcherThread: EventDispatcherThread; + private odpConfig: OdpConfig; + private eventQueue: Array; + private eventDispatcher: OdpEventDispatcher; public constructor(odpConfig: OdpConfig, odpClient: OdpClient, logger: LogHandler, - batchSize = OdpEventManager.DEFAULT_BATCH_SIZE, - queueSize = OdpEventManager.DEFAULT_QUEUE_SIZE) { - this._odpConfig = odpConfig; - this._logger = logger; - this._batchSize = batchSize; - this._queueSize = queueSize; + batchSize = DEFAULT_BATCH_SIZE, + queueSize = DEFAULT_QUEUE_SIZE, + flushInterval = DEFAULT_FLUSH_INTERVAL) { + this.apiManager = odpClient; + this.logger = logger; + this.queueSize = queueSize; + this.batchSize = batchSize; + this.flushInterval = flushInterval; + + this.odpConfig = odpConfig; + this.eventQueue = new Array(); + this.eventDispatcher = new OdpEventDispatcher(this.logger, this.flushInterval, this.batchSize); } public registerVUID(vuid: string): void { @@ -64,18 +78,17 @@ export class OdpEventManager implements IOdpEventManager { } public start(): void { - this._isRunning = true; - eventDispatcherThread = new EventDispatcherThread(); - eventDispatcherThread.start(); + this.isRunning = true; + this.eventDispatcher.run(); } public stop(): void { - logger.debug('Sending stop signal to ODP Event Dispatcher Thread'); - eventDispatcherThread.signalStop(); + this.logger.log(LogLevel.DEBUG, 'Sending stop signal to ODP Event Dispatcher'); + this.eventDispatcher.signalStop(); } public updateSettings(odpConfig: OdpConfig): void { - this._odpConfig = odpConfig; + this.odpConfig = odpConfig; } public sendEvents(events: [OdpEvent]): void { @@ -83,11 +96,31 @@ export class OdpEventManager implements IOdpEventManager { } public sendEvent(event: OdpEvent): void { - event.setData(augmentCommonData(event.getData())); + event.data = this.augmentCommonData(event.data); this.processEvent(event); } private processEvent(event: OdpEvent): void { } + + private augmentCommonData(sourceData: Map): Map { + let sourceVersion = ''; + if (window) { + sourceVersion = window.navigator.userAgent; + } else { + if (process) { + sourceVersion = process.version; + } + } + + const data = new Map(); + data.set('idempotence_id', uuid()); + data.set('data_source_type', 'sdk'); + data.set('data_source', 'javascript-sdk'); + data.set('data_source_version', sourceVersion); + sourceData.forEach(item => data.set(item.key, item.value)); + + return data; + } } From a5ca29111b3fbc4d67a36ac8c9de236e78de8391 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 20 Sep 2022 11:47:29 -0400 Subject: [PATCH 06/32] WIP ODP event manager needs tests and REST API sendEvents() --- .../lib/plugins/odp/odp_config.ts | 66 ++++--- .../lib/plugins/odp/odp_event_dispatcher.ts | 93 ---------- .../lib/plugins/odp/odp_event_manager.ts | 164 ++++++++++++++---- .../optimizely-sdk/lib/utils/enums/index.ts | 22 ++- 4 files changed, 190 insertions(+), 155 deletions(-) delete mode 100644 packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts index 348cde962..37ae8842b 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts @@ -13,63 +13,79 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { ODP_CONFIG_STATE } from '../../utils/enums'; - -enum OdpConfigState { - notDetermined = 0, - integrated = 1, - notIntegrated = 2, -} - -class OdpConfig { +export class OdpConfig { /** - * Fully-qualified URL for the ODP audience segments API (optional). + * Host of ODP audience segments API. * @private */ - private _apiHost?: string; + private _apiHost: string; + + /** + * Getter to retrieve the ODP server host + * @public + */ + public get apiHost() { + return this._apiHost; + } /** * Public API key for the ODP account from which the audience segments will be fetched (optional). * @private */ - private _apiKey?: string; + private _apiKey: string; + + /** + * Getter to retrieve the ODP API key + * @public + */ + public get apiKey() { + return this._apiKey; + } /** * All ODP segments used in the current datafile (associated with apiHost/apiKey). * @private */ - private _segmentsToCheck?: string[]; + private _segmentsToCheck: string[]; + + /** + * Getter for ODP segments to check + * @public + */ + public get segmentsToCheck() { + return this._segmentsToCheck; + } /** * Indicates whether ODP is integrated for the project * @private */ - private _odpServiceIntegrated?: OdpConfigState; + private _odpServiceIntegrated = ODP_CONFIG_STATE.UNDETERMINED; - //let queue = DispatchQueue(label: "odpConfig"); - - constructor(apiKey?: string, apiHost?: string, segmentsToCheck?: string[]) { + constructor(apiKey: string, apiHost: string, segmentsToCheck: string[]) { this._apiKey = apiKey; this._apiHost = apiHost; this._segmentsToCheck = segmentsToCheck ?? [] as string[]; - this._odpServiceIntegrated = OdpConfigState.notDetermined; + this._odpServiceIntegrated = this._apiKey && this._apiHost ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.UNDETERMINED; } - public update(apiKey?: string, apiHost?: string, segmentsToCheck?: string[]): boolean { - if (apiKey && apiHost) { - this._odpServiceIntegrated = OdpConfigState.integrated; - } else { - this._odpServiceIntegrated = OdpConfigState.notIntegrated; - } + public update(apiKey: string, apiEndpoint: string, segmentsToCheck: string[]): boolean { + this._odpServiceIntegrated = apiKey && apiEndpoint ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.NOT_INTEGRATED; - if (this._apiKey === apiKey && this._apiHost === apiHost && this._segmentsToCheck === segmentsToCheck) { + if (this._apiKey === apiKey && this._apiHost === apiEndpoint && this._segmentsToCheck === segmentsToCheck) { return false; } else { this._apiKey = apiKey; - this._apiHost = apiHost; + this._apiHost = apiEndpoint; this._segmentsToCheck = segmentsToCheck; return true; } } + + public isReady(): boolean { + return this._apiKey !== null && this._apiKey !== '' && this._apiHost !== null && this._apiHost !== ''; + } } diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts deleted file mode 100644 index bf8c2f691..000000000 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2022, Optimizely - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { OdpEvent } from './odp_event'; -import { LogHandler, LogLevel } from '../../modules/logging'; - -export class OdpEventDispatcher { - private shouldStop = false; - private currentBatch = new Array(); - private nextFlushTime: number = Date.now(); - - private readonly logger: LogHandler; - private readonly flushInterval: number; - private readonly batchSize: number; - - public constructor(logger: LogHandler, flushInterval: number, batchSize: number) { - this.logger = logger; - this.flushInterval = flushInterval; - this.batchSize = batchSize; - } - - public run(): void { - while (!this.shouldStop) { - try { - let nextEvent: OdpEvent; - - // If batch has events, set the timeout to remaining time for flush interval, - // otherwise wait for the new event indefinitely - if (this.currentBatch.length > 0) { - nextEvent = this.eventQueue.poll(this.nextFlushTime - new Date().getTime(), this.flushInterval); - } else { - nextEvent = this.eventQueue.poll(); - } - - if (nextEvent == null) { - // null means no new events received and flush interval is over, dispatch whatever is in the batch. - if (this.currentBatch.length > 0) { - this.flush(); - } - continue; - } - - if (this.currentBatch.length == 0) { - // Batch starting, create a new flush time - this.nextFlushTime = Date.now() + this.flushInterval; - } - - this.currentBatch.push(nextEvent); - - if (this.currentBatch.length >= this.batchSize) { - this.flush(); - } - } catch (e: any) { - this.logger.log(LogLevel.ERROR, e.toString()); - } - } - - this.logger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.'); - } - - private flush(): void { - if (odpConfig.isReady()) { - const payload = ODPJsonSerializerFactory.getSerializer().serializeEvents(this.currentBatch); - const endpoint = odpConfig.getApiHost() + EVENT_URL_PATH; - const statusCode: number; - let numAttempts = 0; - do { - statusCode = apiManager.sendEvents(odpConfig.getApiKey(), endpoint, payload); - numAttempts += 1; - } while (numAttempts < MAX_RETRIES && statusCode != null && (statusCode == 0 || statusCode >= 500)); - } else { - this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch'); - } - this.currentBatch = new Array(); - } - - public signalStop(): void { - this.shouldStop = true; - } -} diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index f207f9f9b..c2f578c13 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -17,32 +17,34 @@ import { LogHandler, LogLevel } from '../../modules/logging'; import { OdpClient } from './odp_client'; import { OdpEvent } from './odp_event'; -import { OdpEventDispatcher } from './odp_event_dispatcher'; -import { DEFAULT_FLUSH_INTERVAL } from '../../modules/event_processor'; import { uuid } from '../../utils/fns'; +import { ODP_USER_KEY } from '../../utils/enums'; +import { OdpConfig } from './odp_config'; const DEFAULT_BATCH_SIZE = 10; const DEFAULT_QUEUE_SIZE = 10000; -const FLUSH_INTERVAL = 1000; +const DEFAULT_FLUSH_INTERVAL = 1000; const MAX_RETRIES = 3; +const EVENT_URL_PATH = '/v3/events'; export interface IOdpEventManager { start(): void; - sendEvent(event: OdpEvent): void; + updateSettings(odpConfig: OdpConfig): void; + + identifyUser(vuid: string, userId: string): void; sendEvents(events: [OdpEvent]): void; - registerVUID(vuid: string): void; + sendEvent(event: OdpEvent): void; - identifyUser(vuid: string, userId: string): void; + stop(): void; } /** * Manager for persisting events to the Optimizely Data Platform */ export class OdpEventManager implements IOdpEventManager { - private readonly apiManager: OdpClient; private readonly logger: LogHandler; private readonly queueSize: number; @@ -53,28 +55,22 @@ export class OdpEventManager implements IOdpEventManager { private odpConfig: OdpConfig; private eventQueue: Array; - private eventDispatcher: OdpEventDispatcher; - - public constructor(odpConfig: OdpConfig, odpClient: OdpClient, logger: LogHandler, - batchSize = DEFAULT_BATCH_SIZE, - queueSize = DEFAULT_QUEUE_SIZE, - flushInterval = DEFAULT_FLUSH_INTERVAL) { - this.apiManager = odpClient; - this.logger = logger; - this.queueSize = queueSize; - this.batchSize = batchSize; - this.flushInterval = flushInterval; + private eventDispatcher; + public constructor(odpConfig: OdpConfig, apiManager: OdpClient, logger: LogHandler, + batchSize: number, + queueSize: number, + flushInterval: number) { this.odpConfig = odpConfig; - this.eventQueue = new Array(); - this.eventDispatcher = new OdpEventDispatcher(this.logger, this.flushInterval, this.batchSize); - } + this.apiManager = apiManager; + this.logger = logger; - public registerVUID(vuid: string): void { + this.batchSize = (batchSize != null && batchSize > 1) ? batchSize : DEFAULT_BATCH_SIZE; + this.queueSize = queueSize != null ? queueSize : DEFAULT_QUEUE_SIZE; + this.flushInterval = (flushInterval != null && flushInterval > 0) ? flushInterval : DEFAULT_FLUSH_INTERVAL; - } - - public identifyUser(vuid: string, userId: string): void { + this.eventDispatcher = new this.OdpEventDispatcher(this); + this.eventQueue = new Array(); } public start(): void { @@ -82,15 +78,21 @@ export class OdpEventManager implements IOdpEventManager { this.eventDispatcher.run(); } - public stop(): void { - this.logger.log(LogLevel.DEBUG, 'Sending stop signal to ODP Event Dispatcher'); - this.eventDispatcher.signalStop(); - } - public updateSettings(odpConfig: OdpConfig): void { this.odpConfig = odpConfig; } + public identifyUser(vuid: string, userId: string): void { + const identifiers = new Map(); + if (vuid != null) { + identifiers.set(ODP_USER_KEY.VUID, vuid); + } + identifiers.set(ODP_USER_KEY.FS_USER_ID, userId); + + const event = new OdpEvent('fullstack', 'client_initialized', identifiers); + this.sendEvent(event); + } + public sendEvents(events: [OdpEvent]): void { events.forEach(event => this.sendEvent(event)); } @@ -100,10 +102,6 @@ export class OdpEventManager implements IOdpEventManager { this.processEvent(event); } - private processEvent(event: OdpEvent): void { - - } - private augmentCommonData(sourceData: Map): Map { let sourceVersion = ''; if (window) { @@ -123,4 +121,102 @@ export class OdpEventManager implements IOdpEventManager { return data; } + + private processEvent(event: OdpEvent): void { + if (!this.isRunning) { + this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running'); + return; + } + + if (!this.odpConfig.isReady()) { + this.logger.log(LogLevel.DEBUG, 'Unable to Process ODP Event. ODPConfig is not ready.'); + return; + } + + if (this.eventQueue.length >= this.queueSize) { + this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = ' + this.queueSize); + return; + } + + if (!this.eventQueue.push(event)) { + this.logger.log(LogLevel.ERROR, 'Failed to Process ODP Event. Event Queue is not accepting any more events'); + } + } + + public stop(): void { + this.logger.log(LogLevel.DEBUG, 'Sending stop signal to ODP Event Dispatcher'); + this.eventDispatcher.signalStop(); + } + + private OdpEventDispatcher = class { + private shouldStop = false; + private currentBatch = new Array(); + private nextFlushTime: number = Date.now(); + + private readonly eventManager: OdpEventManager; + + public constructor(eventManager: OdpEventManager) { + this.eventManager = eventManager; + } + + public run(): void { + while (!this.shouldStop) { + try { + let nextEvent: OdpEvent; + + // If batch has events, set the timeout to remaining time for flush interval, + // otherwise wait for the new event indefinitely + if (this.currentBatch.length > 0) { + nextEvent = this.eventManager.eventQueue.poll(this.nextFlushTime - Date.now(), this.eventManager.flushInterval); + } else { + nextEvent = this.eventManager.eventQueue.poll(); + } + + if (nextEvent == null) { + // null means no new events received and flush interval is over, dispatch whatever is in the batch. + if (this.currentBatch.length > 0) { + this.flush(); + } + continue; + } + + if (this.currentBatch.length == 0) { + // Batch starting, create a new flush time + this.nextFlushTime = Date.now() + this.eventManager.flushInterval; + } + + this.currentBatch.push(nextEvent); + + if (this.currentBatch.length >= this.eventManager.batchSize) { + this.flush(); + } + } catch (err: any) { + this.eventManager.logger.log(LogLevel.ERROR, err.toString()); + } + } + + this.eventManager.logger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.'); + this.eventManager.isRunning = false; + } + + private flush(): void { + if (this.eventManager.odpConfig.isReady()) { + const payload = JSON.stringify(this.currentBatch); + const endpoint = this.eventManager.odpConfig.apiHost + EVENT_URL_PATH; + let statusCode: number; + let numAttempts = 0; + do { + statusCode = this.eventManager.apiManager.sendEvents(this.eventManager.odpConfig.apiKey, endpoint, payload); + numAttempts += 1; + } while (numAttempts < MAX_RETRIES && statusCode != null && (statusCode == 0 || statusCode >= 500)); + } else { + this.eventManager.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch'); + } + this.currentBatch = new Array(); + } + + public signalStop(): void { + this.shouldStop = true; + } + }; } diff --git a/packages/optimizely-sdk/lib/utils/enums/index.ts b/packages/optimizely-sdk/lib/utils/enums/index.ts index e9eecdcf4..49e048573 100644 --- a/packages/optimizely-sdk/lib/utils/enums/index.ts +++ b/packages/optimizely-sdk/lib/utils/enums/index.ts @@ -172,7 +172,7 @@ export const CONTROL_ATTRIBUTES = { BUCKETING_ID: '$opt_bucketing_id', STICKY_BUCKETING_KEY: '$opt_experiment_bucket_map', USER_AGENT: '$opt_user_agent', - FORCED_DECISION_NULL_RULE_KEY: '$opt_null_rule_key' + FORCED_DECISION_NULL_RULE_KEY: '$opt_null_rule_key', }; export const JAVASCRIPT_CLIENT_ENGINE = 'javascript-sdk'; @@ -240,7 +240,7 @@ export const DECISION_MESSAGES = { SDK_NOT_READY: 'Optimizely SDK not configured properly yet.', FLAG_KEY_INVALID: 'No flag was found for key "%s".', VARIABLE_VALUE_INVALID: 'Variable value for key "%s" is invalid or wrong type.', -} +}; /* * Notification types for use with NotificationCenter @@ -292,7 +292,6 @@ export enum NOTIFICATION_TYPES { TRACK = 'TRACK:event_key, user_id, attributes, event_tags, event', } - /** * Valid types of Javascript contexts in which this code is executing */ @@ -301,3 +300,20 @@ export enum EXECUTION_CONTEXT_TYPE { BROWSER, NODE, } + +/** + * ODP User Key + */ +export enum ODP_USER_KEY { + VUID = 'vuid', + FS_USER_ID = 'fs_user_id', +} + +/** + * Possible states of ODP integration + */ +export enum ODP_CONFIG_STATE { + UNDETERMINED = 0, + INTEGRATED, + NOT_INTEGRATED = 2, +} From 6c6ff2f528aab951a587d65f7dfeecc03d03308a Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Wed, 21 Sep 2022 13:56:10 -0400 Subject: [PATCH 07/32] Async and lint warns silenced --- .../lib/plugins/odp/odp_client.ts | 2 ++ .../lib/plugins/odp/odp_event_manager.ts | 32 +++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_client.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_client.ts index 416f312b5..6fd657497 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_client.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_client.ts @@ -81,6 +81,8 @@ export class OdpClient implements IOdpClient { try { const request = this._requestHandler.makeRequest(url, headers, method, data); response = await request.responsePromise; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { this._errorHandler.handleError(error); diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index c2f578c13..d88b44dd2 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -15,11 +15,11 @@ */ import { LogHandler, LogLevel } from '../../modules/logging'; -import { OdpClient } from './odp_client'; import { OdpEvent } from './odp_event'; import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; +import { RestApiManager } from './rest_api_manager'; const DEFAULT_BATCH_SIZE = 10; const DEFAULT_QUEUE_SIZE = 10000; @@ -45,7 +45,7 @@ export interface IOdpEventManager { * Manager for persisting events to the Optimizely Data Platform */ export class OdpEventManager implements IOdpEventManager { - private readonly apiManager: OdpClient; + private readonly apiManager: RestApiManager; private readonly logger: LogHandler; private readonly queueSize: number; private readonly batchSize: number; @@ -55,9 +55,11 @@ export class OdpEventManager implements IOdpEventManager { private odpConfig: OdpConfig; private eventQueue: Array; - private eventDispatcher; - public constructor(odpConfig: OdpConfig, apiManager: OdpClient, logger: LogHandler, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private eventDispatcher: any; + + public constructor(odpConfig: OdpConfig, apiManager: RestApiManager, logger: LogHandler, batchSize: number, queueSize: number, flushInterval: number) { @@ -102,6 +104,7 @@ export class OdpEventManager implements IOdpEventManager { this.processEvent(event); } + // eslint-disable-next-line @typescript-eslint/no-explicit-any private augmentCommonData(sourceData: Map): Map { let sourceVersion = ''; if (window) { @@ -112,6 +115,7 @@ export class OdpEventManager implements IOdpEventManager { } } + // eslint-disable-next-line @typescript-eslint/no-explicit-any const data = new Map(); data.set('idempotence_id', uuid()); data.set('data_source_type', 'sdk'); @@ -152,14 +156,13 @@ export class OdpEventManager implements IOdpEventManager { private shouldStop = false; private currentBatch = new Array(); private nextFlushTime: number = Date.now(); - private readonly eventManager: OdpEventManager; public constructor(eventManager: OdpEventManager) { this.eventManager = eventManager; } - public run(): void { + public async run(): Promise { while (!this.shouldStop) { try { let nextEvent: OdpEvent; @@ -175,7 +178,7 @@ export class OdpEventManager implements IOdpEventManager { if (nextEvent == null) { // null means no new events received and flush interval is over, dispatch whatever is in the batch. if (this.currentBatch.length > 0) { - this.flush(); + await this.flush(); } continue; } @@ -188,8 +191,11 @@ export class OdpEventManager implements IOdpEventManager { this.currentBatch.push(nextEvent); if (this.currentBatch.length >= this.eventManager.batchSize) { - this.flush(); + await this.flush(); } + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (err: any) { this.eventManager.logger.log(LogLevel.ERROR, err.toString()); } @@ -199,16 +205,16 @@ export class OdpEventManager implements IOdpEventManager { this.eventManager.isRunning = false; } - private flush(): void { + private async flush(): Promise { if (this.eventManager.odpConfig.isReady()) { - const payload = JSON.stringify(this.currentBatch); + const payload = this.currentBatch; const endpoint = this.eventManager.odpConfig.apiHost + EVENT_URL_PATH; - let statusCode: number; + let shouldRetry: boolean; let numAttempts = 0; do { - statusCode = this.eventManager.apiManager.sendEvents(this.eventManager.odpConfig.apiKey, endpoint, payload); + shouldRetry = await this.eventManager.apiManager.sendEvents(this.eventManager.odpConfig.apiKey, endpoint, payload); numAttempts += 1; - } while (numAttempts < MAX_RETRIES && statusCode != null && (statusCode == 0 || statusCode >= 500)); + } while (numAttempts < MAX_RETRIES && shouldRetry); } else { this.eventManager.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch'); } From 010370ea1ae66617073165360b773d624ab89b27 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Wed, 21 Sep 2022 13:56:26 -0400 Subject: [PATCH 08/32] ODP Event Manager test layout --- .../tests/odpEventManager.spec.ts | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 packages/optimizely-sdk/tests/odpEventManager.spec.ts diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts new file mode 100644 index 000000000..397806640 --- /dev/null +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -0,0 +1,59 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// + +describe('OdpEventManager', () => { + it('should log and discard events when event manager not running', () => { + + }); + + it('should log and discard events when event manager is not ready', () => { + + }); + + it('should log a max queue hit and discard ', () => { + + }); + + it('should dispatch events in correct number of batches', () => { + + }); + + it('should dispatch events with correct payload', () => { + + }); + + it('should dispatch events with correct flush interval', () => { + + }); + + it('should retry failed events', () => { + + }); + + it('should flush all scheduled events before stopping', () => { + + }); + + it('should prepare correct payload for identify user', () => { + + }); + + it('should apply updated ODP configuration when available', () => { + + }); +}); From 11659c87a6b9bebfb84c431eef0dfbbc0b5fd69c Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Wed, 21 Sep 2022 17:55:03 -0400 Subject: [PATCH 09/32] Finish bug fixes --- .../lib/plugins/odp/odp_event_manager.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index d88b44dd2..106af0688 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -165,26 +165,22 @@ export class OdpEventManager implements IOdpEventManager { public async run(): Promise { while (!this.shouldStop) { try { - let nextEvent: OdpEvent; - - // If batch has events, set the timeout to remaining time for flush interval, - // otherwise wait for the new event indefinitely if (this.currentBatch.length > 0) { - nextEvent = this.eventManager.eventQueue.poll(this.nextFlushTime - Date.now(), this.eventManager.flushInterval); - } else { - nextEvent = this.eventManager.eventQueue.poll(); + const remainingTimeout = this.nextFlushTime - Date.now(); + await this.pause(remainingTimeout); } - if (nextEvent == null) { - // null means no new events received and flush interval is over, dispatch whatever is in the batch. + const [nextEvent, ...remainingEventsInQueue] = this.eventManager.eventQueue; + this.eventManager.eventQueue = remainingEventsInQueue; + + if (!nextEvent) { if (this.currentBatch.length > 0) { await this.flush(); } continue; } - if (this.currentBatch.length == 0) { - // Batch starting, create a new flush time + if (this.currentBatch.length === 0) { this.nextFlushTime = Date.now() + this.eventManager.flushInterval; } @@ -205,6 +201,10 @@ export class OdpEventManager implements IOdpEventManager { this.eventManager.isRunning = false; } + private pause(timeoutMilliseconds: number): Promise { + return new Promise(resolve => setTimeout(resolve, timeoutMilliseconds)); + } + private async flush(): Promise { if (this.eventManager.odpConfig.isReady()) { const payload = this.currentBatch; From c83d58461b7749ff37b0285c553b185ba43e8037 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Mon, 26 Sep 2022 17:56:13 -0400 Subject: [PATCH 10/32] WIP ODP event manager unit tests --- .../lib/plugins/odp/odp_config.ts | 27 ++++-- .../lib/plugins/odp/odp_event_manager.ts | 41 ++++----- .../tests/odpEventManager.spec.ts | 87 ++++++++++++++++++- 3 files changed, 124 insertions(+), 31 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts index 37ae8842b..1345a767b 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ODP_CONFIG_STATE } from '../../utils/enums'; export class OdpConfig { @@ -26,7 +27,7 @@ export class OdpConfig { * Getter to retrieve the ODP server host * @public */ - public get apiHost() { + public get apiHost(): string { return this._apiHost; } @@ -40,7 +41,7 @@ export class OdpConfig { * Getter to retrieve the ODP API key * @public */ - public get apiKey() { + public get apiKey(): string { return this._apiKey; } @@ -54,7 +55,7 @@ export class OdpConfig { * Getter for ODP segments to check * @public */ - public get segmentsToCheck() { + public get segmentsToCheck(): string[] { return this._segmentsToCheck; } @@ -64,27 +65,37 @@ export class OdpConfig { */ private _odpServiceIntegrated = ODP_CONFIG_STATE.UNDETERMINED; - constructor(apiKey: string, apiHost: string, segmentsToCheck: string[]) { + constructor(apiKey: string, apiHost: string, segmentsToCheck?: string[]) { this._apiKey = apiKey; this._apiHost = apiHost; this._segmentsToCheck = segmentsToCheck ?? [] as string[]; this._odpServiceIntegrated = this._apiKey && this._apiHost ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.UNDETERMINED; } - public update(apiKey: string, apiEndpoint: string, segmentsToCheck: string[]): boolean { - this._odpServiceIntegrated = apiKey && apiEndpoint ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.NOT_INTEGRATED; + /** + * Update the ODP configuration details + * @param apiKey Public API key for the ODP account + * @param apiHost Host of ODP audience segments API + * @param segmentsToCheck Audience segments + * @returns true if configuration was updated successfully + */ + public update(apiKey: string, apiHost: string, segmentsToCheck: string[]): boolean { + this._odpServiceIntegrated = apiKey && apiHost ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.NOT_INTEGRATED; - if (this._apiKey === apiKey && this._apiHost === apiEndpoint && this._segmentsToCheck === segmentsToCheck) { + if (this._apiKey === apiKey && this._apiHost === apiHost && this._segmentsToCheck === segmentsToCheck) { return false; } else { this._apiKey = apiKey; - this._apiHost = apiEndpoint; + this._apiHost = apiHost; this._segmentsToCheck = segmentsToCheck; return true; } } + /** + * Determines if ODP configuration has the minimum amount of information + */ public isReady(): boolean { return this._apiKey !== null && this._apiKey !== '' && this._apiHost !== null && this._apiHost !== ''; } diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 106af0688..49e1054bb 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -56,20 +56,19 @@ export class OdpEventManager implements IOdpEventManager { private odpConfig: OdpConfig; private eventQueue: Array; - // eslint-disable-next-line @typescript-eslint/no-explicit-any private eventDispatcher: any; public constructor(odpConfig: OdpConfig, apiManager: RestApiManager, logger: LogHandler, - batchSize: number, - queueSize: number, - flushInterval: number) { + batchSize?: number, + queueSize?: number, + flushInterval?: number) { this.odpConfig = odpConfig; this.apiManager = apiManager; this.logger = logger; - this.batchSize = (batchSize != null && batchSize > 1) ? batchSize : DEFAULT_BATCH_SIZE; - this.queueSize = queueSize != null ? queueSize : DEFAULT_QUEUE_SIZE; - this.flushInterval = (flushInterval != null && flushInterval > 0) ? flushInterval : DEFAULT_FLUSH_INTERVAL; + this.batchSize = batchSize && batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE; + this.queueSize = queueSize && queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE; + this.flushInterval = flushInterval && flushInterval > 0 ? flushInterval : DEFAULT_FLUSH_INTERVAL; this.eventDispatcher = new this.OdpEventDispatcher(this); this.eventQueue = new Array(); @@ -95,7 +94,7 @@ export class OdpEventManager implements IOdpEventManager { this.sendEvent(event); } - public sendEvents(events: [OdpEvent]): void { + public sendEvents(events: OdpEvent[]): void { events.forEach(event => this.sendEvent(event)); } @@ -104,8 +103,8 @@ export class OdpEventManager implements IOdpEventManager { this.processEvent(event); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private augmentCommonData(sourceData: Map): Map { + private augmentCommonData(sourceData: Map): Map { + // Try to get information from the current execution context let sourceVersion = ''; if (window) { sourceVersion = window.navigator.userAgent; @@ -115,20 +114,21 @@ export class OdpEventManager implements IOdpEventManager { } } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const data = new Map(); + const data = new Map(); data.set('idempotence_id', uuid()); data.set('data_source_type', 'sdk'); data.set('data_source', 'javascript-sdk'); - data.set('data_source_version', sourceVersion); - sourceData.forEach(item => data.set(item.key, item.value)); + if (sourceVersion) { + data.set('data_source_version', sourceVersion); + } + sourceData.forEach((value, key) => data.set(key, value)); return data; } private processEvent(event: OdpEvent): void { if (!this.isRunning) { - this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running'); + this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.'); return; } @@ -138,12 +138,12 @@ export class OdpEventManager implements IOdpEventManager { } if (this.eventQueue.length >= this.queueSize) { - this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = ' + this.queueSize); + this.logger.log(LogLevel.WARNING, `Failed to Process ODP Event. Event Queue full. queueSize = ${this.queueSize}.`); return; } if (!this.eventQueue.push(event)) { - this.logger.log(LogLevel.ERROR, 'Failed to Process ODP Event. Event Queue is not accepting any more events'); + this.logger.log(LogLevel.ERROR, 'Failed to Process ODP Event. Event Queue is not accepting any more events.'); } } @@ -189,11 +189,8 @@ export class OdpEventManager implements IOdpEventManager { if (this.currentBatch.length >= this.eventManager.batchSize) { await this.flush(); } - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } catch (err: any) { - this.eventManager.logger.log(LogLevel.ERROR, err.toString()); + } catch (err) { + this.eventManager.logger.log(LogLevel.ERROR, err); } } diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index 397806640..79c6eaffe 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -14,19 +14,104 @@ * limitations under the License. */ -/// +import { OdpConfig } from '../lib/plugins/odp/odp_config'; +import { OdpEventManager } from '../lib/plugins/odp/odp_event_manager'; +import { instance, mock, resetCalls, verify, when } from 'ts-mockito'; +import { RestApiManager } from '../lib/plugins/odp/rest_api_manager'; +import { LogHandler, LogLevel } from '../lib/modules/logging'; +import { OdpEvent } from '../lib/plugins/odp/odp_event'; + +const API_KEY = 'test-api-key'; +const API_HOST = 'https://odp.example.com/'; +const MOCK_PROCESS_VERSION = 'v16.17.0'; +const MOCK_IDEMPOTENCE_ID = 'c1dc758e-f095-4f09-9b49-172d74c53880'; +const EVENTS: OdpEvent[] = [ + new OdpEvent( + 't1', + 'a1', + new Map([['id-key-1', 'id-value-1']]), + new Map(Object.entries({ 'key-1': 'value1', 'key-2': null, 'key-3': 3.3, 'key-4': true })), + ), + new OdpEvent( + 't2', + 'a2', + new Map([['id-key-2', 'id-value-2']]), + new Map(Object.entries({ 'key-2': 'value2' })), + ), +]; +const PROCESSED_EVENTS = [ + { + 'type': 't1', + 'action': 'a1', + 'identifiers': { 'id-key-1': 'id-value-1' }, + 'data': { + 'idempotence_id': MOCK_IDEMPOTENCE_ID, + 'data_source_type': 'sdk', + 'data_source': 'javascript-sdk', + 'data_source_version': MOCK_PROCESS_VERSION, + 'key-1': 'value1', + 'key-2': null, + 'key-3': 3.3, + 'key-4': true, + }, + }, + { + 'type': 't2', + 'action': 'a2', + 'identifiers': { 'id-key-2': 'id-value-2' }, + 'data': { + 'idempotence_id': MOCK_IDEMPOTENCE_ID, + 'data_source_type': 'sdk', + 'data_source': 'javascript-sdk', + 'data_source_version': '', + 'key-2': 'value2', + }, + }, +]; describe('OdpEventManager', () => { + let mockLogger: LogHandler; + let mockOdpConfig: OdpConfig; + let mockRestApiManager: RestApiManager; + + beforeAll(() => { + mockLogger = mock(); + mockOdpConfig = mock(); + mockRestApiManager = mock(); + }); + + beforeEach(() => { + resetCalls(mockLogger); + resetCalls(mockOdpConfig); + resetCalls(mockRestApiManager); + }); + it('should log and discard events when event manager not running', () => { + const eventManager = new OdpEventManager(instance(mockOdpConfig), instance(mockRestApiManager), instance(mockLogger)); + eventManager.sendEvent(EVENTS[0]); + + verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.')).once(); }); it('should log and discard events when event manager is not ready', () => { + when(mockOdpConfig.isReady()).thenReturn(false); + const eventManager = new OdpEventManager(instance(mockOdpConfig), instance(mockRestApiManager), instance(mockLogger)); + + eventManager.sendEvent(EVENTS[0]); + verify(mockLogger.log(LogLevel.DEBUG, 'Unable to Process ODP Event. ODPConfig is not ready.')).once(); }); it('should log a max queue hit and discard ', () => { + when(mockOdpConfig.isReady()).thenReturn(true); + const eventManager = new OdpEventManager(instance(mockOdpConfig), instance(mockRestApiManager), instance(mockLogger), 1, 1, 1); + eventManager['isRunning'] = true; // simulate dispatcher already running + eventManager['eventQueue'].push(EVENTS[0]); // simulate queue already having 1 + + eventManager.sendEvent(EVENTS[1]); // try adding to queue + verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = 1.')).once(); }); it('should dispatch events in correct number of batches', () => { From f28430a1fe721401f66c2b86581196c97342e4a2 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 27 Sep 2022 14:18:48 -0400 Subject: [PATCH 11/32] WIP help from John --- .../lib/plugins/odp/odp_event_manager.ts | 5 +- .../tests/odpEventManager.spec.ts | 193 +++++++++++++++--- 2 files changed, 163 insertions(+), 35 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 49e1054bb..52ef4f7b8 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -165,9 +165,12 @@ export class OdpEventManager implements IOdpEventManager { public async run(): Promise { while (!this.shouldStop) { try { + console.log('batch length', this.currentBatch.length); if (this.currentBatch.length > 0) { const remainingTimeout = this.nextFlushTime - Date.now(); + console.log('waiting'); await this.pause(remainingTimeout); + console.log('waited'); } const [nextEvent, ...remainingEventsInQueue] = this.eventManager.eventQueue; @@ -190,7 +193,7 @@ export class OdpEventManager implements IOdpEventManager { await this.flush(); } } catch (err) { - this.eventManager.logger.log(LogLevel.ERROR, err); + this.eventManager.logger.log(LogLevel.ERROR, err as string); } } diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index 79c6eaffe..f6bea4190 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -16,13 +16,14 @@ import { OdpConfig } from '../lib/plugins/odp/odp_config'; import { OdpEventManager } from '../lib/plugins/odp/odp_event_manager'; -import { instance, mock, resetCalls, verify, when } from 'ts-mockito'; +import { anything, capture, instance, mock, resetCalls, spy, verify, when } from 'ts-mockito'; import { RestApiManager } from '../lib/plugins/odp/rest_api_manager'; import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; +import { RequestHandler } from '../lib/utils/http_request_handler/http'; const API_KEY = 'test-api-key'; -const API_HOST = 'https://odp.example.com/'; +const API_HOST = 'https://odp.example.com'; const MOCK_PROCESS_VERSION = 'v16.17.0'; const MOCK_IDEMPOTENCE_ID = 'c1dc758e-f095-4f09-9b49-172d74c53880'; const EVENTS: OdpEvent[] = [ @@ -30,21 +31,28 @@ const EVENTS: OdpEvent[] = [ 't1', 'a1', new Map([['id-key-1', 'id-value-1']]), - new Map(Object.entries({ 'key-1': 'value1', 'key-2': null, 'key-3': 3.3, 'key-4': true })), + new Map(Object.entries({ + 'key-1': 'value1', + 'key-2': null, + 'key-3': 3.3, + 'key-4': true, + })), ), new OdpEvent( 't2', 'a2', new Map([['id-key-2', 'id-value-2']]), - new Map(Object.entries({ 'key-2': 'value2' })), + new Map(Object.entries({ + 'key-2': 'value2', + })), ), ]; -const PROCESSED_EVENTS = [ - { - 'type': 't1', - 'action': 'a1', - 'identifiers': { 'id-key-1': 'id-value-1' }, - 'data': { +const PROCESSED_EVENTS: OdpEvent[] = [ + new OdpEvent( + 't1', + 'a1', + new Map([['id-key-1', 'id-value-1']]), + new Map(Object.entries({ 'idempotence_id': MOCK_IDEMPOTENCE_ID, 'data_source_type': 'sdk', 'data_source': 'javascript-sdk', @@ -53,41 +61,72 @@ const PROCESSED_EVENTS = [ 'key-2': null, 'key-3': 3.3, 'key-4': true, - }, - }, - { - 'type': 't2', - 'action': 'a2', - 'identifiers': { 'id-key-2': 'id-value-2' }, - 'data': { + })), + ), + new OdpEvent( + 't2', + 'a2', + new Map([['id-key-2', 'id-value-2']]), + new Map(Object.entries({ 'idempotence_id': MOCK_IDEMPOTENCE_ID, 'data_source_type': 'sdk', 'data_source': 'javascript-sdk', - 'data_source_version': '', + 'data_source_version': MOCK_PROCESS_VERSION, 'key-2': 'value2', - }, - }, + })), + ), ]; +const makeEvent = (id: number) => { + const identifiers = new Map(); + identifiers.set('identifier1', 'value1-' + id); + identifiers.set('identifier2', 'value2-' + id); + + const data = new Map(); + data.set('data1', 'data-value1-' + id); + data.set('data2', id); + + return new OdpEvent('test-type-' + id, 'test-action-' + id, identifiers, data); +}; +const pause = (timeoutMilliseconds: number): Promise => { + return new Promise(resolve => setTimeout(resolve, timeoutMilliseconds)); +}; +const abortableRequest = (statusCode: number, body: string) => { + return { + abort: () => { + }, + responsePromise: Promise.resolve({ + statusCode, + body, + headers: {}, + }), + }; +}; describe('OdpEventManager', () => { let mockLogger: LogHandler; - let mockOdpConfig: OdpConfig; + let spiedOdpConfig: OdpConfig; let mockRestApiManager: RestApiManager; beforeAll(() => { mockLogger = mock(); - mockOdpConfig = mock(); + spiedOdpConfig = spy(new OdpConfig(API_KEY, API_HOST, [])); mockRestApiManager = mock(); }); beforeEach(() => { resetCalls(mockLogger); - resetCalls(mockOdpConfig); + + resetCalls(spiedOdpConfig); + resetCalls(mockRestApiManager); + // all sendEvents should succeed ie shouldRetry = false unless specified by test + when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); }); + const managerInstance = () => new OdpEventManager(spiedOdpConfig, instance(mockRestApiManager), instance(mockLogger)); + it('should log and discard events when event manager not running', () => { - const eventManager = new OdpEventManager(instance(mockOdpConfig), instance(mockRestApiManager), instance(mockLogger)); + const eventManager = managerInstance(); eventManager.sendEvent(EVENTS[0]); @@ -95,8 +134,9 @@ describe('OdpEventManager', () => { }); it('should log and discard events when event manager is not ready', () => { - when(mockOdpConfig.isReady()).thenReturn(false); - const eventManager = new OdpEventManager(instance(mockOdpConfig), instance(mockRestApiManager), instance(mockLogger)); + when(spiedOdpConfig.isReady()).thenReturn(false); + const eventManager = managerInstance(); + eventManager['isRunning'] = true; // simulate dispatcher already running eventManager.sendEvent(EVENTS[0]); @@ -104,8 +144,7 @@ describe('OdpEventManager', () => { }); it('should log a max queue hit and discard ', () => { - when(mockOdpConfig.isReady()).thenReturn(true); - const eventManager = new OdpEventManager(instance(mockOdpConfig), instance(mockRestApiManager), instance(mockLogger), 1, 1, 1); + const eventManager = new OdpEventManager(spiedOdpConfig, instance(mockRestApiManager), instance(mockLogger), 1, 1, 1); eventManager['isRunning'] = true; // simulate dispatcher already running eventManager['eventQueue'].push(EVENTS[0]); // simulate queue already having 1 @@ -114,31 +153,117 @@ describe('OdpEventManager', () => { verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = 1.')).once(); }); - it('should dispatch events in correct number of batches', () => { + it('should add additional information to each event', () => { + const eventManager = managerInstance(); + const processedEventData = PROCESSED_EVENTS[0].data; + + const eventData = eventManager['augmentCommonData'](EVENTS[0].data); + expect((eventData.get('idempotence_id') as string).length).toEqual((processedEventData.get('idempotence_id') as string).length); + expect(eventData.get('data_source_type')).toEqual(processedEventData.get('data_source_type')); + expect(eventData.get('data_source')).toEqual(processedEventData.get('data_source')); + expect(eventData.get('data_source_version')).not.toBeNull(); + expect(eventData.get('key-1')).toEqual(processedEventData.get('key-1')); + expect(eventData.get('key-2')).toEqual(processedEventData.get('key-2')); + expect(eventData.get('key-3')).toEqual(processedEventData.get('key-3')); + expect(eventData.get('key-4')).toEqual(processedEventData.get('key-4')); }); - it('should dispatch events with correct payload', () => { + it('should dispatch events in correct number of batches', async () => { + const eventManager = managerInstance(); + eventManager.start(); + for (let i = 0; i < 25; i += 1) { + eventManager.sendEvent(makeEvent(i)); + } + await pause(1500); + eventManager.stop(); + + verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).thrice(); }); - it('should dispatch events with correct flush interval', () => { + it('should dispatch events with correct payload', async () => { + const eventManager = new OdpEventManager(spiedOdpConfig, instance(mockRestApiManager), instance(mockLogger), 1); + const processedEvent = PROCESSED_EVENTS[0]; + + eventManager.start(); + eventManager.sendEvent(EVENTS[0]); + await pause(1500); + verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).once(); + const [apiKey, apiHost, events] = capture(mockRestApiManager.sendEvents).last(); + expect(apiKey).toEqual(API_KEY); + expect(apiHost).toEqual(API_HOST); + expect(events.length).toEqual(2); + const expectedEvent = events[0]; + expect(expectedEvent.identifiers.size).toEqual(processedEvent.identifiers.size); + expect(expectedEvent.data.size).toEqual(processedEvent.data.size); }); - it('should retry failed events', () => { + it('should retry failed events', async () => { + // all events should fail ie shouldRetry = true + when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(true); + const eventManager = managerInstance(); + + eventManager.start(); + for (let i = 0; i < 25; i += 1) { + eventManager.sendEvent(makeEvent(i)); + } + await pause(500); + verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).times(6); + + await pause(1500); + verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).times(9); }); - it('should flush all scheduled events before stopping', () => { + it('should flush all scheduled events before stopping', async () => { + const eventManager = managerInstance(); + + eventManager.start(); + for (let i = 0; i < 25; i += 1) { + eventManager.sendEvent(makeEvent(i)); + } + eventManager.stop(); + await pause(1500); + verify(mockLogger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.')); }); - it('should prepare correct payload for identify user', () => { + it('should prepare correct payload for identify user', async () => { + const mockRequestHandler: RequestHandler = mock(); + when(mockRequestHandler.makeRequest(anything(), anything(), anything(), anything())).thenReturn(abortableRequest(200, '')); + const spiedRestApiManager = spy(new RestApiManager(instance(mockRequestHandler), instance(mockLogger))); + const eventManager = new OdpEventManager(spiedOdpConfig, spiedRestApiManager, instance(mockLogger), 1); + const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; + const fsUserId = 'test-fs-user-id'; + eventManager.start(); + eventManager.identifyUser(vuid, fsUserId); + await pause(1500); + + verify(spiedRestApiManager.sendEvents(anything(), anything(), anything())).thrice(); + const [requestUrl, headers, method, data] = capture(mockRequestHandler.makeRequest).last(); + expect(requestUrl).toEqual(API_HOST); + expect(headers).toContain('Content-Type'); + expect(headers).toContain('x-api-key'); + expect(method).toEqual('POST'); + expect((data as string).includes(vuid)).toBe(true); + expect((data as string).includes(fsUserId)).toBe(true); }); it('should apply updated ODP configuration when available', () => { + const eventManager = managerInstance(); + const apiKey = 'testing-api-key'; + const apiHost = 'https://some.other.example.com'; + const segmentsToCheck = ['empty-cart', '1-item-cart']; + const differentOdpConfig = new OdpConfig(apiKey, apiHost, segmentsToCheck); + + eventManager.updateSettings(differentOdpConfig); + expect(eventManager['odpConfig'].apiKey).toEqual(apiKey); + expect(eventManager['odpConfig'].apiHost).toEqual(apiHost); + expect(eventManager['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[0]); + expect(eventManager['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[1]); }); }); From dfda1cde9971b7a80948268e020a47f8f1dee2df Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 27 Sep 2022 17:57:03 -0400 Subject: [PATCH 12/32] WIP observer patternizification --- .../lib/plugins/odp/odp_event_dispatcher.ts | 141 +++++++++++++++++ .../lib/plugins/odp/odp_event_manager.ts | 142 +++++------------- .../tests/odpEventManager.spec.ts | 5 +- 3 files changed, 185 insertions(+), 103 deletions(-) create mode 100644 packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts new file mode 100644 index 000000000..1e5dba8dd --- /dev/null +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -0,0 +1,141 @@ +/** + * Copyright 2022, Optimizely + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { OdpEvent } from './odp_event'; +import { LogHandler } from '../../modules/logging'; + +const MAX_RETRIES = 3; +const EVENT_URL_PATH = '/v3/events'; +const DEFAULT_BATCH_SIZE = 10; +const DEFAULT_QUEUE_SIZE = 10000; +const DEFAULT_FLUSH_INTERVAL = 1000; + +export enum STATE { + STOPPED, + RUNNING, + FLUSHING, +} + +/** + * The Observer interface declares the update method, used by subjects. + */ +export interface Observer { + start(): void; + + enqueue(events: OdpEvent): void; + + flush(): void; + + stop(): void; +} + +export class OdpEventDispatcher implements Observer { + public state: STATE = STATE.STOPPED; + private currentBatch = new Array(); + private nextFlushTime: number = Date.now(); + private intervalId: typeof setInterval; + + private readonly logger: LogHandler; + private readonly queueSize: number; + private readonly batchSize: number; + private readonly flushInterval: number; + + public constructor(logger: LogHandler, + batchSize?: number, + queueSize?: number, + flushInterval?: number) { + this.logger = logger; + + this.batchSize = batchSize && batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE; + this.queueSize = queueSize && queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE; + this.flushInterval = flushInterval && flushInterval > 0 ? flushInterval : DEFAULT_FLUSH_INTERVAL; + + this.intervalId = setInterval(() => { + }, 1); + } + + public start(): void { + this.state = STATE.RUNNING; + } + + public async enqueue(event: OdpEvent): Promise { + this.currentBatch.push(event); + + if (this.currentBatch.length >= this.batchSize) { + clearInterval(this.intervalId); + await this.flush(); + this.intervalId = setInterval(this.flush, this.flushInterval); + } + } + + public async flush(): Promise { + this.state = STATE.FLUSHING; + // if (this.eventManager.odpConfig.isReady()) { + // const payload = this.currentBatch; + // const endpoint = this.eventManager.odpConfig.apiHost + EVENT_URL_PATH; + // let shouldRetry: boolean; + // let numAttempts = 0; + // do { + // shouldRetry = await this.eventManager.apiManager.sendEvents(this.eventManager.odpConfig.apiKey, endpoint, payload); + // numAttempts += 1; + // } while (numAttempts < MAX_RETRIES && shouldRetry); + // } else { + // this.eventManager.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch'); + // } + this.currentBatch = new Array(); + this.state = STATE.RUNNING; + } + + + private async dispatch(): Promise { + + } + + public async stop(): Promise { + this.state = STATE.FLUSHING; + await this.flush(); + this.state = STATE.STOPPED; + } + + // public async run(): Promise { + // try { + // if (this.currentBatch.length > 0) { + // const remainingTimeout = this.nextFlushTime - Date.now(); + // await this.pause(remainingTimeout); + // } + // + // if (this.currentBatch.length === 0) { + // this.nextFlushTime = Date.now() + this.eventManager.flushInterval; + // } + // + // this.currentBatch.push(nextEvent); + // + // if (this.currentBatch.length >= this.eventManager.batchSize) { + // await this.flush(); + // } + // } catch (err) { + // this.eventManager.logger.log(LogLevel.ERROR, err as string); + // } + // } + // + // this.eventManager.logger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.'); + // this.eventManager.isRunning = false; + // } + + // private pause(timeoutMilliseconds: number): Promise { + // return new Promise(resolve => setTimeout(resolve, timeoutMilliseconds)); + // } +} diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 52ef4f7b8..65f2cf283 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -20,12 +20,7 @@ import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; import { RestApiManager } from './rest_api_manager'; - -const DEFAULT_BATCH_SIZE = 10; -const DEFAULT_QUEUE_SIZE = 10000; -const DEFAULT_FLUSH_INTERVAL = 1000; -const MAX_RETRIES = 3; -const EVENT_URL_PATH = '/v3/events'; +import { Observer } from './odp_event_dispatcher'; export interface IOdpEventManager { start(): void; @@ -34,49 +29,43 @@ export interface IOdpEventManager { identifyUser(vuid: string, userId: string): void; - sendEvents(events: [OdpEvent]): void; + sendEvents(events: OdpEvent[]): void; sendEvent(event: OdpEvent): void; stop(): void; } +export interface Subject { + attach(observer: Observer): void; + + detach(observer: Observer): void; + + enqueue(event: OdpEvent[]): void; + + flush(): void; +} + /** * Manager for persisting events to the Optimizely Data Platform */ -export class OdpEventManager implements IOdpEventManager { - private readonly apiManager: RestApiManager; - private readonly logger: LogHandler; - private readonly queueSize: number; - private readonly batchSize: number; - private readonly flushInterval: number; - - private isRunning = false; - +export class OdpEventManager implements IOdpEventManager, Subject { + private observers: Observer[] = []; + public isRunning = false; private odpConfig: OdpConfig; - private eventQueue: Array; - private eventDispatcher: any; + private readonly apiManager: RestApiManager; + private readonly logger: LogHandler; - public constructor(odpConfig: OdpConfig, apiManager: RestApiManager, logger: LogHandler, - batchSize?: number, - queueSize?: number, - flushInterval?: number) { + public constructor(odpConfig: OdpConfig, apiManager: RestApiManager, logger: LogHandler) { this.odpConfig = odpConfig; this.apiManager = apiManager; this.logger = logger; - - this.batchSize = batchSize && batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE; - this.queueSize = queueSize && queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE; - this.flushInterval = flushInterval && flushInterval > 0 ? flushInterval : DEFAULT_FLUSH_INTERVAL; - - this.eventDispatcher = new this.OdpEventDispatcher(this); - this.eventQueue = new Array(); } public start(): void { this.isRunning = true; - this.eventDispatcher.run(); + this.observers.forEach(observer => observer.start()); } public updateSettings(odpConfig: OdpConfig): void { @@ -142,87 +131,36 @@ export class OdpEventManager implements IOdpEventManager { return; } - if (!this.eventQueue.push(event)) { - this.logger.log(LogLevel.ERROR, 'Failed to Process ODP Event. Event Queue is not accepting any more events.'); - } + this.observers.forEach(observer => observer.enqueue(event)); } public stop(): void { - this.logger.log(LogLevel.DEBUG, 'Sending stop signal to ODP Event Dispatcher'); - this.eventDispatcher.signalStop(); + this.isRunning = false; + this.observers.forEach(observer => observer.stop()); } - private OdpEventDispatcher = class { - private shouldStop = false; - private currentBatch = new Array(); - private nextFlushTime: number = Date.now(); - private readonly eventManager: OdpEventManager; - - public constructor(eventManager: OdpEventManager) { - this.eventManager = eventManager; + public attach(observer: Observer): void { + const isExist = this.observers.includes(observer); + if (isExist) { + return console.log('Observer already attached.'); } - public async run(): Promise { - while (!this.shouldStop) { - try { - console.log('batch length', this.currentBatch.length); - if (this.currentBatch.length > 0) { - const remainingTimeout = this.nextFlushTime - Date.now(); - console.log('waiting'); - await this.pause(remainingTimeout); - console.log('waited'); - } - - const [nextEvent, ...remainingEventsInQueue] = this.eventManager.eventQueue; - this.eventManager.eventQueue = remainingEventsInQueue; - - if (!nextEvent) { - if (this.currentBatch.length > 0) { - await this.flush(); - } - continue; - } - - if (this.currentBatch.length === 0) { - this.nextFlushTime = Date.now() + this.eventManager.flushInterval; - } - - this.currentBatch.push(nextEvent); - - if (this.currentBatch.length >= this.eventManager.batchSize) { - await this.flush(); - } - } catch (err) { - this.eventManager.logger.log(LogLevel.ERROR, err as string); - } - } - - this.eventManager.logger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.'); - this.eventManager.isRunning = false; - } + console.log('Attached an observer.'); + this.observers.push(observer); + } - private pause(timeoutMilliseconds: number): Promise { - return new Promise(resolve => setTimeout(resolve, timeoutMilliseconds)); + public detach(observer: Observer): void { + const observerIndex = this.observers.indexOf(observer); + if (observerIndex === -1) { + return console.log('Observer does not exist.'); } - private async flush(): Promise { - if (this.eventManager.odpConfig.isReady()) { - const payload = this.currentBatch; - const endpoint = this.eventManager.odpConfig.apiHost + EVENT_URL_PATH; - let shouldRetry: boolean; - let numAttempts = 0; - do { - shouldRetry = await this.eventManager.apiManager.sendEvents(this.eventManager.odpConfig.apiKey, endpoint, payload); - numAttempts += 1; - } while (numAttempts < MAX_RETRIES && shouldRetry); - } else { - this.eventManager.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch'); - } - this.currentBatch = new Array(); - } + this.observers.splice(observerIndex, 1); + console.log('Detached an observer.'); + } - public signalStop(): void { - this.shouldStop = true; - } - }; + public enqueue(events: OdpEvent[]): void { + this.observers.forEach(observer => observer.enqueue(events)); + } } + diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index f6bea4190..f600883db 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -21,6 +21,7 @@ import { RestApiManager } from '../lib/plugins/odp/rest_api_manager'; import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; import { RequestHandler } from '../lib/utils/http_request_handler/http'; +import { OdpEventDispatcher } from '../lib/plugins/odp/odp_event_dispatcher'; const API_KEY = 'test-api-key'; const API_HOST = 'https://odp.example.com'; @@ -184,11 +185,13 @@ describe('OdpEventManager', () => { it('should dispatch events with correct payload', async () => { const eventManager = new OdpEventManager(spiedOdpConfig, instance(mockRestApiManager), instance(mockLogger), 1); + const eventDispatcher = new OdpEventDispatcher(); + eventManager.attach(eventDispatcher); const processedEvent = PROCESSED_EVENTS[0]; eventManager.start(); eventManager.sendEvent(EVENTS[0]); - await pause(1500); + //await pause(1500); verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).once(); const [apiKey, apiHost, events] = capture(mockRestApiManager.sendEvents).last(); From e2808d5b7acae0c9f223d97de56f0342cc1d4979 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Thu, 29 Sep 2022 17:25:55 -0400 Subject: [PATCH 13/32] Event manager and dispatcher with unit tests --- .../lib/plugins/odp/odp_event_dispatcher.ts | 162 ++++++++++-------- .../lib/plugins/odp/odp_event_manager.ts | 91 ++-------- .../lib/plugins/odp/rest_api_manager.ts | 10 +- .../tests/odpEventManager.spec.ts | 132 ++++++++------ 4 files changed, 193 insertions(+), 202 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index 1e5dba8dd..cfa437a46 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -15,10 +15,11 @@ */ import { OdpEvent } from './odp_event'; -import { LogHandler } from '../../modules/logging'; +import { LogHandler, LogLevel } from '../../modules/logging'; +import { OdpConfig } from './odp_config'; +import { RestApiManager } from './rest_api_manager'; const MAX_RETRIES = 3; -const EVENT_URL_PATH = '/v3/events'; const DEFAULT_BATCH_SIZE = 10; const DEFAULT_QUEUE_SIZE = 10000; const DEFAULT_FLUSH_INTERVAL = 1000; @@ -26,116 +27,131 @@ const DEFAULT_FLUSH_INTERVAL = 1000; export enum STATE { STOPPED, RUNNING, - FLUSHING, + PROCESSING, } -/** - * The Observer interface declares the update method, used by subjects. - */ -export interface Observer { +export interface IOdpEventDispatcher { start(): void; - enqueue(events: OdpEvent): void; + updateSettings(odpConfig: OdpConfig): void; - flush(): void; + enqueue(event: OdpEvent): void; - stop(): void; + stop(): Promise; } -export class OdpEventDispatcher implements Observer { +export class OdpEventDispatcher implements IOdpEventDispatcher { public state: STATE = STATE.STOPPED; - private currentBatch = new Array(); - private nextFlushTime: number = Date.now(); - private intervalId: typeof setInterval; + private queue = new Array(); + private batch = new Array(); + private intervalId: number | NodeJS.Timer; + private odpConfig: OdpConfig; + private shouldStopAndDrain = false; + + private readonly apiManager: RestApiManager; private readonly logger: LogHandler; private readonly queueSize: number; private readonly batchSize: number; private readonly flushInterval: number; - public constructor(logger: LogHandler, - batchSize?: number, + public constructor(odpConfig: OdpConfig, + apiManager: RestApiManager, + logger: LogHandler, queueSize?: number, + batchSize?: number, flushInterval?: number) { + this.odpConfig = odpConfig; + this.apiManager = apiManager; this.logger = logger; - this.batchSize = batchSize && batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE; this.queueSize = queueSize && queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE; + this.batchSize = batchSize && batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE; this.flushInterval = flushInterval && flushInterval > 0 ? flushInterval : DEFAULT_FLUSH_INTERVAL; + this.state = STATE.STOPPED; + // initialize this way due to different types based on execution context this.intervalId = setInterval(() => { - }, 1); + }); } public start(): void { this.state = STATE.RUNNING; + (async () => await this.processQueue())(); } - public async enqueue(event: OdpEvent): Promise { - this.currentBatch.push(event); + public updateSettings(odpConfig: OdpConfig): void { + this.odpConfig = odpConfig; + } - if (this.currentBatch.length >= this.batchSize) { - clearInterval(this.intervalId); - await this.flush(); - this.intervalId = setInterval(this.flush, this.flushInterval); + public enqueue(event: OdpEvent): void { + if (this.state != STATE.RUNNING) { + this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.'); + return; } - } - public async flush(): Promise { - this.state = STATE.FLUSHING; - // if (this.eventManager.odpConfig.isReady()) { - // const payload = this.currentBatch; - // const endpoint = this.eventManager.odpConfig.apiHost + EVENT_URL_PATH; - // let shouldRetry: boolean; - // let numAttempts = 0; - // do { - // shouldRetry = await this.eventManager.apiManager.sendEvents(this.eventManager.odpConfig.apiKey, endpoint, payload); - // numAttempts += 1; - // } while (numAttempts < MAX_RETRIES && shouldRetry); - // } else { - // this.eventManager.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch'); - // } - this.currentBatch = new Array(); - this.state = STATE.RUNNING; + if (!this.odpConfig.isReady()) { + this.logger.log(LogLevel.DEBUG, 'Unable to Process ODP Event. ODPConfig is not ready.'); + return; + } + + if (this.queue.length >= this.queueSize) { + this.logger.log(LogLevel.WARNING, `Failed to Process ODP Event. Event Queue full. queueSize = ${this.queueSize}.`); + return; + } + + this.queue.push(event); } + private async processQueue(): Promise { + if (this.state !== STATE.RUNNING) { + return; + } + + clearInterval(this.intervalId); + + if (this.odpConfig.isReady() && this.queue.length > 0) { + this.state = STATE.PROCESSING; + + for (let count = 0; count < this.batchSize; count += 1) { + const event = this.queue.shift(); + if (event) { + this.batch.push(event); + } else { + break; + } + } + + if (this.batch.length > 0) { + let shouldRetry: boolean; + let numAttempts = 0; + do { + shouldRetry = await this.apiManager.sendEvents(this.odpConfig.apiKey, this.odpConfig.apiHost, this.batch); + numAttempts += 1; + } while (shouldRetry && numAttempts < MAX_RETRIES); + } + + this.batch = new Array(); + + if (this.shouldStopAndDrain && this.queue.length > 0) { + this.logger.log(LogLevel.DEBUG, 'EventDispatcher draining queue without flush interval.'); + await this.processQueue(); + } + } else { + this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch.'); + } - private async dispatch(): Promise { + if (!this.shouldStopAndDrain) { + this.intervalId = setInterval(() => this.processQueue(), this.flushInterval); + } + this.state = STATE.RUNNING; } public async stop(): Promise { - this.state = STATE.FLUSHING; - await this.flush(); + this.logger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.'); + this.shouldStopAndDrain = true; + await this.processQueue(); this.state = STATE.STOPPED; } - - // public async run(): Promise { - // try { - // if (this.currentBatch.length > 0) { - // const remainingTimeout = this.nextFlushTime - Date.now(); - // await this.pause(remainingTimeout); - // } - // - // if (this.currentBatch.length === 0) { - // this.nextFlushTime = Date.now() + this.eventManager.flushInterval; - // } - // - // this.currentBatch.push(nextEvent); - // - // if (this.currentBatch.length >= this.eventManager.batchSize) { - // await this.flush(); - // } - // } catch (err) { - // this.eventManager.logger.log(LogLevel.ERROR, err as string); - // } - // } - // - // this.eventManager.logger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.'); - // this.eventManager.isRunning = false; - // } - - // private pause(timeoutMilliseconds: number): Promise { - // return new Promise(resolve => setTimeout(resolve, timeoutMilliseconds)); - // } } diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 65f2cf283..12cd8a14a 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -14,62 +14,44 @@ * limitations under the License. */ -import { LogHandler, LogLevel } from '../../modules/logging'; +import { LogHandler } from '../../modules/logging'; import { OdpEvent } from './odp_event'; import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; -import { RestApiManager } from './rest_api_manager'; -import { Observer } from './odp_event_dispatcher'; +import { OdpEventDispatcher } from './odp_event_dispatcher'; export interface IOdpEventManager { start(): void; - updateSettings(odpConfig: OdpConfig): void; - identifyUser(vuid: string, userId: string): void; + updateSettings(odpConfig: OdpConfig): void; + sendEvents(events: OdpEvent[]): void; sendEvent(event: OdpEvent): void; - stop(): void; -} - -export interface Subject { - attach(observer: Observer): void; - - detach(observer: Observer): void; - - enqueue(event: OdpEvent[]): void; - - flush(): void; + signalStop(): void; } /** * Manager for persisting events to the Optimizely Data Platform */ -export class OdpEventManager implements IOdpEventManager, Subject { - private observers: Observer[] = []; +export class OdpEventManager implements IOdpEventManager { public isRunning = false; - private odpConfig: OdpConfig; - private readonly apiManager: RestApiManager; + private readonly eventDispatcher: OdpEventDispatcher; private readonly logger: LogHandler; - public constructor(odpConfig: OdpConfig, apiManager: RestApiManager, logger: LogHandler) { - this.odpConfig = odpConfig; - this.apiManager = apiManager; + public constructor(eventDispatcher: OdpEventDispatcher, logger: LogHandler) { + this.eventDispatcher = eventDispatcher; this.logger = logger; } public start(): void { this.isRunning = true; - this.observers.forEach(observer => observer.start()); - } - - public updateSettings(odpConfig: OdpConfig): void { - this.odpConfig = odpConfig; + this.eventDispatcher.start(); } public identifyUser(vuid: string, userId: string): void { @@ -83,13 +65,17 @@ export class OdpEventManager implements IOdpEventManager, Subject { this.sendEvent(event); } + public updateSettings(odpConfig: OdpConfig): void { + this.eventDispatcher.updateSettings(odpConfig); + } + public sendEvents(events: OdpEvent[]): void { events.forEach(event => this.sendEvent(event)); } public sendEvent(event: OdpEvent): void { event.data = this.augmentCommonData(event.data); - this.processEvent(event); + (async () => await this.eventDispatcher.enqueue(event))(); } private augmentCommonData(sourceData: Map): Map { @@ -115,52 +101,9 @@ export class OdpEventManager implements IOdpEventManager, Subject { return data; } - private processEvent(event: OdpEvent): void { - if (!this.isRunning) { - this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.'); - return; - } - - if (!this.odpConfig.isReady()) { - this.logger.log(LogLevel.DEBUG, 'Unable to Process ODP Event. ODPConfig is not ready.'); - return; - } - - if (this.eventQueue.length >= this.queueSize) { - this.logger.log(LogLevel.WARNING, `Failed to Process ODP Event. Event Queue full. queueSize = ${this.queueSize}.`); - return; - } - - this.observers.forEach(observer => observer.enqueue(event)); - } - - public stop(): void { + public signalStop(): void { + (async () => await this.eventDispatcher.stop())(); this.isRunning = false; - this.observers.forEach(observer => observer.stop()); - } - - public attach(observer: Observer): void { - const isExist = this.observers.includes(observer); - if (isExist) { - return console.log('Observer already attached.'); - } - - console.log('Attached an observer.'); - this.observers.push(observer); - } - - public detach(observer: Observer): void { - const observerIndex = this.observers.indexOf(observer); - if (observerIndex === -1) { - return console.log('Observer does not exist.'); - } - - this.observers.splice(observerIndex, 1); - console.log('Detached an observer.'); - } - - public enqueue(events: OdpEvent[]): void { - this.observers.forEach(observer => observer.enqueue(events)); } } diff --git a/packages/optimizely-sdk/lib/plugins/odp/rest_api_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/rest_api_manager.ts index de872f3cd..8a58de202 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/rest_api_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/rest_api_manager.ts @@ -65,7 +65,7 @@ export class RestApiManager implements IRestApiManager { } const endpoint = `${apiHost}/v3/events`; - const data = JSON.stringify(events); + const data = JSON.stringify(events, this.replacer); const method = 'POST'; const headers = { @@ -97,4 +97,12 @@ export class RestApiManager implements IRestApiManager { return shouldRetry; } + + private replacer(_: unknown, value: unknown) { + if (value instanceof Map) { + return Object.fromEntries(value); + } else { + return value; + } + } } diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index f600883db..e51339e3f 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -16,12 +16,12 @@ import { OdpConfig } from '../lib/plugins/odp/odp_config'; import { OdpEventManager } from '../lib/plugins/odp/odp_event_manager'; -import { anything, capture, instance, mock, resetCalls, spy, verify, when } from 'ts-mockito'; +import { anything, capture, instance, mock, resetCalls, verify, when } from 'ts-mockito'; import { RestApiManager } from '../lib/plugins/odp/rest_api_manager'; import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; import { RequestHandler } from '../lib/utils/http_request_handler/http'; -import { OdpEventDispatcher } from '../lib/plugins/odp/odp_event_dispatcher'; +import { OdpEventDispatcher, STATE } from '../lib/plugins/odp/odp_event_dispatcher'; const API_KEY = 'test-api-key'; const API_HOST = 'https://odp.example.com'; @@ -105,39 +105,39 @@ const abortableRequest = (statusCode: number, body: string) => { describe('OdpEventManager', () => { let mockLogger: LogHandler; - let spiedOdpConfig: OdpConfig; let mockRestApiManager: RestApiManager; + let odpConfig: OdpConfig; beforeAll(() => { mockLogger = mock(); - spiedOdpConfig = spy(new OdpConfig(API_KEY, API_HOST, [])); mockRestApiManager = mock(); + odpConfig = new OdpConfig(API_KEY, API_HOST, []); }); beforeEach(() => { resetCalls(mockLogger); - - resetCalls(spiedOdpConfig); - resetCalls(mockRestApiManager); - // all sendEvents should succeed ie shouldRetry = false unless specified by test - when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); }); - const managerInstance = () => new OdpEventManager(spiedOdpConfig, instance(mockRestApiManager), instance(mockLogger)); - it('should log and discard events when event manager not running', () => { - const eventManager = managerInstance(); + const logger = instance(mockLogger); + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger); + const eventManager = new OdpEventManager(eventDispatcher, logger); + // since we've not called start() then... eventManager.sendEvent(EVENTS[0]); + // ...we should get a notice after trying to send an event verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.')).once(); }); it('should log and discard events when event manager is not ready', () => { - when(spiedOdpConfig.isReady()).thenReturn(false); - const eventManager = managerInstance(); - eventManager['isRunning'] = true; // simulate dispatcher already running + const logger = instance(mockLogger); + const mockOdpConfig = mock(); + when(mockOdpConfig.isReady()).thenReturn(false); + const eventDispatcher = new OdpEventDispatcher(instance(mockOdpConfig), instance(mockRestApiManager), logger); + eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already running + const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.sendEvent(EVENTS[0]); @@ -145,17 +145,23 @@ describe('OdpEventManager', () => { }); it('should log a max queue hit and discard ', () => { - const eventManager = new OdpEventManager(spiedOdpConfig, instance(mockRestApiManager), instance(mockLogger), 1, 1, 1); - eventManager['isRunning'] = true; // simulate dispatcher already running - eventManager['eventQueue'].push(EVENTS[0]); // simulate queue already having 1 + const logger = instance(mockLogger); + const mockOdpConfig = mock(); + when(mockOdpConfig.isReady()).thenReturn(false); + const eventDispatcher = new OdpEventDispatcher(mockOdpConfig, instance(mockRestApiManager), logger, 1); + eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already running + eventDispatcher['queue'].push(EVENTS[0]); + const eventManager = new OdpEventManager(eventDispatcher, logger); - eventManager.sendEvent(EVENTS[1]); // try adding to queue + eventManager.sendEvent(EVENTS[1]); verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = 1.')).once(); }); it('should add additional information to each event', () => { - const eventManager = managerInstance(); + const logger = instance(mockLogger); + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger); + const eventManager = new OdpEventManager(eventDispatcher, logger); const processedEventData = PROCESSED_EVENTS[0].data; const eventData = eventManager['augmentCommonData'](EVENTS[0].data); @@ -171,73 +177,84 @@ describe('OdpEventManager', () => { }); it('should dispatch events in correct number of batches', async () => { - const eventManager = managerInstance(); + const logger = instance(mockLogger); + when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 10, 100); + const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); for (let i = 0; i < 25; i += 1) { eventManager.sendEvent(makeEvent(i)); } await pause(1500); - eventManager.stop(); + // 3 batches: batch #1 with 10, batch #2 with 10, and batch #3 with 5 = 25 events verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).thrice(); }); it('should dispatch events with correct payload', async () => { - const eventManager = new OdpEventManager(spiedOdpConfig, instance(mockRestApiManager), instance(mockLogger), 1); - const eventDispatcher = new OdpEventDispatcher(); - eventManager.attach(eventDispatcher); - const processedEvent = PROCESSED_EVENTS[0]; + const logger = instance(mockLogger); + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 10, 100); + const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); - eventManager.sendEvent(EVENTS[0]); - //await pause(1500); + eventManager.sendEvents(EVENTS); + await pause(1500); verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).once(); const [apiKey, apiHost, events] = capture(mockRestApiManager.sendEvents).last(); expect(apiKey).toEqual(API_KEY); expect(apiHost).toEqual(API_HOST); expect(events.length).toEqual(2); - const expectedEvent = events[0]; - expect(expectedEvent.identifiers.size).toEqual(processedEvent.identifiers.size); - expect(expectedEvent.data.size).toEqual(processedEvent.data.size); + expect(events[0].identifiers.size).toEqual(PROCESSED_EVENTS[0].identifiers.size); + expect(events[0].data.size).toEqual(PROCESSED_EVENTS[0].data.size); + expect(events[1].identifiers.size).toEqual(PROCESSED_EVENTS[1].identifiers.size); + expect(events[1].data.size).toEqual(PROCESSED_EVENTS[1].data.size); }); it('should retry failed events', async () => { // all events should fail ie shouldRetry = true when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(true); - const eventManager = managerInstance(); + const logger = instance(mockLogger); + // batch size of 2 + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 2, 100); + const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); - for (let i = 0; i < 25; i += 1) { + // send 4 events + for (let i = 0; i < 4; i += 1) { eventManager.sendEvent(makeEvent(i)); } - await pause(500); + await pause(1500); + // retry 3x for 2 batches or 6 calls to attempt to process the 4 events verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).times(6); - - await pause(1500); - verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).times(9); }); it('should flush all scheduled events before stopping', async () => { - const eventManager = managerInstance(); + const logger = instance(mockLogger); + when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 10, 100); + const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); for (let i = 0; i < 25; i += 1) { eventManager.sendEvent(makeEvent(i)); } - eventManager.stop(); await pause(1500); - - verify(mockLogger.log(LogLevel.DEBUG, 'Exiting ODP Event Dispatcher Thread.')); + // eventManager.signalStop(); + // TODO: These can't be succeeding since signalStop() not called above + verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.')); + verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher draining queue without flush interval.')); }); it('should prepare correct payload for identify user', async () => { const mockRequestHandler: RequestHandler = mock(); when(mockRequestHandler.makeRequest(anything(), anything(), anything(), anything())).thenReturn(abortableRequest(200, '')); - const spiedRestApiManager = spy(new RestApiManager(instance(mockRequestHandler), instance(mockLogger))); - const eventManager = new OdpEventManager(spiedOdpConfig, spiedRestApiManager, instance(mockLogger), 1); + const logger = instance(mockLogger); + const restApiManager = new RestApiManager(instance(mockRequestHandler), logger); + const eventDispatcher = new OdpEventDispatcher(odpConfig, restApiManager, logger, 100, 10, 100); + const eventManager = new OdpEventManager(eventDispatcher, logger); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; const fsUserId = 'test-fs-user-id'; @@ -245,18 +262,25 @@ describe('OdpEventManager', () => { eventManager.identifyUser(vuid, fsUserId); await pause(1500); - verify(spiedRestApiManager.sendEvents(anything(), anything(), anything())).thrice(); const [requestUrl, headers, method, data] = capture(mockRequestHandler.makeRequest).last(); - expect(requestUrl).toEqual(API_HOST); - expect(headers).toContain('Content-Type'); - expect(headers).toContain('x-api-key'); + expect(requestUrl).toEqual(`${API_HOST}/v3/events`); + expect(headers['Content-Type']).toEqual('application/json'); + expect(headers['x-api-key']).toEqual('test-api-key'); expect(method).toEqual('POST'); - expect((data as string).includes(vuid)).toBe(true); - expect((data as string).includes(fsUserId)).toBe(true); + const events = JSON.parse(data as string); + const event = events[0]; + expect(event.type).toEqual('fullstack'); + expect(event.action).toEqual('client_initialized'); + expect(event.identifiers).toEqual({ 'vuid': vuid, 'fs_user_id': fsUserId }); + expect(event.data.idempotence_id.length).toBe(36); // uuid length + expect(event.data.data_source_type).toEqual('sdk'); + expect(event.data.data_source).toEqual('javascript-sdk'); + expect(event.data.data_source_version).not.toBeNull(); }); it('should apply updated ODP configuration when available', () => { - const eventManager = managerInstance(); + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), instance(mockLogger)); + const eventManager = new OdpEventManager(eventDispatcher, mockLogger); const apiKey = 'testing-api-key'; const apiHost = 'https://some.other.example.com'; const segmentsToCheck = ['empty-cart', '1-item-cart']; @@ -264,9 +288,9 @@ describe('OdpEventManager', () => { eventManager.updateSettings(differentOdpConfig); - expect(eventManager['odpConfig'].apiKey).toEqual(apiKey); - expect(eventManager['odpConfig'].apiHost).toEqual(apiHost); - expect(eventManager['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[0]); - expect(eventManager['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[1]); + expect(eventManager['eventDispatcher']['odpConfig'].apiKey).toEqual(apiKey); + expect(eventManager['eventDispatcher']['odpConfig'].apiHost).toEqual(apiHost); + expect(eventManager['eventDispatcher']['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[0]); + expect(eventManager['eventDispatcher']['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[1]); }); }); From b2b8be0acc8534bc7d4209baf087a02d0b351c8d Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Fri, 30 Sep 2022 11:06:42 -0400 Subject: [PATCH 14/32] Fixes to for unit tests --- .../lib/plugins/odp/odp_event_dispatcher.ts | 42 ++++++++++--------- .../tests/odpEventManager.spec.ts | 27 +++++++----- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index cfa437a46..69c7b855d 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -104,34 +104,37 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { } private async processQueue(): Promise { - if (this.state !== STATE.RUNNING) { + if (this.state !== STATE.RUNNING && !this.shouldStopAndDrain) { return; } clearInterval(this.intervalId); - if (this.odpConfig.isReady() && this.queue.length > 0) { - this.state = STATE.PROCESSING; + if (this.odpConfig.isReady()) { + if (this.queue.length > 0) { - for (let count = 0; count < this.batchSize; count += 1) { - const event = this.queue.shift(); - if (event) { - this.batch.push(event); - } else { - break; + this.state = STATE.PROCESSING; + + for (let count = 0; count < this.batchSize; count += 1) { + const event = this.queue.shift(); + if (event) { + this.batch.push(event); + } else { + break; + } } - } - if (this.batch.length > 0) { - let shouldRetry: boolean; - let numAttempts = 0; - do { - shouldRetry = await this.apiManager.sendEvents(this.odpConfig.apiKey, this.odpConfig.apiHost, this.batch); - numAttempts += 1; - } while (shouldRetry && numAttempts < MAX_RETRIES); - } + if (this.batch.length > 0) { + let shouldRetry: boolean; + let numAttempts = 0; + do { + shouldRetry = await this.apiManager.sendEvents(this.odpConfig.apiKey, this.odpConfig.apiHost, this.batch); + numAttempts += 1; + } while (shouldRetry && numAttempts < MAX_RETRIES); + } - this.batch = new Array(); + this.batch = new Array(); + } if (this.shouldStopAndDrain && this.queue.length > 0) { this.logger.log(LogLevel.DEBUG, 'EventDispatcher draining queue without flush interval.'); @@ -153,5 +156,6 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.shouldStopAndDrain = true; await this.processQueue(); this.state = STATE.STOPPED; + this.logger.log(LogLevel.DEBUG, `EventDispatcher stopped. Queue Count: ${this.queue.length}`); } } diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index e51339e3f..291925a5d 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -131,12 +131,12 @@ describe('OdpEventManager', () => { verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.')).once(); }); - it('should log and discard events when event manager is not ready', () => { + it('should log and discard events when event manager\'s config is not ready', () => { const logger = instance(mockLogger); const mockOdpConfig = mock(); when(mockOdpConfig.isReady()).thenReturn(false); const eventDispatcher = new OdpEventDispatcher(instance(mockOdpConfig), instance(mockRestApiManager), logger); - eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already running + eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already in running state const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.sendEvent(EVENTS[0]); @@ -148,11 +148,13 @@ describe('OdpEventManager', () => { const logger = instance(mockLogger); const mockOdpConfig = mock(); when(mockOdpConfig.isReady()).thenReturn(false); + // set queue to maximum of 1 const eventDispatcher = new OdpEventDispatcher(mockOdpConfig, instance(mockRestApiManager), logger, 1); - eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already running - eventDispatcher['queue'].push(EVENTS[0]); + eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher running + eventDispatcher['queue'].push(EVENTS[0]); // simulate event already in queue const eventManager = new OdpEventManager(eventDispatcher, logger); + // try adding the second event eventManager.sendEvent(EVENTS[1]); verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = 1.')).once(); @@ -227,25 +229,30 @@ describe('OdpEventManager', () => { } await pause(1500); - // retry 3x for 2 batches or 6 calls to attempt to process the 4 events + // retry 3x (default) for 2 batches or 6 calls to attempt to process verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).times(6); }); it('should flush all scheduled events before stopping', async () => { const logger = instance(mockLogger); when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 10, 100); + // batches of 2 with... + const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 2, 100); const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); + // ...25 events should... for (let i = 0; i < 25; i += 1) { eventManager.sendEvent(makeEvent(i)); } + await pause(300); + eventManager.signalStop(); await pause(1500); - // eventManager.signalStop(); - // TODO: These can't be succeeding since signalStop() not called above - verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.')); - verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher draining queue without flush interval.')); + + verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.')).once(); + // ...never exceed 14 + verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher draining queue without flush interval.')).atMost(14); + verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stopped. Queue Count: 0')).once(); }); it('should prepare correct payload for identify user', async () => { From 74032bfeca9eaa38c1ffba2325d014c6e0126cf6 Mon Sep 17 00:00:00 2001 From: Mike Chu <104384559+mikechu-optimizely@users.noreply.github.com> Date: Mon, 3 Oct 2022 17:23:56 -0400 Subject: [PATCH 15/32] Update packages/optimizely-sdk/lib/plugins/odp/odp_config.ts Co-authored-by: Zeeshan Ashraf <35262377+zashraf1985@users.noreply.github.com> --- packages/optimizely-sdk/lib/plugins/odp/odp_config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts index 1345a767b..da5e0513a 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts @@ -97,6 +97,6 @@ export class OdpConfig { * Determines if ODP configuration has the minimum amount of information */ public isReady(): boolean { - return this._apiKey !== null && this._apiKey !== '' && this._apiHost !== null && this._apiHost !== ''; + return !!this._apiKey && !!this._apiHost; } } From 7e0a88195d9fb826dcfc08515aadec48ff54d862 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Mon, 3 Oct 2022 17:27:01 -0400 Subject: [PATCH 16/32] Review code changes --- .../optimizely-sdk/lib/plugins/odp/odp_config.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts index da5e0513a..dc5fcdff6 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts @@ -68,8 +68,18 @@ export class OdpConfig { constructor(apiKey: string, apiHost: string, segmentsToCheck?: string[]) { this._apiKey = apiKey; this._apiHost = apiHost; - this._segmentsToCheck = segmentsToCheck ?? [] as string[]; - this._odpServiceIntegrated = this._apiKey && this._apiHost ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.UNDETERMINED; + this._segmentsToCheck = segmentsToCheck ?? []; + this._odpServiceIntegrated = OdpConfig.determineOdpServiceIntegration(this._apiKey, this.apiHost); + } + + /** + * Determine the value of Service Integration enum + * @param apiKey ODP API Key + * @param apiHost Server host of the API + * @private + */ + private static determineOdpServiceIntegration(apiKey: string, apiHost: string): ODP_CONFIG_STATE { + return apiKey && apiHost ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.NOT_INTEGRATED; } /** @@ -80,7 +90,7 @@ export class OdpConfig { * @returns true if configuration was updated successfully */ public update(apiKey: string, apiHost: string, segmentsToCheck: string[]): boolean { - this._odpServiceIntegrated = apiKey && apiHost ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.NOT_INTEGRATED; + this._odpServiceIntegrated = OdpConfig.determineOdpServiceIntegration(apiKey, apiHost); if (this._apiKey === apiKey && this._apiHost === apiHost && this._segmentsToCheck === segmentsToCheck) { return false; From d87ed4ff6e3c1ab01bade111ef4a4c99f9f404b7 Mon Sep 17 00:00:00 2001 From: Mike Chu <104384559+mikechu-optimizely@users.noreply.github.com> Date: Tue, 4 Oct 2022 08:46:54 -0400 Subject: [PATCH 17/32] Update packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts Co-authored-by: Zeeshan Ashraf <35262377+zashraf1985@users.noreply.github.com> --- .../optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index 69c7b855d..c3f65de1e 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -65,9 +65,9 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.apiManager = apiManager; this.logger = logger; - this.queueSize = queueSize && queueSize > 0 ? queueSize : DEFAULT_QUEUE_SIZE; - this.batchSize = batchSize && batchSize > 0 ? batchSize : DEFAULT_BATCH_SIZE; - this.flushInterval = flushInterval && flushInterval > 0 ? flushInterval : DEFAULT_FLUSH_INTERVAL; + this.queueSize = queueSize || DEFAULT_QUEUE_SIZE; + this.batchSize = batchSize || DEFAULT_BATCH_SIZE; + this.flushInterval = flushInterval || DEFAULT_FLUSH_INTERVAL; this.state = STATE.STOPPED; // initialize this way due to different types based on execution context From 471a9c83ba222b5f510eb5370dd4bce4c6b44688 Mon Sep 17 00:00:00 2001 From: Mike Chu <104384559+mikechu-optimizely@users.noreply.github.com> Date: Tue, 4 Oct 2022 08:48:21 -0400 Subject: [PATCH 18/32] Update packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts Co-authored-by: Jae Kim <45045038+jaeopt@users.noreply.github.com> --- packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index c3f65de1e..9dda1ca36 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -22,7 +22,7 @@ import { RestApiManager } from './rest_api_manager'; const MAX_RETRIES = 3; const DEFAULT_BATCH_SIZE = 10; const DEFAULT_QUEUE_SIZE = 10000; -const DEFAULT_FLUSH_INTERVAL = 1000; +const DEFAULT_FLUSH_INTERVAL_MSECS = 1000; export enum STATE { STOPPED, From a8f7df281e4e871e3de03b395d1fdd00be11d34b Mon Sep 17 00:00:00 2001 From: Mike Chu <104384559+mikechu-optimizely@users.noreply.github.com> Date: Tue, 4 Oct 2022 08:49:04 -0400 Subject: [PATCH 19/32] Update packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts Co-authored-by: Jae Kim <45045038+jaeopt@users.noreply.github.com> --- packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 12cd8a14a..6979eb18f 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -54,7 +54,7 @@ export class OdpEventManager implements IOdpEventManager { this.eventDispatcher.start(); } - public identifyUser(vuid: string, userId: string): void { + public identifyUser(vuid: string?, userId: string): void { const identifiers = new Map(); if (vuid != null) { identifiers.set(ODP_USER_KEY.VUID, vuid); From 0157b90c8d1105d84e2d8a0f55d7c8c6dcc6ab42 Mon Sep 17 00:00:00 2001 From: Mike Chu <104384559+mikechu-optimizely@users.noreply.github.com> Date: Tue, 4 Oct 2022 09:16:56 -0400 Subject: [PATCH 20/32] Update packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts Co-authored-by: Jae Kim <45045038+jaeopt@users.noreply.github.com> --- packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 6979eb18f..94ebc4f6b 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -61,7 +61,7 @@ export class OdpEventManager implements IOdpEventManager { } identifiers.set(ODP_USER_KEY.FS_USER_ID, userId); - const event = new OdpEvent('fullstack', 'client_initialized', identifiers); + const event = new OdpEvent('fullstack', 'identified', identifiers); this.sendEvent(event); } From 98c75d70c24024660c6e452e7890e3668e1ead25 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 4 Oct 2022 09:53:01 -0400 Subject: [PATCH 21/32] Code review changes --- .../lib/plugins/odp/odp_event_dispatcher.ts | 36 ++--- .../lib/plugins/odp/odp_event_manager.ts | 32 ++--- .../tests/odpEventManager.spec.ts | 136 ++++++++++++++---- 3 files changed, 144 insertions(+), 60 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index 9dda1ca36..07bef5b2e 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -21,7 +21,6 @@ import { RestApiManager } from './rest_api_manager'; const MAX_RETRIES = 3; const DEFAULT_BATCH_SIZE = 10; -const DEFAULT_QUEUE_SIZE = 10000; const DEFAULT_FLUSH_INTERVAL_MSECS = 1000; export enum STATE { @@ -45,7 +44,8 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { private queue = new Array(); private batch = new Array(); - private intervalId: number | NodeJS.Timer; + private timeoutId: ReturnType = setTimeout(() => { + }); private odpConfig: OdpConfig; private shouldStopAndDrain = false; @@ -55,24 +55,25 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { private readonly batchSize: number; private readonly flushInterval: number; - public constructor(odpConfig: OdpConfig, - apiManager: RestApiManager, - logger: LogHandler, - queueSize?: number, - batchSize?: number, - flushInterval?: number) { + public constructor( + { odpConfig, apiManager, logger, queueSize, batchSize, flushInterval }: { + odpConfig: OdpConfig, + apiManager: RestApiManager, + logger: LogHandler, + queueSize?: number, + batchSize?: number, + flushInterval?: number + }) { this.odpConfig = odpConfig; this.apiManager = apiManager; this.logger = logger; - this.queueSize = queueSize || DEFAULT_QUEUE_SIZE; + // if `process` exists Node/server-side execution context otherwise Browser + this.queueSize = queueSize || (process ? 10000 : 100); this.batchSize = batchSize || DEFAULT_BATCH_SIZE; - this.flushInterval = flushInterval || DEFAULT_FLUSH_INTERVAL; + this.flushInterval = flushInterval || DEFAULT_FLUSH_INTERVAL_MSECS; this.state = STATE.STOPPED; - // initialize this way due to different types based on execution context - this.intervalId = setInterval(() => { - }); } public start(): void { @@ -108,7 +109,7 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { return; } - clearInterval(this.intervalId); + clearInterval(this.timeoutId); if (this.odpConfig.isReady()) { if (this.queue.length > 0) { @@ -134,6 +135,7 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { } this.batch = new Array(); + this.state = STATE.RUNNING; } if (this.shouldStopAndDrain && this.queue.length > 0) { @@ -141,14 +143,12 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { await this.processQueue(); } } else { - this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready, discarding event batch.'); + this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready.'); } if (!this.shouldStopAndDrain) { - this.intervalId = setInterval(() => this.processQueue(), this.flushInterval); + this.timeoutId = setTimeout(() => this.processQueue(), this.flushInterval); } - - this.state = STATE.RUNNING; } public async stop(): Promise { diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 94ebc4f6b..d6340e34e 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -24,11 +24,11 @@ import { OdpEventDispatcher } from './odp_event_dispatcher'; export interface IOdpEventManager { start(): void; - identifyUser(vuid: string, userId: string): void; + registerVuid(vuid: string): void; - updateSettings(odpConfig: OdpConfig): void; + identifyUser(userId: string, vuid?: string): void; - sendEvents(events: OdpEvent[]): void; + updateSettings(odpConfig: OdpConfig): void; sendEvent(event: OdpEvent): void; @@ -39,8 +39,6 @@ export interface IOdpEventManager { * Manager for persisting events to the Optimizely Data Platform */ export class OdpEventManager implements IOdpEventManager { - public isRunning = false; - private readonly eventDispatcher: OdpEventDispatcher; private readonly logger: LogHandler; @@ -50,13 +48,20 @@ export class OdpEventManager implements IOdpEventManager { } public start(): void { - this.isRunning = true; this.eventDispatcher.start(); } - public identifyUser(vuid: string?, userId: string): void { + public registerVuid(vuid: string): void { + const identifiers = new Map(); + identifiers.set(ODP_USER_KEY.VUID, vuid); + + const event = new OdpEvent('fullstack', 'client_initialized', identifiers); + this.sendEvent(event); + } + + public identifyUser(userId: string, vuid?: string): void { const identifiers = new Map(); - if (vuid != null) { + if (vuid) { identifiers.set(ODP_USER_KEY.VUID, vuid); } identifiers.set(ODP_USER_KEY.FS_USER_ID, userId); @@ -69,13 +74,9 @@ export class OdpEventManager implements IOdpEventManager { this.eventDispatcher.updateSettings(odpConfig); } - public sendEvents(events: OdpEvent[]): void { - events.forEach(event => this.sendEvent(event)); - } - public sendEvent(event: OdpEvent): void { event.data = this.augmentCommonData(event.data); - (async () => await this.eventDispatcher.enqueue(event))(); + this.eventDispatcher.enqueue(event); } private augmentCommonData(sourceData: Map): Map { @@ -101,9 +102,8 @@ export class OdpEventManager implements IOdpEventManager { return data; } - public signalStop(): void { - (async () => await this.eventDispatcher.stop())(); - this.isRunning = false; + public async signalStop(): Promise { + await this.eventDispatcher.stop(); } } diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index 291925a5d..87ec2624e 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -45,6 +45,7 @@ const EVENTS: OdpEvent[] = [ new Map([['id-key-2', 'id-value-2']]), new Map(Object.entries({ 'key-2': 'value2', + 'data_source': 'my-source', })), ), ]; @@ -105,23 +106,23 @@ const abortableRequest = (statusCode: number, body: string) => { describe('OdpEventManager', () => { let mockLogger: LogHandler; - let mockRestApiManager: RestApiManager; + let mockApiManager: RestApiManager; let odpConfig: OdpConfig; beforeAll(() => { mockLogger = mock(); - mockRestApiManager = mock(); + mockApiManager = mock(); odpConfig = new OdpConfig(API_KEY, API_HOST, []); }); beforeEach(() => { resetCalls(mockLogger); - resetCalls(mockRestApiManager); + resetCalls(mockApiManager); }); it('should log and discard events when event manager not running', () => { const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger); + const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); const eventManager = new OdpEventManager(eventDispatcher, logger); // since we've not called start() then... @@ -135,7 +136,11 @@ describe('OdpEventManager', () => { const logger = instance(mockLogger); const mockOdpConfig = mock(); when(mockOdpConfig.isReady()).thenReturn(false); - const eventDispatcher = new OdpEventDispatcher(instance(mockOdpConfig), instance(mockRestApiManager), logger); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig: instance(mockOdpConfig), + apiManager: instance(mockApiManager), + logger, + }); eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already in running state const eventManager = new OdpEventManager(eventDispatcher, logger); @@ -149,7 +154,12 @@ describe('OdpEventManager', () => { const mockOdpConfig = mock(); when(mockOdpConfig.isReady()).thenReturn(false); // set queue to maximum of 1 - const eventDispatcher = new OdpEventDispatcher(mockOdpConfig, instance(mockRestApiManager), logger, 1); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig: mockOdpConfig, + apiManager: instance(mockApiManager), + logger, + queueSize: 1, + }); eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher running eventDispatcher['queue'].push(EVENTS[0]); // simulate event already in queue const eventManager = new OdpEventManager(eventDispatcher, logger); @@ -162,7 +172,7 @@ describe('OdpEventManager', () => { it('should add additional information to each event', () => { const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger); + const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); const eventManager = new OdpEventManager(eventDispatcher, logger); const processedEventData = PROCESSED_EVENTS[0].data; @@ -180,8 +190,15 @@ describe('OdpEventManager', () => { it('should dispatch events in correct number of batches', async () => { const logger = instance(mockLogger); - when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 10, 100); + when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig, + apiManager: instance(mockApiManager), + logger, + queueSize: 100, + batchSize: 10, + flushInterval: 100, + }); const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); @@ -191,20 +208,27 @@ describe('OdpEventManager', () => { await pause(1500); // 3 batches: batch #1 with 10, batch #2 with 10, and batch #3 with 5 = 25 events - verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).thrice(); + verify(mockApiManager.sendEvents(anything(), anything(), anything())).thrice(); }); it('should dispatch events with correct payload', async () => { const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 10, 100); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig, + apiManager: instance(mockApiManager), + logger, + queueSize: 100, + batchSize: 10, + flushInterval: 100, + }); const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); - eventManager.sendEvents(EVENTS); + EVENTS.forEach(event => eventManager.sendEvent(event)); await pause(1500); - verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).once(); - const [apiKey, apiHost, events] = capture(mockRestApiManager.sendEvents).last(); + verify(mockApiManager.sendEvents(anything(), anything(), anything())).once(); + const [apiKey, apiHost, events] = capture(mockApiManager.sendEvents).last(); expect(apiKey).toEqual(API_KEY); expect(apiHost).toEqual(API_HOST); expect(events.length).toEqual(2); @@ -216,10 +240,17 @@ describe('OdpEventManager', () => { it('should retry failed events', async () => { // all events should fail ie shouldRetry = true - when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(true); + when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(true); const logger = instance(mockLogger); // batch size of 2 - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 2, 100); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig, + apiManager: instance(mockApiManager), + logger, + queueSize: 100, + batchSize: 2, + flushInterval: 100, + }); const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); @@ -230,14 +261,21 @@ describe('OdpEventManager', () => { await pause(1500); // retry 3x (default) for 2 batches or 6 calls to attempt to process - verify(mockRestApiManager.sendEvents(anything(), anything(), anything())).times(6); + verify(mockApiManager.sendEvents(anything(), anything(), anything())).times(6); }); it('should flush all scheduled events before stopping', async () => { const logger = instance(mockLogger); - when(mockRestApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); + when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); // batches of 2 with... - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), logger, 100, 2, 100); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig, + apiManager: instance(mockApiManager), + logger, + queueSize: 100, + batchSize: 2, + flushInterval: 100, + }); const eventManager = new OdpEventManager(eventDispatcher, logger); eventManager.start(); @@ -246,8 +284,7 @@ describe('OdpEventManager', () => { eventManager.sendEvent(makeEvent(i)); } await pause(300); - eventManager.signalStop(); - await pause(1500); + await eventManager.signalStop(); verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.')).once(); // ...never exceed 14 @@ -255,18 +292,61 @@ describe('OdpEventManager', () => { verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stopped. Queue Count: 0')).once(); }); + it('should prepare correct payload for register VUID', async () => { + const mockRequestHandler: RequestHandler = mock(); + when(mockRequestHandler.makeRequest(anything(), anything(), anything(), anything())).thenReturn(abortableRequest(200, '')); + const logger = instance(mockLogger); + const apiManager = new RestApiManager(instance(mockRequestHandler), logger); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig, + apiManager, + logger, + queueSize: 100, + batchSize: 10, + flushInterval: 100, + }); + const eventManager = new OdpEventManager(eventDispatcher, logger); + const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; + + eventManager.start(); + eventManager.registerVuid(vuid); + await pause(1500); + + const [requestUrl, headers, method, data] = capture(mockRequestHandler.makeRequest).last(); + expect(requestUrl).toEqual(`${API_HOST}/v3/events`); + expect(headers['Content-Type']).toEqual('application/json'); + expect(headers['x-api-key']).toEqual('test-api-key'); + expect(method).toEqual('POST'); + const events = JSON.parse(data as string); + const event = events[0]; + expect(event.type).toEqual('fullstack'); + expect(event.action).toEqual('client_initialized'); + expect(event.identifiers).toEqual({ 'vuid': vuid }); + expect(event.data.idempotence_id.length).toBe(36); // uuid length + expect(event.data.data_source_type).toEqual('sdk'); + expect(event.data.data_source).toEqual('javascript-sdk'); + expect(event.data.data_source_version).not.toBeNull(); + }); + it('should prepare correct payload for identify user', async () => { const mockRequestHandler: RequestHandler = mock(); when(mockRequestHandler.makeRequest(anything(), anything(), anything(), anything())).thenReturn(abortableRequest(200, '')); const logger = instance(mockLogger); - const restApiManager = new RestApiManager(instance(mockRequestHandler), logger); - const eventDispatcher = new OdpEventDispatcher(odpConfig, restApiManager, logger, 100, 10, 100); + const apiManager = new RestApiManager(instance(mockRequestHandler), logger); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig, + apiManager, + logger, + queueSize: 100, + batchSize: 10, + flushInterval: 100, + }); const eventManager = new OdpEventManager(eventDispatcher, logger); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; const fsUserId = 'test-fs-user-id'; eventManager.start(); - eventManager.identifyUser(vuid, fsUserId); + eventManager.identifyUser(fsUserId, vuid); await pause(1500); const [requestUrl, headers, method, data] = capture(mockRequestHandler.makeRequest).last(); @@ -277,7 +357,7 @@ describe('OdpEventManager', () => { const events = JSON.parse(data as string); const event = events[0]; expect(event.type).toEqual('fullstack'); - expect(event.action).toEqual('client_initialized'); + expect(event.action).toEqual('identified'); expect(event.identifiers).toEqual({ 'vuid': vuid, 'fs_user_id': fsUserId }); expect(event.data.idempotence_id.length).toBe(36); // uuid length expect(event.data.data_source_type).toEqual('sdk'); @@ -286,7 +366,11 @@ describe('OdpEventManager', () => { }); it('should apply updated ODP configuration when available', () => { - const eventDispatcher = new OdpEventDispatcher(odpConfig, instance(mockRestApiManager), instance(mockLogger)); + const eventDispatcher = new OdpEventDispatcher({ + odpConfig, + apiManager: instance(mockApiManager), + logger: instance(mockLogger), + }); const eventManager = new OdpEventManager(eventDispatcher, mockLogger); const apiKey = 'testing-api-key'; const apiHost = 'https://some.other.example.com'; From 58a6bd59b259e1dc5772211961fe122c5ff5a2a7 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 4 Oct 2022 13:03:17 -0400 Subject: [PATCH 22/32] Code review edits and JS Doc --- .../lib/plugins/odp/odp_event_dispatcher.ts | 82 ++++++++++++++++++- .../lib/plugins/odp/odp_event_manager.ts | 74 ++++++++++++++--- 2 files changed, 143 insertions(+), 13 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index 07bef5b2e..baa908994 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -23,12 +23,18 @@ const MAX_RETRIES = 3; const DEFAULT_BATCH_SIZE = 10; const DEFAULT_FLUSH_INTERVAL_MSECS = 1000; +/** + * Event dispatcher's execution states + */ export enum STATE { STOPPED, RUNNING, PROCESSING, } +/** + * Queue processor for dispatching events to the Optimizely Data Platform (ODP) + */ export interface IOdpEventDispatcher { start(): void; @@ -39,20 +45,65 @@ export interface IOdpEventDispatcher { stop(): Promise; } +/** + * Concreate implementation of a processor for dispatching events to the Optimizely Data Platform (ODP) + */ export class OdpEventDispatcher implements IOdpEventDispatcher { + /** + * Current state of the event processor + */ public state: STATE = STATE.STOPPED; - + /** + * Queue for holding all events to be eventually dispatched + * @private + */ private queue = new Array(); + /** + * Current batch of events being processed + * @private + */ private batch = new Array(); + /** + * Identifier of the currently running timeout so clearTimeout() can be called + * @private + */ private timeoutId: ReturnType = setTimeout(() => { }); + /** + * ODP configuration settings in used + * @private + */ private odpConfig: OdpConfig; + /** + * Signal that the dispatcher should drain the queue and shutdown + * @private + */ private shouldStopAndDrain = false; + /** + * REST API Manager used to send the events + * @private + */ private readonly apiManager: RestApiManager; + /** + * Handler for recording execution logs + * @private + */ private readonly logger: LogHandler; + /** + * Maximum queue size + * @private + */ private readonly queueSize: number; + /** + * Maximum number of events to process at once + * @private + */ private readonly batchSize: number; + /** + * Milliseconds between setTimeout() to process new batches + * @private + */ private readonly flushInterval: number; public constructor( @@ -76,15 +127,26 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.state = STATE.STOPPED; } + /** + * Begin processing any events in the queue + */ public start(): void { this.state = STATE.RUNNING; (async () => await this.processQueue())(); } + /** + * Update the ODP configuration in use + * @param odpConfig New settings to apply + */ public updateSettings(odpConfig: OdpConfig): void { this.odpConfig = odpConfig; } + /** + * Add a new event to the main queue + * @param event ODP Event to be queued + */ public enqueue(event: OdpEvent): void { if (this.state != STATE.RUNNING) { this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.'); @@ -104,12 +166,16 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.queue.push(event); } + /** + * Process any events in the main queue in batches + * @private + */ private async processQueue(): Promise { if (this.state !== STATE.RUNNING && !this.shouldStopAndDrain) { return; } - clearInterval(this.timeoutId); + clearTimeout(this.timeoutId); if (this.odpConfig.isReady()) { if (this.queue.length > 0) { @@ -143,7 +209,14 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { await this.processQueue(); } } else { - this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready.'); + if (process) { + // if Node/server-side context, empty queue items before ready state + this.logger.log(LogLevel.WARNING, 'ODPConfig not ready. Leaving events in queue.'); + this.queue = new Array(); + } else { + // in Browser/client-side context, give debug message but leave events in queue + this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready. Leaving events in queue.'); + } } if (!this.shouldStopAndDrain) { @@ -151,6 +224,9 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { } } + /** + * Drain the event queue sending all remaining events in batches to ODP then shutdown processing + */ public async stop(): Promise { this.logger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.'); this.shouldStopAndDrain = true; diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index d6340e34e..3a5b3d0fc 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -14,29 +14,32 @@ * limitations under the License. */ -import { LogHandler } from '../../modules/logging'; +import { LogHandler, LogLevel } from '../../modules/logging'; import { OdpEvent } from './odp_event'; import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; import { OdpEventDispatcher } from './odp_event_dispatcher'; +/** + * Manager for persisting events to the Optimizely Data Platform (ODP) + */ export interface IOdpEventManager { + updateSettings(odpConfig: OdpConfig): void; + start(): void; registerVuid(vuid: string): void; identifyUser(userId: string, vuid?: string): void; - updateSettings(odpConfig: OdpConfig): void; - sendEvent(event: OdpEvent): void; - signalStop(): void; + signalStop(): Promise; } /** - * Manager for persisting events to the Optimizely Data Platform + * Concrete implementation of a manager for persisting events to the Optimizely Data Platform */ export class OdpEventManager implements IOdpEventManager { private readonly eventDispatcher: OdpEventDispatcher; @@ -47,10 +50,25 @@ export class OdpEventManager implements IOdpEventManager { this.logger = logger; } + /** + * Update ODP configuration settings + * @param odpConfig New configuration to apply + */ + public updateSettings(odpConfig: OdpConfig): void { + this.eventDispatcher.updateSettings(odpConfig); + } + + /** + * Start processing events in the dispatcher's queue + */ public start(): void { this.eventDispatcher.start(); } + /** + * Register a new visitor user id (VUID) in ODP + * @param vuid Visitor user id to send + */ public registerVuid(vuid: string): void { const identifiers = new Map(); identifiers.set(ODP_USER_KEY.VUID, vuid); @@ -59,6 +77,11 @@ export class OdpEventManager implements IOdpEventManager { this.sendEvent(event); } + /** + * Associate a full-stack userid with an established VUID + * @param userId Full-stack User ID + * @param vuid Visitor User ID + */ public identifyUser(userId: string, vuid?: string): void { const identifiers = new Map(); if (vuid) { @@ -70,15 +93,43 @@ export class OdpEventManager implements IOdpEventManager { this.sendEvent(event); } - public updateSettings(odpConfig: OdpConfig): void { - this.eventDispatcher.updateSettings(odpConfig); + /** + * Send an event to ODP via dispatch queue + * @param event ODP Event to forward + */ + public sendEvent(event: OdpEvent): void { + const foundInvalidDataInKeys = this.findKeysWithInvalidData(event.data); + if (foundInvalidDataInKeys.length > 0) { + this.logger.log(LogLevel.ERROR, `Event data found to be invalid (${foundInvalidDataInKeys.join(', ')}`); + } else { + event.data = this.augmentCommonData(event.data); + this.eventDispatcher.enqueue(event); + } } - public sendEvent(event: OdpEvent): void { - event.data = this.augmentCommonData(event.data); - this.eventDispatcher.enqueue(event); + /** + * Validate event data value types + * @param data Event data to be validated + * @returns Array of event data keys that were found to be invalid + * @private + */ + private findKeysWithInvalidData(data: Map): string[] { + const validTypes: string[] = ['string', 'number', 'boolean', 'bigint']; + const invalidKeys: string[] = []; + data.forEach((value, key) => { + if (!validTypes.includes(typeof value) && value !== null) { + invalidKeys.push(key); + } + }); + return invalidKeys; } + /** + * Add additional common data including an idempotent ID and execution context to event data + * @param sourceData Existing event data to augment + * @returns Augmented event data + * @private + */ private augmentCommonData(sourceData: Map): Map { // Try to get information from the current execution context let sourceVersion = ''; @@ -102,6 +153,9 @@ export class OdpEventManager implements IOdpEventManager { return data; } + /** + * Signal to event dispatcher to drain the event queue and stop + */ public async signalStop(): Promise { await this.eventDispatcher.stop(); } From 60d435a0a599b598ac70846c3a3d3e6563914e5f Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 4 Oct 2022 13:11:09 -0400 Subject: [PATCH 23/32] Fix message --- packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index baa908994..58a3da65a 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -211,7 +211,7 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { } else { if (process) { // if Node/server-side context, empty queue items before ready state - this.logger.log(LogLevel.WARNING, 'ODPConfig not ready. Leaving events in queue.'); + this.logger.log(LogLevel.WARNING, 'ODPConfig not ready. Discarding events in queue.'); this.queue = new Array(); } else { // in Browser/client-side context, give debug message but leave events in queue From ab63fab875c6cc6aae813e527ad9bb144a9ce9fe Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Wed, 5 Oct 2022 14:34:36 -0400 Subject: [PATCH 24/32] Code review requested changes --- .../lib/plugins/odp/odp_event_dispatcher.ts | 190 +++++++++++------- .../lib/plugins/odp/odp_event_manager.ts | 6 +- .../tests/odpEventManager.spec.ts | 47 +++-- packages/optimizely-sdk/tsconfig.spec.json | 14 ++ 4 files changed, 168 insertions(+), 89 deletions(-) create mode 100644 packages/optimizely-sdk/tsconfig.spec.json diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index 58a3da65a..f7a8eaf14 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -36,17 +36,17 @@ export enum STATE { * Queue processor for dispatching events to the Optimizely Data Platform (ODP) */ export interface IOdpEventDispatcher { - start(): void; - updateSettings(odpConfig: OdpConfig): void; + start(): void; + enqueue(event: OdpEvent): void; stop(): Promise; } /** - * Concreate implementation of a processor for dispatching events to the Optimizely Data Platform (ODP) + * Concrete implementation of a processor for dispatching events to the Optimizely Data Platform (ODP) */ export class OdpEventDispatcher implements IOdpEventDispatcher { /** @@ -59,27 +59,15 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { */ private queue = new Array(); /** - * Current batch of events being processed + * Identifier of the currently running timeout so clearCurrentTimeout() can be called * @private */ - private batch = new Array(); - /** - * Identifier of the currently running timeout so clearTimeout() can be called - * @private - */ - private timeoutId: ReturnType = setTimeout(() => { - }); + private timeoutId?: NodeJS.Timeout | number; /** * ODP configuration settings in used * @private */ private odpConfig: OdpConfig; - /** - * Signal that the dispatcher should drain the queue and shutdown - * @private - */ - private shouldStopAndDrain = false; - /** * REST API Manager used to send the events * @private @@ -127,14 +115,6 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.state = STATE.STOPPED; } - /** - * Begin processing any events in the queue - */ - public start(): void { - this.state = STATE.RUNNING; - (async () => await this.processQueue())(); - } - /** * Update the ODP configuration in use * @param odpConfig New settings to apply @@ -143,12 +123,21 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.odpConfig = odpConfig; } + /** + * Begin processing any events in the queue + */ + public start(): void { + this.state = STATE.RUNNING; + + this.setNewTimeout(); + } + /** * Add a new event to the main queue * @param event ODP Event to be queued */ public enqueue(event: OdpEvent): void { - if (this.state != STATE.RUNNING) { + if (this.state === STATE.STOPPED) { this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.'); return; } @@ -164,6 +153,8 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { } this.queue.push(event); + + this.processQueue(); } /** @@ -171,66 +162,127 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { * @private */ private async processQueue(): Promise { - if (this.state !== STATE.RUNNING && !this.shouldStopAndDrain) { + if (this.state !== STATE.RUNNING) { return; } + if (!this.isOdpConfigurationReady()) { + return; + } + + if (!this.queueHasBatches()) { + return; + } + + this.clearCurrentTimeout(); + + this.state = STATE.PROCESSING; + + while (this.queueHasBatches()) { + await this.makeAndSendBatch(); + } + + this.state = STATE.RUNNING; + + this.setNewTimeout(); + } + + /** + * Process all events in the main queue in batches until empty + * @private + */ + private async flushQueue(): Promise { + if (this.state !== STATE.RUNNING) { + return; + } + + if (!this.isOdpConfigurationReady()) { + return; + } + + if (!this.queueContainsItems()) { + return; + } + + this.clearCurrentTimeout(); + + this.state = STATE.PROCESSING; + + while (this.queueContainsItems()) { + await this.makeAndSendBatch(); + } + + this.state = STATE.RUNNING; + + this.setNewTimeout(); + } + + private clearCurrentTimeout(): void { clearTimeout(this.timeoutId); + this.timeoutId = undefined; + } - if (this.odpConfig.isReady()) { - if (this.queue.length > 0) { - - this.state = STATE.PROCESSING; - - for (let count = 0; count < this.batchSize; count += 1) { - const event = this.queue.shift(); - if (event) { - this.batch.push(event); - } else { - break; - } - } - - if (this.batch.length > 0) { - let shouldRetry: boolean; - let numAttempts = 0; - do { - shouldRetry = await this.apiManager.sendEvents(this.odpConfig.apiKey, this.odpConfig.apiHost, this.batch); - numAttempts += 1; - } while (shouldRetry && numAttempts < MAX_RETRIES); - } - - this.batch = new Array(); - this.state = STATE.RUNNING; - } + private setNewTimeout(): void { + if (this.timeoutId !== undefined) { + return; + } + this.timeoutId = setTimeout(() => this.flushQueue(), this.flushInterval); + } - if (this.shouldStopAndDrain && this.queue.length > 0) { - this.logger.log(LogLevel.DEBUG, 'EventDispatcher draining queue without flush interval.'); - await this.processQueue(); - } - } else { - if (process) { - // if Node/server-side context, empty queue items before ready state - this.logger.log(LogLevel.WARNING, 'ODPConfig not ready. Discarding events in queue.'); - this.queue = new Array(); + private async makeAndSendBatch(): Promise { + const batch = new Array(); + + for (let count = 0; count < this.batchSize; count += 1) { + const event = this.queue.shift(); + if (event) { + batch.push(event); } else { - // in Browser/client-side context, give debug message but leave events in queue - this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready. Leaving events in queue.'); + break; } } - if (!this.shouldStopAndDrain) { - this.timeoutId = setTimeout(() => this.processQueue(), this.flushInterval); + if (batch.length > 0) { + let shouldRetry: boolean; + let attemptNumber = 0; + do { + shouldRetry = await this.apiManager.sendEvents(this.odpConfig.apiKey, this.odpConfig.apiHost, batch); + attemptNumber += 1; + } while (shouldRetry && attemptNumber < MAX_RETRIES); + } + } + + private queueHasBatches(): boolean { + return this.queueContainsItems() && this.queue.length % this.batchSize === 0; + } + + private queueContainsItems(): boolean { + return this.queue.length > 0; + } + + private isOdpConfigurationReady(): boolean { + if (this.odpConfig.isReady()) { + return true; + } + + if (process) { + // if Node/server-side context, empty queue items before ready state + this.logger.log(LogLevel.WARNING, 'ODPConfig not ready. Discarding events in queue.'); + this.queue = new Array(); + } else { + // in Browser/client-side context, give debug message but leave events in queue + this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready. Leaving events in queue.'); } + return false; } /** - * Drain the event queue sending all remaining events in batches to ODP then shutdown processing + * Drain the queue sending all remaining events in batches then stop processing */ public async stop(): Promise { this.logger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.'); - this.shouldStopAndDrain = true; - await this.processQueue(); + + await this.flushQueue(); + this.state = STATE.STOPPED; this.logger.log(LogLevel.DEBUG, `EventDispatcher stopped. Queue Count: ${this.queue.length}`); } diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 3a5b3d0fc..7b2b6d5d4 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -67,7 +67,7 @@ export class OdpEventManager implements IOdpEventManager { /** * Register a new visitor user id (VUID) in ODP - * @param vuid Visitor user id to send + * @param vuid Visitor User ID to send */ public registerVuid(vuid: string): void { const identifiers = new Map(); @@ -100,7 +100,8 @@ export class OdpEventManager implements IOdpEventManager { public sendEvent(event: OdpEvent): void { const foundInvalidDataInKeys = this.findKeysWithInvalidData(event.data); if (foundInvalidDataInKeys.length > 0) { - this.logger.log(LogLevel.ERROR, `Event data found to be invalid (${foundInvalidDataInKeys.join(', ')}`); + this.logger.log(LogLevel.ERROR, `Event data found to be invalid.`); + this.logger.log(LogLevel.DEBUG, `Invalid event data keys (${foundInvalidDataInKeys.join(', ')})`); } else { event.data = this.augmentCommonData(event.data); this.eventDispatcher.enqueue(event); @@ -160,4 +161,3 @@ export class OdpEventManager implements IOdpEventManager { await this.eventDispatcher.stop(); } } - diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index 87ec2624e..cc4b75dce 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -16,7 +16,7 @@ import { OdpConfig } from '../lib/plugins/odp/odp_config'; import { OdpEventManager } from '../lib/plugins/odp/odp_event_manager'; -import { anything, capture, instance, mock, resetCalls, verify, when } from 'ts-mockito'; +import { anyString, anything, capture, instance, mock, resetCalls, verify, when } from 'ts-mockito'; import { RestApiManager } from '../lib/plugins/odp/rest_api_manager'; import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; @@ -132,7 +132,7 @@ describe('OdpEventManager', () => { verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.')).once(); }); - it('should log and discard events when event manager\'s config is not ready', () => { + it('should log and discard events when event manager config is not ready', () => { const logger = instance(mockLogger); const mockOdpConfig = mock(); when(mockOdpConfig.isReady()).thenReturn(false); @@ -149,6 +149,26 @@ describe('OdpEventManager', () => { verify(mockLogger.log(LogLevel.DEBUG, 'Unable to Process ODP Event. ODPConfig is not ready.')).once(); }); + it('should discard events with invalid data', () => { + const logger = instance(mockLogger); + const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); + const eventManager = new OdpEventManager(eventDispatcher, logger); + // make an event with invalid data key-value entry + const badEvent = new OdpEvent( + 't3', + 'a3', + new Map([['id-key-3', 'id-value-3']]), + new Map(Object.entries({ + 'key-1': false, + 'key-2': { random: 'object', whichShouldFail: true }, + })), + ); + eventManager.sendEvent(badEvent); + + verify(mockLogger.log(LogLevel.ERROR, 'Event data found to be invalid.')).once(); + verify(mockLogger.log(LogLevel.DEBUG, anyString())).once(); + }); + it('should log a max queue hit and discard ', () => { const logger = instance(mockLogger); const mockOdpConfig = mock(); @@ -195,9 +215,8 @@ describe('OdpEventManager', () => { odpConfig, apiManager: instance(mockApiManager), logger, - queueSize: 100, - batchSize: 10, - flushInterval: 100, + batchSize: 10, // with batch size of 10... + flushInterval: 250, }); const eventManager = new OdpEventManager(eventDispatcher, logger); @@ -207,7 +226,8 @@ describe('OdpEventManager', () => { } await pause(1500); - // 3 batches: batch #1 with 10, batch #2 with 10, and batch #3 with 5 = 25 events + // ...there should be 3 batches: + // batch #1 with 10, batch #2 with 10, and batch #3 (after flushInterval lapsed) with 5 = 25 events verify(mockApiManager.sendEvents(anything(), anything(), anything())).thrice(); }); @@ -217,7 +237,6 @@ describe('OdpEventManager', () => { odpConfig, apiManager: instance(mockApiManager), logger, - queueSize: 100, batchSize: 10, flushInterval: 100, }); @@ -225,8 +244,9 @@ describe('OdpEventManager', () => { eventManager.start(); EVENTS.forEach(event => eventManager.sendEvent(event)); - await pause(1500); + await pause(1000); + // sending 1 batch of 2 events after flushInterval since batchSize is 10 verify(mockApiManager.sendEvents(anything(), anything(), anything())).once(); const [apiKey, apiHost, events] = capture(mockApiManager.sendEvents).last(); expect(apiKey).toEqual(API_KEY); @@ -242,13 +262,11 @@ describe('OdpEventManager', () => { // all events should fail ie shouldRetry = true when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(true); const logger = instance(mockLogger); - // batch size of 2 const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger, - queueSize: 100, - batchSize: 2, + batchSize: 2, // batch size of 2 flushInterval: 100, }); const eventManager = new OdpEventManager(eventDispatcher, logger); @@ -267,13 +285,11 @@ describe('OdpEventManager', () => { it('should flush all scheduled events before stopping', async () => { const logger = instance(mockLogger); when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); - // batches of 2 with... const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger, - queueSize: 100, - batchSize: 2, + batchSize: 2, // batches of 2 with... flushInterval: 100, }); const eventManager = new OdpEventManager(eventDispatcher, logger); @@ -301,7 +317,6 @@ describe('OdpEventManager', () => { odpConfig, apiManager, logger, - queueSize: 100, batchSize: 10, flushInterval: 100, }); @@ -337,8 +352,6 @@ describe('OdpEventManager', () => { odpConfig, apiManager, logger, - queueSize: 100, - batchSize: 10, flushInterval: 100, }); const eventManager = new OdpEventManager(eventDispatcher, logger); diff --git a/packages/optimizely-sdk/tsconfig.spec.json b/packages/optimizely-sdk/tsconfig.spec.json new file mode 100644 index 000000000..877e2b462 --- /dev/null +++ b/packages/optimizely-sdk/tsconfig.spec.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": [ + "jest" + ], + "typeRoots": [ + "./node_modules/@types" + ] + }, + "include": [ + "**/*.spec.ts" + ] +} From b50d357e6a8b3ad2349cc8f7df44267a7a82723b Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Wed, 5 Oct 2022 14:38:38 -0400 Subject: [PATCH 25/32] JS Docs --- .../lib/plugins/odp/odp_event_dispatcher.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index f7a8eaf14..99bb5d676 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -217,11 +217,19 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.setNewTimeout(); } + /** + * Clear the currently running timout + * @private + */ private clearCurrentTimeout(): void { clearTimeout(this.timeoutId); this.timeoutId = undefined; } + /** + * Start a new timeout + * @private + */ private setNewTimeout(): void { if (this.timeoutId !== undefined) { return; @@ -229,6 +237,10 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { this.timeoutId = setTimeout(() => this.flushQueue(), this.flushInterval); } + /** + * Make a batch and send it to ODP + * @private + */ private async makeAndSendBatch(): Promise { const batch = new Array(); @@ -251,14 +263,27 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { } } + /** + * Check if main queue has any full/even batches available + * @private + */ private queueHasBatches(): boolean { return this.queueContainsItems() && this.queue.length % this.batchSize === 0; } + /** + * Check if main queue has any items + * @private + */ private queueContainsItems(): boolean { return this.queue.length > 0; } + /** + * Check if the ODP Configuration is ready and log if not. + * Potentially clear queue if server-side + * @private + */ private isOdpConfigurationReady(): boolean { if (this.odpConfig.isReady()) { return true; From dc14b73ed0516a84f25c2e6f47273798cc6b02cc Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Fri, 7 Oct 2022 15:22:51 -0400 Subject: [PATCH 26/32] Fix data_source_version --- .../lib/plugins/odp/odp_event_manager.ts | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 7b2b6d5d4..a135bc03e 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -20,6 +20,7 @@ import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; import { OdpEventDispatcher } from './odp_event_dispatcher'; +import packageJson from '../../../package.json'; /** * Manager for persisting events to the Optimizely Data Platform (ODP) @@ -132,25 +133,13 @@ export class OdpEventManager implements IOdpEventManager { * @private */ private augmentCommonData(sourceData: Map): Map { - // Try to get information from the current execution context - let sourceVersion = ''; - if (window) { - sourceVersion = window.navigator.userAgent; - } else { - if (process) { - sourceVersion = process.version; - } - } - const data = new Map(); data.set('idempotence_id', uuid()); data.set('data_source_type', 'sdk'); data.set('data_source', 'javascript-sdk'); - if (sourceVersion) { - data.set('data_source_version', sourceVersion); - } - sourceData.forEach((value, key) => data.set(key, value)); + data.set('data_source_version', packageJson.version); + sourceData.forEach((value, key) => data.set(key, value)); return data; } From 1d2abb83958b6a6115496ec9b3cb45e9897a8317 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Fri, 7 Oct 2022 15:38:44 -0400 Subject: [PATCH 27/32] Remove tsconfig.spec.json --- packages/optimizely-sdk/tsconfig.spec.json | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 packages/optimizely-sdk/tsconfig.spec.json diff --git a/packages/optimizely-sdk/tsconfig.spec.json b/packages/optimizely-sdk/tsconfig.spec.json deleted file mode 100644 index 877e2b462..000000000 --- a/packages/optimizely-sdk/tsconfig.spec.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "types": [ - "jest" - ], - "typeRoots": [ - "./node_modules/@types" - ] - }, - "include": [ - "**/*.spec.ts" - ] -} From f82fbff278599a6933551171006dcd6ba8173599 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Mon, 10 Oct 2022 11:32:29 -0400 Subject: [PATCH 28/32] Read client engine and version from OptiOptions --- .../lib/plugins/odp/odp_event_manager.ts | 15 +++++++++++---- .../optimizely-sdk/tests/odpEventManager.spec.ts | 15 +++++++++++---- packages/optimizely-sdk/tsconfig.spec.json | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 packages/optimizely-sdk/tsconfig.spec.json diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index a135bc03e..5d4c2ffc4 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -20,7 +20,7 @@ import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; import { OdpEventDispatcher } from './odp_event_dispatcher'; -import packageJson from '../../../package.json'; +import { OptimizelyOptions } from '../../shared_types'; /** * Manager for persisting events to the Optimizely Data Platform (ODP) @@ -45,10 +45,12 @@ export interface IOdpEventManager { export class OdpEventManager implements IOdpEventManager { private readonly eventDispatcher: OdpEventDispatcher; private readonly logger: LogHandler; + private readonly config?: OptimizelyOptions; - public constructor(eventDispatcher: OdpEventDispatcher, logger: LogHandler) { + public constructor(eventDispatcher: OdpEventDispatcher, logger: LogHandler, config?: OptimizelyOptions) { this.eventDispatcher = eventDispatcher; this.logger = logger; + this.config = config; } /** @@ -136,8 +138,13 @@ export class OdpEventManager implements IOdpEventManager { const data = new Map(); data.set('idempotence_id', uuid()); data.set('data_source_type', 'sdk'); - data.set('data_source', 'javascript-sdk'); - data.set('data_source_version', packageJson.version); + + if (this.config?.clientEngine) { + data.set('data_source', this.config?.clientEngine); + } + if (this.config?.clientVersion) { + data.set('data_source_version', this.config?.clientVersion); + } sourceData.forEach((value, key) => data.set(key, value)); return data; diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index cc4b75dce..af7f85e0b 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -22,6 +22,7 @@ import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; import { RequestHandler } from '../lib/utils/http_request_handler/http'; import { OdpEventDispatcher, STATE } from '../lib/plugins/odp/odp_event_dispatcher'; +import { OptimizelyOptions } from '../lib/shared_types'; const API_KEY = 'test-api-key'; const API_HOST = 'https://odp.example.com'; @@ -108,16 +109,22 @@ describe('OdpEventManager', () => { let mockLogger: LogHandler; let mockApiManager: RestApiManager; let odpConfig: OdpConfig; + let mockOptimizelyOptions: OptimizelyOptions; beforeAll(() => { mockLogger = mock(); mockApiManager = mock(); odpConfig = new OdpConfig(API_KEY, API_HOST, []); + + mockOptimizelyOptions = mock(); + when(mockOptimizelyOptions.clientEngine).thenReturn('javascript-sdk'); + when(mockOptimizelyOptions.clientVersion).thenReturn('4.9.2'); }); beforeEach(() => { resetCalls(mockLogger); resetCalls(mockApiManager); + resetCalls(mockOptimizelyOptions); }); it('should log and discard events when event manager not running', () => { @@ -193,7 +200,7 @@ describe('OdpEventManager', () => { it('should add additional information to each event', () => { const logger = instance(mockLogger); const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); const processedEventData = PROCESSED_EVENTS[0].data; const eventData = eventManager['augmentCommonData'](EVENTS[0].data); @@ -240,7 +247,7 @@ describe('OdpEventManager', () => { batchSize: 10, flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); eventManager.start(); EVENTS.forEach(event => eventManager.sendEvent(event)); @@ -320,7 +327,7 @@ describe('OdpEventManager', () => { batchSize: 10, flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; eventManager.start(); @@ -354,7 +361,7 @@ describe('OdpEventManager', () => { logger, flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; const fsUserId = 'test-fs-user-id'; diff --git a/packages/optimizely-sdk/tsconfig.spec.json b/packages/optimizely-sdk/tsconfig.spec.json new file mode 100644 index 000000000..877e2b462 --- /dev/null +++ b/packages/optimizely-sdk/tsconfig.spec.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": [ + "jest" + ], + "typeRoots": [ + "./node_modules/@types" + ] + }, + "include": [ + "**/*.spec.ts" + ] +} From 46c27202cc240af92146ac1b53e74d01b4886eaa Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Mon, 10 Oct 2022 17:33:37 -0400 Subject: [PATCH 29/32] Code review changes --- .../lib/plugins/odp/odp_event_dispatcher.ts | 63 +++++++------------ .../lib/plugins/odp/odp_event_manager.ts | 24 ++++--- .../tests/odpEventManager.spec.ts | 48 +++++++------- 3 files changed, 61 insertions(+), 74 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts index 99bb5d676..29a688368 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts @@ -158,10 +158,11 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { } /** - * Process any events in the main queue in batches + * Process events in the main queue + * @param shouldFlush Flush all events regardless of available queue event count * @private */ - private async processQueue(): Promise { + private async processQueue(shouldFlush = false): Promise { if (this.state !== STATE.RUNNING) { return; } @@ -170,51 +171,35 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { return; } - if (!this.queueHasBatches()) { - return; - } + // Flush interval occurred & queue has items + if (shouldFlush && this.queueContainsItems()) { + // clear the queue completely + this.clearCurrentTimeout(); - this.clearCurrentTimeout(); + this.state = STATE.PROCESSING; - this.state = STATE.PROCESSING; - - while (this.queueHasBatches()) { - await this.makeAndSendBatch(); - } + while (this.queueContainsItems()) { + await this.makeAndSendBatch(); + } - this.state = STATE.RUNNING; + this.state = STATE.RUNNING; - this.setNewTimeout(); - } - - /** - * Process all events in the main queue in batches until empty - * @private - */ - private async flushQueue(): Promise { - if (this.state !== STATE.RUNNING) { - return; - } - - if (!this.isOdpConfigurationReady()) { - return; + this.setNewTimeout(); } + // Check if queue has a full batch available + else if (this.queueHasBatches()) { + this.clearCurrentTimeout(); - if (!this.queueContainsItems()) { - return; - } + this.state = STATE.PROCESSING; - this.clearCurrentTimeout(); + while (this.queueHasBatches()) { + await this.makeAndSendBatch(); + } - this.state = STATE.PROCESSING; + this.state = STATE.RUNNING; - while (this.queueContainsItems()) { - await this.makeAndSendBatch(); + this.setNewTimeout(); } - - this.state = STATE.RUNNING; - - this.setNewTimeout(); } /** @@ -234,7 +219,7 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { if (this.timeoutId !== undefined) { return; } - this.timeoutId = setTimeout(() => this.flushQueue(), this.flushInterval); + this.timeoutId = setTimeout(() => this.processQueue(true), this.flushInterval); } /** @@ -306,7 +291,7 @@ export class OdpEventDispatcher implements IOdpEventDispatcher { public async stop(): Promise { this.logger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.'); - await this.flushQueue(); + await this.processQueue(true); this.state = STATE.STOPPED; this.logger.log(LogLevel.DEBUG, `EventDispatcher stopped. Queue Count: ${this.queue.length}`); diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 5d4c2ffc4..3a14143c9 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -20,7 +20,6 @@ import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; import { OdpEventDispatcher } from './odp_event_dispatcher'; -import { OptimizelyOptions } from '../../shared_types'; /** * Manager for persisting events to the Optimizely Data Platform (ODP) @@ -45,12 +44,19 @@ export interface IOdpEventManager { export class OdpEventManager implements IOdpEventManager { private readonly eventDispatcher: OdpEventDispatcher; private readonly logger: LogHandler; - private readonly config?: OptimizelyOptions; - - public constructor(eventDispatcher: OdpEventDispatcher, logger: LogHandler, config?: OptimizelyOptions) { + private readonly clientEngine?: string; + private readonly clientVersion?: string; + + public constructor({ eventDispatcher, logger, clientEngine, clientVersion }: { + eventDispatcher: OdpEventDispatcher, + logger: LogHandler, + clientEngine?: string, + clientVersion?: string, + }) { this.eventDispatcher = eventDispatcher; this.logger = logger; - this.config = config; + this.clientEngine = clientEngine; + this.clientVersion = clientVersion; } /** @@ -139,11 +145,11 @@ export class OdpEventManager implements IOdpEventManager { data.set('idempotence_id', uuid()); data.set('data_source_type', 'sdk'); - if (this.config?.clientEngine) { - data.set('data_source', this.config?.clientEngine); + if (this.clientEngine) { + data.set('data_source', this.clientEngine); } - if (this.config?.clientVersion) { - data.set('data_source_version', this.config?.clientVersion); + if (this.clientVersion) { + data.set('data_source_version', this.clientVersion); } sourceData.forEach((value, key) => data.set(key, value)); diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index af7f85e0b..04152f54e 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -22,11 +22,9 @@ import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; import { RequestHandler } from '../lib/utils/http_request_handler/http'; import { OdpEventDispatcher, STATE } from '../lib/plugins/odp/odp_event_dispatcher'; -import { OptimizelyOptions } from '../lib/shared_types'; const API_KEY = 'test-api-key'; const API_HOST = 'https://odp.example.com'; -const MOCK_PROCESS_VERSION = 'v16.17.0'; const MOCK_IDEMPOTENCE_ID = 'c1dc758e-f095-4f09-9b49-172d74c53880'; const EVENTS: OdpEvent[] = [ new OdpEvent( @@ -50,6 +48,9 @@ const EVENTS: OdpEvent[] = [ })), ), ]; +// naming for object destructuring +const clientEngine = 'javascript-sdk'; +const clientVersion = '4.9.2'; const PROCESSED_EVENTS: OdpEvent[] = [ new OdpEvent( 't1', @@ -58,8 +59,8 @@ const PROCESSED_EVENTS: OdpEvent[] = [ new Map(Object.entries({ 'idempotence_id': MOCK_IDEMPOTENCE_ID, 'data_source_type': 'sdk', - 'data_source': 'javascript-sdk', - 'data_source_version': MOCK_PROCESS_VERSION, + 'data_source': clientEngine, + 'data_source_version': clientVersion, 'key-1': 'value1', 'key-2': null, 'key-3': 3.3, @@ -73,8 +74,8 @@ const PROCESSED_EVENTS: OdpEvent[] = [ new Map(Object.entries({ 'idempotence_id': MOCK_IDEMPOTENCE_ID, 'data_source_type': 'sdk', - 'data_source': 'javascript-sdk', - 'data_source_version': MOCK_PROCESS_VERSION, + 'data_source': clientEngine, + 'data_source_version': clientVersion, 'key-2': 'value2', })), ), @@ -109,28 +110,22 @@ describe('OdpEventManager', () => { let mockLogger: LogHandler; let mockApiManager: RestApiManager; let odpConfig: OdpConfig; - let mockOptimizelyOptions: OptimizelyOptions; beforeAll(() => { mockLogger = mock(); mockApiManager = mock(); odpConfig = new OdpConfig(API_KEY, API_HOST, []); - - mockOptimizelyOptions = mock(); - when(mockOptimizelyOptions.clientEngine).thenReturn('javascript-sdk'); - when(mockOptimizelyOptions.clientVersion).thenReturn('4.9.2'); }); beforeEach(() => { resetCalls(mockLogger); resetCalls(mockApiManager); - resetCalls(mockOptimizelyOptions); }); it('should log and discard events when event manager not running', () => { const logger = instance(mockLogger); const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); // since we've not called start() then... eventManager.sendEvent(EVENTS[0]); @@ -149,7 +144,7 @@ describe('OdpEventManager', () => { logger, }); eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already in running state - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); eventManager.sendEvent(EVENTS[0]); @@ -159,7 +154,7 @@ describe('OdpEventManager', () => { it('should discard events with invalid data', () => { const logger = instance(mockLogger); const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); // make an event with invalid data key-value entry const badEvent = new OdpEvent( 't3', @@ -189,7 +184,7 @@ describe('OdpEventManager', () => { }); eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher running eventDispatcher['queue'].push(EVENTS[0]); // simulate event already in queue - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); // try adding the second event eventManager.sendEvent(EVENTS[1]); @@ -200,7 +195,7 @@ describe('OdpEventManager', () => { it('should add additional information to each event', () => { const logger = instance(mockLogger); const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); const processedEventData = PROCESSED_EVENTS[0].data; const eventData = eventManager['augmentCommonData'](EVENTS[0].data); @@ -208,7 +203,7 @@ describe('OdpEventManager', () => { expect((eventData.get('idempotence_id') as string).length).toEqual((processedEventData.get('idempotence_id') as string).length); expect(eventData.get('data_source_type')).toEqual(processedEventData.get('data_source_type')); expect(eventData.get('data_source')).toEqual(processedEventData.get('data_source')); - expect(eventData.get('data_source_version')).not.toBeNull(); + expect(eventData.get('data_source_version')).toEqual(processedEventData.get('data_source_version')); expect(eventData.get('key-1')).toEqual(processedEventData.get('key-1')); expect(eventData.get('key-2')).toEqual(processedEventData.get('key-2')); expect(eventData.get('key-3')).toEqual(processedEventData.get('key-3')); @@ -225,7 +220,7 @@ describe('OdpEventManager', () => { batchSize: 10, // with batch size of 10... flushInterval: 250, }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); eventManager.start(); for (let i = 0; i < 25; i += 1) { @@ -247,7 +242,7 @@ describe('OdpEventManager', () => { batchSize: 10, flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); EVENTS.forEach(event => eventManager.sendEvent(event)); @@ -276,7 +271,7 @@ describe('OdpEventManager', () => { batchSize: 2, // batch size of 2 flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); eventManager.start(); // send 4 events @@ -299,7 +294,7 @@ describe('OdpEventManager', () => { batchSize: 2, // batches of 2 with... flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); eventManager.start(); // ...25 events should... @@ -327,7 +322,7 @@ describe('OdpEventManager', () => { batchSize: 10, flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; eventManager.start(); @@ -361,7 +356,7 @@ describe('OdpEventManager', () => { logger, flushInterval: 100, }); - const eventManager = new OdpEventManager(eventDispatcher, logger, instance(mockOptimizelyOptions)); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; const fsUserId = 'test-fs-user-id'; @@ -386,12 +381,13 @@ describe('OdpEventManager', () => { }); it('should apply updated ODP configuration when available', () => { + const logger = instance(mockLogger); const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), - logger: instance(mockLogger), + logger, }); - const eventManager = new OdpEventManager(eventDispatcher, mockLogger); + const eventManager = new OdpEventManager({ eventDispatcher, logger }); const apiKey = 'testing-api-key'; const apiHost = 'https://some.other.example.com'; const segmentsToCheck = ['empty-cart', '1-item-cart']; From 554773e1707b9326e1587aef31047e8df449cf91 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 11 Oct 2022 08:25:37 -0400 Subject: [PATCH 30/32] Require client engine & version for ODP Event Manager --- .../lib/plugins/odp/odp_event_manager.ts | 17 ++++++----------- .../tests/odpEventManager.spec.ts | 16 ++++++++-------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 3a14143c9..c7e3a5ed7 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -44,14 +44,14 @@ export interface IOdpEventManager { export class OdpEventManager implements IOdpEventManager { private readonly eventDispatcher: OdpEventDispatcher; private readonly logger: LogHandler; - private readonly clientEngine?: string; - private readonly clientVersion?: string; + private readonly clientEngine: string; + private readonly clientVersion: string; public constructor({ eventDispatcher, logger, clientEngine, clientVersion }: { eventDispatcher: OdpEventDispatcher, logger: LogHandler, - clientEngine?: string, - clientVersion?: string, + clientEngine: string, + clientVersion: string, }) { this.eventDispatcher = eventDispatcher; this.logger = logger; @@ -144,13 +144,8 @@ export class OdpEventManager implements IOdpEventManager { const data = new Map(); data.set('idempotence_id', uuid()); data.set('data_source_type', 'sdk'); - - if (this.clientEngine) { - data.set('data_source', this.clientEngine); - } - if (this.clientVersion) { - data.set('data_source_version', this.clientVersion); - } + data.set('data_source', this.clientEngine); + data.set('data_source_version', this.clientVersion); sourceData.forEach((value, key) => data.set(key, value)); return data; diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index 04152f54e..7f22988aa 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -125,7 +125,7 @@ describe('OdpEventManager', () => { it('should log and discard events when event manager not running', () => { const logger = instance(mockLogger); const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); // since we've not called start() then... eventManager.sendEvent(EVENTS[0]); @@ -144,7 +144,7 @@ describe('OdpEventManager', () => { logger, }); eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already in running state - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.sendEvent(EVENTS[0]); @@ -154,7 +154,7 @@ describe('OdpEventManager', () => { it('should discard events with invalid data', () => { const logger = instance(mockLogger); const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); // make an event with invalid data key-value entry const badEvent = new OdpEvent( 't3', @@ -184,7 +184,7 @@ describe('OdpEventManager', () => { }); eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher running eventDispatcher['queue'].push(EVENTS[0]); // simulate event already in queue - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); // try adding the second event eventManager.sendEvent(EVENTS[1]); @@ -220,7 +220,7 @@ describe('OdpEventManager', () => { batchSize: 10, // with batch size of 10... flushInterval: 250, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); for (let i = 0; i < 25; i += 1) { @@ -271,7 +271,7 @@ describe('OdpEventManager', () => { batchSize: 2, // batch size of 2 flushInterval: 100, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); // send 4 events @@ -294,7 +294,7 @@ describe('OdpEventManager', () => { batchSize: 2, // batches of 2 with... flushInterval: 100, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); // ...25 events should... @@ -387,7 +387,7 @@ describe('OdpEventManager', () => { apiManager: instance(mockApiManager), logger, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger }); + const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); const apiKey = 'testing-api-key'; const apiHost = 'https://some.other.example.com'; const segmentsToCheck = ['empty-cart', '1-item-cart']; From 32a0c1095849f3595e6a0a4da50755f9a3f2914b Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Tue, 11 Oct 2022 16:24:04 -0400 Subject: [PATCH 31/32] Code review changes --- .../lib/plugins/odp/odp_config.ts | 21 -- .../lib/plugins/odp/odp_event_dispatcher.ts | 299 ----------------- .../lib/plugins/odp/odp_event_manager.ts | 307 ++++++++++++++++-- .../optimizely-sdk/lib/utils/enums/index.ts | 9 - .../tests/odpEventManager.spec.ts | 140 +++----- 5 files changed, 329 insertions(+), 447 deletions(-) delete mode 100644 packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts index dc5fcdff6..215d3655c 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_config.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import { ODP_CONFIG_STATE } from '../../utils/enums'; - export class OdpConfig { /** * Host of ODP audience segments API. @@ -59,27 +57,10 @@ export class OdpConfig { return this._segmentsToCheck; } - /** - * Indicates whether ODP is integrated for the project - * @private - */ - private _odpServiceIntegrated = ODP_CONFIG_STATE.UNDETERMINED; - constructor(apiKey: string, apiHost: string, segmentsToCheck?: string[]) { this._apiKey = apiKey; this._apiHost = apiHost; this._segmentsToCheck = segmentsToCheck ?? []; - this._odpServiceIntegrated = OdpConfig.determineOdpServiceIntegration(this._apiKey, this.apiHost); - } - - /** - * Determine the value of Service Integration enum - * @param apiKey ODP API Key - * @param apiHost Server host of the API - * @private - */ - private static determineOdpServiceIntegration(apiKey: string, apiHost: string): ODP_CONFIG_STATE { - return apiKey && apiHost ? ODP_CONFIG_STATE.INTEGRATED : ODP_CONFIG_STATE.NOT_INTEGRATED; } /** @@ -90,8 +71,6 @@ export class OdpConfig { * @returns true if configuration was updated successfully */ public update(apiKey: string, apiHost: string, segmentsToCheck: string[]): boolean { - this._odpServiceIntegrated = OdpConfig.determineOdpServiceIntegration(apiKey, apiHost); - if (this._apiKey === apiKey && this._apiHost === apiHost && this._segmentsToCheck === segmentsToCheck) { return false; } else { diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts deleted file mode 100644 index 29a688368..000000000 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_dispatcher.ts +++ /dev/null @@ -1,299 +0,0 @@ -/** - * Copyright 2022, Optimizely - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { OdpEvent } from './odp_event'; -import { LogHandler, LogLevel } from '../../modules/logging'; -import { OdpConfig } from './odp_config'; -import { RestApiManager } from './rest_api_manager'; - -const MAX_RETRIES = 3; -const DEFAULT_BATCH_SIZE = 10; -const DEFAULT_FLUSH_INTERVAL_MSECS = 1000; - -/** - * Event dispatcher's execution states - */ -export enum STATE { - STOPPED, - RUNNING, - PROCESSING, -} - -/** - * Queue processor for dispatching events to the Optimizely Data Platform (ODP) - */ -export interface IOdpEventDispatcher { - updateSettings(odpConfig: OdpConfig): void; - - start(): void; - - enqueue(event: OdpEvent): void; - - stop(): Promise; -} - -/** - * Concrete implementation of a processor for dispatching events to the Optimizely Data Platform (ODP) - */ -export class OdpEventDispatcher implements IOdpEventDispatcher { - /** - * Current state of the event processor - */ - public state: STATE = STATE.STOPPED; - /** - * Queue for holding all events to be eventually dispatched - * @private - */ - private queue = new Array(); - /** - * Identifier of the currently running timeout so clearCurrentTimeout() can be called - * @private - */ - private timeoutId?: NodeJS.Timeout | number; - /** - * ODP configuration settings in used - * @private - */ - private odpConfig: OdpConfig; - /** - * REST API Manager used to send the events - * @private - */ - private readonly apiManager: RestApiManager; - /** - * Handler for recording execution logs - * @private - */ - private readonly logger: LogHandler; - /** - * Maximum queue size - * @private - */ - private readonly queueSize: number; - /** - * Maximum number of events to process at once - * @private - */ - private readonly batchSize: number; - /** - * Milliseconds between setTimeout() to process new batches - * @private - */ - private readonly flushInterval: number; - - public constructor( - { odpConfig, apiManager, logger, queueSize, batchSize, flushInterval }: { - odpConfig: OdpConfig, - apiManager: RestApiManager, - logger: LogHandler, - queueSize?: number, - batchSize?: number, - flushInterval?: number - }) { - this.odpConfig = odpConfig; - this.apiManager = apiManager; - this.logger = logger; - - // if `process` exists Node/server-side execution context otherwise Browser - this.queueSize = queueSize || (process ? 10000 : 100); - this.batchSize = batchSize || DEFAULT_BATCH_SIZE; - this.flushInterval = flushInterval || DEFAULT_FLUSH_INTERVAL_MSECS; - - this.state = STATE.STOPPED; - } - - /** - * Update the ODP configuration in use - * @param odpConfig New settings to apply - */ - public updateSettings(odpConfig: OdpConfig): void { - this.odpConfig = odpConfig; - } - - /** - * Begin processing any events in the queue - */ - public start(): void { - this.state = STATE.RUNNING; - - this.setNewTimeout(); - } - - /** - * Add a new event to the main queue - * @param event ODP Event to be queued - */ - public enqueue(event: OdpEvent): void { - if (this.state === STATE.STOPPED) { - this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.'); - return; - } - - if (!this.odpConfig.isReady()) { - this.logger.log(LogLevel.DEBUG, 'Unable to Process ODP Event. ODPConfig is not ready.'); - return; - } - - if (this.queue.length >= this.queueSize) { - this.logger.log(LogLevel.WARNING, `Failed to Process ODP Event. Event Queue full. queueSize = ${this.queueSize}.`); - return; - } - - this.queue.push(event); - - this.processQueue(); - } - - /** - * Process events in the main queue - * @param shouldFlush Flush all events regardless of available queue event count - * @private - */ - private async processQueue(shouldFlush = false): Promise { - if (this.state !== STATE.RUNNING) { - return; - } - - if (!this.isOdpConfigurationReady()) { - return; - } - - // Flush interval occurred & queue has items - if (shouldFlush && this.queueContainsItems()) { - // clear the queue completely - this.clearCurrentTimeout(); - - this.state = STATE.PROCESSING; - - while (this.queueContainsItems()) { - await this.makeAndSendBatch(); - } - - this.state = STATE.RUNNING; - - this.setNewTimeout(); - } - // Check if queue has a full batch available - else if (this.queueHasBatches()) { - this.clearCurrentTimeout(); - - this.state = STATE.PROCESSING; - - while (this.queueHasBatches()) { - await this.makeAndSendBatch(); - } - - this.state = STATE.RUNNING; - - this.setNewTimeout(); - } - } - - /** - * Clear the currently running timout - * @private - */ - private clearCurrentTimeout(): void { - clearTimeout(this.timeoutId); - this.timeoutId = undefined; - } - - /** - * Start a new timeout - * @private - */ - private setNewTimeout(): void { - if (this.timeoutId !== undefined) { - return; - } - this.timeoutId = setTimeout(() => this.processQueue(true), this.flushInterval); - } - - /** - * Make a batch and send it to ODP - * @private - */ - private async makeAndSendBatch(): Promise { - const batch = new Array(); - - for (let count = 0; count < this.batchSize; count += 1) { - const event = this.queue.shift(); - if (event) { - batch.push(event); - } else { - break; - } - } - - if (batch.length > 0) { - let shouldRetry: boolean; - let attemptNumber = 0; - do { - shouldRetry = await this.apiManager.sendEvents(this.odpConfig.apiKey, this.odpConfig.apiHost, batch); - attemptNumber += 1; - } while (shouldRetry && attemptNumber < MAX_RETRIES); - } - } - - /** - * Check if main queue has any full/even batches available - * @private - */ - private queueHasBatches(): boolean { - return this.queueContainsItems() && this.queue.length % this.batchSize === 0; - } - - /** - * Check if main queue has any items - * @private - */ - private queueContainsItems(): boolean { - return this.queue.length > 0; - } - - /** - * Check if the ODP Configuration is ready and log if not. - * Potentially clear queue if server-side - * @private - */ - private isOdpConfigurationReady(): boolean { - if (this.odpConfig.isReady()) { - return true; - } - - if (process) { - // if Node/server-side context, empty queue items before ready state - this.logger.log(LogLevel.WARNING, 'ODPConfig not ready. Discarding events in queue.'); - this.queue = new Array(); - } else { - // in Browser/client-side context, give debug message but leave events in queue - this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready. Leaving events in queue.'); - } - return false; - } - - /** - * Drain the queue sending all remaining events in batches then stop processing - */ - public async stop(): Promise { - this.logger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.'); - - await this.processQueue(true); - - this.state = STATE.STOPPED; - this.logger.log(LogLevel.DEBUG, `EventDispatcher stopped. Queue Count: ${this.queue.length}`); - } -} diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index c7e3a5ed7..09dbaa196 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -19,7 +19,22 @@ import { OdpEvent } from './odp_event'; import { uuid } from '../../utils/fns'; import { ODP_USER_KEY } from '../../utils/enums'; import { OdpConfig } from './odp_config'; -import { OdpEventDispatcher } from './odp_event_dispatcher'; +import { RestApiManager } from './rest_api_manager'; + +const MAX_RETRIES = 3; +const DEFAULT_BATCH_SIZE = 10; +const DEFAULT_FLUSH_INTERVAL_MSECS = 1000; +const DEFAULT_BROWSER_QUEUE_SIZE = 10; +const DEFAULT_SERVER_QUEUE_SIZE = 10000; + +/** + * Event dispatcher's execution states + */ +export enum STATE { + STOPPED, + RUNNING, + PROCESSING, +} /** * Manager for persisting events to the Optimizely Data Platform (ODP) @@ -29,34 +44,104 @@ export interface IOdpEventManager { start(): void; + stop(): Promise; + registerVuid(vuid: string): void; identifyUser(userId: string, vuid?: string): void; sendEvent(event: OdpEvent): void; - - signalStop(): Promise; } /** * Concrete implementation of a manager for persisting events to the Optimizely Data Platform */ export class OdpEventManager implements IOdpEventManager { - private readonly eventDispatcher: OdpEventDispatcher; + /** + * Current state of the event processor + */ + public state: STATE = STATE.STOPPED; + /** + * Queue for holding all events to be eventually dispatched + * @private + */ + private queue = new Array(); + /** + * Identifier of the currently running timeout so clearCurrentTimeout() can be called + * @private + */ + private timeoutId?: NodeJS.Timeout | number; + /** + * ODP configuration settings in used + * @private + */ + private odpConfig: OdpConfig; + /** + * REST API Manager used to send the events + * @private + */ + private readonly apiManager: RestApiManager; + /** + * Handler for recording execution logs + * @private + */ private readonly logger: LogHandler; + /** + * Maximum queue size + * @private + */ + private readonly queueSize: number; + /** + * Maximum number of events to process at once + * @private + */ + private readonly batchSize: number; + /** + * Milliseconds between setTimeout() to process new batches + * @private + */ + private readonly flushInterval: number; + /** + * Type of execution context eg node, js, react + * @private + */ private readonly clientEngine: string; + /** + * Version of the client being used + * @private + */ private readonly clientVersion: string; - public constructor({ eventDispatcher, logger, clientEngine, clientVersion }: { - eventDispatcher: OdpEventDispatcher, + public constructor({ + odpConfig, + apiManager, + logger, + clientEngine, + clientVersion, + queueSize, + batchSize, + flushInterval, + }: { + odpConfig: OdpConfig, + apiManager: RestApiManager, logger: LogHandler, clientEngine: string, clientVersion: string, + queueSize?: number, + batchSize?: number, + flushInterval?: number }) { - this.eventDispatcher = eventDispatcher; + this.odpConfig = odpConfig; + this.apiManager = apiManager; this.logger = logger; this.clientEngine = clientEngine; this.clientVersion = clientVersion; + + this.queueSize = queueSize || (process ? DEFAULT_SERVER_QUEUE_SIZE : DEFAULT_BROWSER_QUEUE_SIZE); + this.batchSize = batchSize || DEFAULT_BATCH_SIZE; + this.flushInterval = flushInterval || DEFAULT_FLUSH_INTERVAL_MSECS; + + this.state = STATE.STOPPED; } /** @@ -64,14 +149,28 @@ export class OdpEventManager implements IOdpEventManager { * @param odpConfig New configuration to apply */ public updateSettings(odpConfig: OdpConfig): void { - this.eventDispatcher.updateSettings(odpConfig); + this.odpConfig = odpConfig; } /** - * Start processing events in the dispatcher's queue + * Start processing events in the queue */ public start(): void { - this.eventDispatcher.start(); + this.state = STATE.RUNNING; + + this.setNewTimeout(); + } + + /** + * Drain the queue sending all remaining events in batches then stop processing + */ + public async stop(): Promise { + this.logger.log(LogLevel.DEBUG, 'Stop requested.'); + + await this.processQueue(true); + + this.state = STATE.STOPPED; + this.logger.log(LogLevel.DEBUG, 'Stopped. Queue Count: %s', this.queue.length); } /** @@ -107,31 +206,188 @@ export class OdpEventManager implements IOdpEventManager { * @param event ODP Event to forward */ public sendEvent(event: OdpEvent): void { - const foundInvalidDataInKeys = this.findKeysWithInvalidData(event.data); - if (foundInvalidDataInKeys.length > 0) { - this.logger.log(LogLevel.ERROR, `Event data found to be invalid.`); - this.logger.log(LogLevel.DEBUG, `Invalid event data keys (${foundInvalidDataInKeys.join(', ')})`); + if (this.invalidDataFound(event.data)) { + this.logger.log(LogLevel.ERROR, 'Event data found to be invalid.'); } else { event.data = this.augmentCommonData(event.data); - this.eventDispatcher.enqueue(event); + this.enqueue(event); } } + /** + * Add a new event to the main queue + * @param event ODP Event to be queued + * @private + */ + private enqueue(event: OdpEvent): void { + if (this.state === STATE.STOPPED) { + this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. ODPEventManager is not running.'); + return; + } + + if (!this.odpConfig.isReady()) { + this.logger.log(LogLevel.DEBUG, 'Unable to Process ODP Event. ODPConfig is not ready.'); + return; + } + + if (this.queue.length >= this.queueSize) { + this.logger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = %s.', this.queue.length); + return; + } + + this.queue.push(event); + + this.processQueue(); + } + + /** + * Process events in the main queue + * @param shouldFlush Flush all events regardless of available queue event count + * @private + */ + private processQueue(shouldFlush = false): void { + if (this.state !== STATE.RUNNING) { + return; + } + + if (!this.isOdpConfigurationReady()) { + return; + } + + // Flush interval occurred & queue has items + if (shouldFlush && this.queueContainsItems()) { + // clear the queue completely + this.clearCurrentTimeout(); + + this.state = STATE.PROCESSING; + + while (this.queueContainsItems()) { + this.makeAndSendBatch(); + } + + this.state = STATE.RUNNING; + + this.setNewTimeout(); + } + // Check if queue has a full batch available + else if (this.queueHasBatches()) { + this.clearCurrentTimeout(); + + this.state = STATE.PROCESSING; + + while (this.queueHasBatches()) { + this.makeAndSendBatch(); + } + + this.state = STATE.RUNNING; + + this.setNewTimeout(); + } + } + + /** + * Clear the currently running timout + * @private + */ + private clearCurrentTimeout(): void { + clearTimeout(this.timeoutId); + this.timeoutId = undefined; + } + + /** + * Start a new timeout + * @private + */ + private setNewTimeout(): void { + if (this.timeoutId !== undefined) { + return; + } + this.timeoutId = setTimeout(() => this.processQueue(true), this.flushInterval); + } + + /** + * Make a batch and send it to ODP + * @private + */ + private makeAndSendBatch(): void { + const batch = new Array(); + + for (let count = 0; count < this.batchSize; count += 1) { + const event = this.queue.shift(); + if (event) { + batch.push(event); + } else { + break; + } + } + + if (batch.length > 0) { + setTimeout(async () => { + let shouldRetry: boolean; + let attemptNumber = 0; + do { + shouldRetry = await this.apiManager.sendEvents(this.odpConfig.apiKey, this.odpConfig.apiHost, batch); + attemptNumber += 1; + } while (shouldRetry && attemptNumber < MAX_RETRIES); + }); + } + } + + /** + * Check if main queue has any full/even batches available + * @returns True if there are event batches available in the queue otherwise False + * @private + */ + private queueHasBatches(): boolean { + return this.queueContainsItems() && this.queue.length % this.batchSize === 0; + } + + /** + * Check if main queue has any items + * @returns True if there are any events in the queue otherwise False + * @private + */ + private queueContainsItems(): boolean { + return this.queue.length > 0; + } + + /** + * Check if the ODP Configuration is ready and log if not. + * Potentially clear queue if server-side + * @returns True if the ODP configuration is ready otherwise False + * @private + */ + private isOdpConfigurationReady(): boolean { + if (this.odpConfig.isReady()) { + return true; + } + + if (process) { + // if Node/server-side context, empty queue items before ready state + this.logger.log(LogLevel.WARNING, 'ODPConfig not ready. Discarding events in queue.'); + this.queue = new Array(); + } else { + // in Browser/client-side context, give debug message but leave events in queue + this.logger.log(LogLevel.DEBUG, 'ODPConfig not ready. Leaving events in queue.'); + } + return false; + } + /** * Validate event data value types * @param data Event data to be validated - * @returns Array of event data keys that were found to be invalid + * @returns True if an invalid type was found in the data otherwise False * @private */ - private findKeysWithInvalidData(data: Map): string[] { - const validTypes: string[] = ['string', 'number', 'boolean', 'bigint']; - const invalidKeys: string[] = []; - data.forEach((value, key) => { + private invalidDataFound(data: Map): boolean { + const validTypes: string[] = ['string', 'number', 'boolean']; + let foundInvalidValue = false; + data.forEach((value) => { if (!validTypes.includes(typeof value) && value !== null) { - invalidKeys.push(key); + foundInvalidValue = true; } }); - return invalidKeys; + return foundInvalidValue; } /** @@ -150,11 +406,4 @@ export class OdpEventManager implements IOdpEventManager { sourceData.forEach((value, key) => data.set(key, value)); return data; } - - /** - * Signal to event dispatcher to drain the event queue and stop - */ - public async signalStop(): Promise { - await this.eventDispatcher.stop(); - } } diff --git a/packages/optimizely-sdk/lib/utils/enums/index.ts b/packages/optimizely-sdk/lib/utils/enums/index.ts index 38ea49a0c..d00e65b66 100644 --- a/packages/optimizely-sdk/lib/utils/enums/index.ts +++ b/packages/optimizely-sdk/lib/utils/enums/index.ts @@ -304,12 +304,3 @@ export enum ODP_USER_KEY { VUID = 'vuid', FS_USER_ID = 'fs_user_id', } - -/** - * Possible states of ODP integration - */ -export enum ODP_CONFIG_STATE { - UNDETERMINED = 0, - INTEGRATED, - NOT_INTEGRATED = 2, -} diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index 7f22988aa..8c20378c4 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -15,13 +15,12 @@ */ import { OdpConfig } from '../lib/plugins/odp/odp_config'; -import { OdpEventManager } from '../lib/plugins/odp/odp_event_manager'; -import { anyString, anything, capture, instance, mock, resetCalls, verify, when } from 'ts-mockito'; +import { OdpEventManager, STATE } from '../lib/plugins/odp/odp_event_manager'; +import { anything, capture, instance, mock, resetCalls, verify, when } from 'ts-mockito'; import { RestApiManager } from '../lib/plugins/odp/rest_api_manager'; import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; import { RequestHandler } from '../lib/utils/http_request_handler/http'; -import { OdpEventDispatcher, STATE } from '../lib/plugins/odp/odp_event_dispatcher'; const API_KEY = 'test-api-key'; const API_HOST = 'https://odp.example.com'; @@ -109,12 +108,18 @@ const abortableRequest = (statusCode: number, body: string) => { describe('OdpEventManager', () => { let mockLogger: LogHandler; let mockApiManager: RestApiManager; + let odpConfig: OdpConfig; + let logger: LogHandler; + let apiManager: RestApiManager; beforeAll(() => { mockLogger = mock(); mockApiManager = mock(); + odpConfig = new OdpConfig(API_KEY, API_HOST, []); + logger = instance(mockLogger); + apiManager = instance(mockApiManager); }); beforeEach(() => { @@ -123,9 +128,9 @@ describe('OdpEventManager', () => { }); it('should log and discard events when event manager not running', () => { - const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, + }); // since we've not called start() then... eventManager.sendEvent(EVENTS[0]); @@ -135,16 +140,13 @@ describe('OdpEventManager', () => { }); it('should log and discard events when event manager config is not ready', () => { - const logger = instance(mockLogger); const mockOdpConfig = mock(); when(mockOdpConfig.isReady()).thenReturn(false); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig: instance(mockOdpConfig), - apiManager: instance(mockApiManager), - logger, + const odpConfig = instance(mockOdpConfig); + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, }); - eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher already in running state - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); + eventManager['state'] = STATE.RUNNING; // simulate running without calling start() eventManager.sendEvent(EVENTS[0]); @@ -152,9 +154,9 @@ describe('OdpEventManager', () => { }); it('should discard events with invalid data', () => { - const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, + }); // make an event with invalid data key-value entry const badEvent = new OdpEvent( 't3', @@ -168,34 +170,26 @@ describe('OdpEventManager', () => { eventManager.sendEvent(badEvent); verify(mockLogger.log(LogLevel.ERROR, 'Event data found to be invalid.')).once(); - verify(mockLogger.log(LogLevel.DEBUG, anyString())).once(); }); it('should log a max queue hit and discard ', () => { - const logger = instance(mockLogger); - const mockOdpConfig = mock(); - when(mockOdpConfig.isReady()).thenReturn(false); // set queue to maximum of 1 - const eventDispatcher = new OdpEventDispatcher({ - odpConfig: mockOdpConfig, - apiManager: instance(mockApiManager), - logger, - queueSize: 1, + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, queueSize: 1, // With max queue size set to 1... }); - eventDispatcher['state'] = STATE.RUNNING; // simulate dispatcher running - eventDispatcher['queue'].push(EVENTS[0]); // simulate event already in queue - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); + eventManager['state'] = STATE.RUNNING; + eventManager['queue'].push(EVENTS[0]); // simulate 1 event already in the queue then... - // try adding the second event + // ...try adding the second event eventManager.sendEvent(EVENTS[1]); - verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = 1.')).once(); + verify(mockLogger.log(LogLevel.WARNING, 'Failed to Process ODP Event. Event Queue full. queueSize = %s.', 1)).once(); }); it('should add additional information to each event', () => { - const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher({ odpConfig, apiManager: instance(mockApiManager), logger }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, + }); const processedEventData = PROCESSED_EVENTS[0].data; const eventData = eventManager['augmentCommonData'](EVENTS[0].data); @@ -211,16 +205,13 @@ describe('OdpEventManager', () => { }); it('should dispatch events in correct number of batches', async () => { - const logger = instance(mockLogger); when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig, - apiManager: instance(mockApiManager), - logger, + const apiManager = instance(mockApiManager); + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, batchSize: 10, // with batch size of 10... flushInterval: 250, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); for (let i = 0; i < 25; i += 1) { @@ -234,15 +225,9 @@ describe('OdpEventManager', () => { }); it('should dispatch events with correct payload', async () => { - const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig, - apiManager: instance(mockApiManager), - logger, - batchSize: 10, - flushInterval: 100, + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, batchSize: 10, flushInterval: 100, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); EVENTS.forEach(event => eventManager.sendEvent(event)); @@ -263,15 +248,12 @@ describe('OdpEventManager', () => { it('should retry failed events', async () => { // all events should fail ie shouldRetry = true when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(true); - const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig, - apiManager: instance(mockApiManager), - logger, + const apiManager = instance(mockApiManager); + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, batchSize: 2, // batch size of 2 flushInterval: 100, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); // send 4 events @@ -285,16 +267,13 @@ describe('OdpEventManager', () => { }); it('should flush all scheduled events before stopping', async () => { - const logger = instance(mockLogger); when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig, - apiManager: instance(mockApiManager), - logger, + const apiManager = instance(mockApiManager); + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, batchSize: 2, // batches of 2 with... flushInterval: 100, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); eventManager.start(); // ...25 events should... @@ -302,27 +281,19 @@ describe('OdpEventManager', () => { eventManager.sendEvent(makeEvent(i)); } await pause(300); - await eventManager.signalStop(); + await eventManager.stop(); - verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stop requested.')).once(); - // ...never exceed 14 - verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher draining queue without flush interval.')).atMost(14); - verify(mockLogger.log(LogLevel.DEBUG, 'EventDispatcher stopped. Queue Count: 0')).once(); + verify(mockLogger.log(LogLevel.DEBUG, 'Stop requested.')).once(); + verify(mockLogger.log(LogLevel.DEBUG, 'Stopped. Queue Count: %s', 0)).once(); }); it('should prepare correct payload for register VUID', async () => { const mockRequestHandler: RequestHandler = mock(); when(mockRequestHandler.makeRequest(anything(), anything(), anything(), anything())).thenReturn(abortableRequest(200, '')); - const logger = instance(mockLogger); const apiManager = new RestApiManager(instance(mockRequestHandler), logger); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig, - apiManager, - logger, - batchSize: 10, - flushInterval: 100, + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, batchSize: 10, flushInterval: 100, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; eventManager.start(); @@ -348,15 +319,10 @@ describe('OdpEventManager', () => { it('should prepare correct payload for identify user', async () => { const mockRequestHandler: RequestHandler = mock(); when(mockRequestHandler.makeRequest(anything(), anything(), anything(), anything())).thenReturn(abortableRequest(200, '')); - const logger = instance(mockLogger); const apiManager = new RestApiManager(instance(mockRequestHandler), logger); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig, - apiManager, - logger, - flushInterval: 100, + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, flushInterval: 100, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); const vuid = 'vuid_330e05cad15746d9af8a75b8d10'; const fsUserId = 'test-fs-user-id'; @@ -381,13 +347,9 @@ describe('OdpEventManager', () => { }); it('should apply updated ODP configuration when available', () => { - const logger = instance(mockLogger); - const eventDispatcher = new OdpEventDispatcher({ - odpConfig, - apiManager: instance(mockApiManager), - logger, + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, }); - const eventManager = new OdpEventManager({ eventDispatcher, logger, clientEngine, clientVersion }); const apiKey = 'testing-api-key'; const apiHost = 'https://some.other.example.com'; const segmentsToCheck = ['empty-cart', '1-item-cart']; @@ -395,9 +357,9 @@ describe('OdpEventManager', () => { eventManager.updateSettings(differentOdpConfig); - expect(eventManager['eventDispatcher']['odpConfig'].apiKey).toEqual(apiKey); - expect(eventManager['eventDispatcher']['odpConfig'].apiHost).toEqual(apiHost); - expect(eventManager['eventDispatcher']['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[0]); - expect(eventManager['eventDispatcher']['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[1]); + expect(eventManager['odpConfig'].apiKey).toEqual(apiKey); + expect(eventManager['odpConfig'].apiHost).toEqual(apiHost); + expect(eventManager['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[0]); + expect(eventManager['odpConfig'].segmentsToCheck).toContain(segmentsToCheck[1]); }); }); From 346da4ee762bc482c85bb8a905a6363d2a9e0fc7 Mon Sep 17 00:00:00 2001 From: Mike Chu Date: Wed, 12 Oct 2022 08:23:47 -0400 Subject: [PATCH 32/32] Fix setting timeout; add multiple flush test --- .../lib/plugins/odp/odp_event_manager.ts | 23 ++++++++----------- .../tests/odpEventManager.spec.ts | 16 ++++++++++++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts index 09dbaa196..766d5fa0e 100644 --- a/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts +++ b/packages/optimizely-sdk/lib/plugins/odp/odp_event_manager.ts @@ -24,7 +24,7 @@ import { RestApiManager } from './rest_api_manager'; const MAX_RETRIES = 3; const DEFAULT_BATCH_SIZE = 10; const DEFAULT_FLUSH_INTERVAL_MSECS = 1000; -const DEFAULT_BROWSER_QUEUE_SIZE = 10; +const DEFAULT_BROWSER_QUEUE_SIZE = 100; const DEFAULT_SERVER_QUEUE_SIZE = 10000; /** @@ -255,19 +255,15 @@ export class OdpEventManager implements IOdpEventManager { } // Flush interval occurred & queue has items - if (shouldFlush && this.queueContainsItems()) { + if (shouldFlush) { // clear the queue completely this.clearCurrentTimeout(); this.state = STATE.PROCESSING; while (this.queueContainsItems()) { - this.makeAndSendBatch(); + this.makeAndSend1Batch(); } - - this.state = STATE.RUNNING; - - this.setNewTimeout(); } // Check if queue has a full batch available else if (this.queueHasBatches()) { @@ -276,13 +272,12 @@ export class OdpEventManager implements IOdpEventManager { this.state = STATE.PROCESSING; while (this.queueHasBatches()) { - this.makeAndSendBatch(); + this.makeAndSend1Batch(); } - - this.state = STATE.RUNNING; - - this.setNewTimeout(); } + + this.state = STATE.RUNNING; + this.setNewTimeout(); } /** @@ -309,9 +304,10 @@ export class OdpEventManager implements IOdpEventManager { * Make a batch and send it to ODP * @private */ - private makeAndSendBatch(): void { + private makeAndSend1Batch(): void { const batch = new Array(); + // remove a batch from the queue for (let count = 0; count < this.batchSize; count += 1) { const event = this.queue.shift(); if (event) { @@ -322,6 +318,7 @@ export class OdpEventManager implements IOdpEventManager { } if (batch.length > 0) { + // put sending the event on another event loop setTimeout(async () => { let shouldRetry: boolean; let attemptNumber = 0; diff --git a/packages/optimizely-sdk/tests/odpEventManager.spec.ts b/packages/optimizely-sdk/tests/odpEventManager.spec.ts index 8c20378c4..12ee14a84 100644 --- a/packages/optimizely-sdk/tests/odpEventManager.spec.ts +++ b/packages/optimizely-sdk/tests/odpEventManager.spec.ts @@ -16,7 +16,7 @@ import { OdpConfig } from '../lib/plugins/odp/odp_config'; import { OdpEventManager, STATE } from '../lib/plugins/odp/odp_event_manager'; -import { anything, capture, instance, mock, resetCalls, verify, when } from 'ts-mockito'; +import { anything, capture, instance, mock, resetCalls, spy, verify, when } from 'ts-mockito'; import { RestApiManager } from '../lib/plugins/odp/rest_api_manager'; import { LogHandler, LogLevel } from '../lib/modules/logging'; import { OdpEvent } from '../lib/plugins/odp/odp_event'; @@ -204,6 +204,20 @@ describe('OdpEventManager', () => { expect(eventData.get('key-4')).toEqual(processedEventData.get('key-4')); }); + it('should attempt to flush an empty queue at flush intervals', async () => { + const eventManager = new OdpEventManager({ + odpConfig, apiManager, logger, clientEngine, clientVersion, + flushInterval: 100, + }); + const spiedEventManager = spy(eventManager); + + eventManager.start(); + // do not add events to the queue, but allow for... + await pause(400); // at least 3 flush intervals executions (giving a little longer) + + verify(spiedEventManager['processQueue'](anything())).atLeast(3); + }); + it('should dispatch events in correct number of batches', async () => { when(mockApiManager.sendEvents(anything(), anything(), anything())).thenResolve(false); const apiManager = instance(mockApiManager);