From ddeafbba9152c6bdbdb57c7791566caec005d647 Mon Sep 17 00:00:00 2001 From: Kevin Freeman Date: Tue, 29 Jul 2025 10:26:08 -0400 Subject: [PATCH 1/2] Added provider and model to ai tracker --- .../__tests__/LDAIConfigTrackerImpl.test.ts | 164 +++++++++++------- packages/sdk/server-ai/src/LDAIClientImpl.ts | 2 + .../server-ai/src/LDAIConfigTrackerImpl.ts | 6 +- 3 files changed, 113 insertions(+), 59 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index 9dd28574c..4e76cda95 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -15,6 +15,8 @@ const testContext: LDContext = { kind: 'user', key: 'test-user' }; const configKey = 'test-config'; const variationKey = 'v1'; const version = 1; +const modelName = 'test-model'; +const providerName = 'test-provider'; beforeEach(() => { jest.clearAllMocks(); @@ -26,6 +28,8 @@ it('tracks duration', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackDuration(1000); @@ -33,7 +37,7 @@ it('tracks duration', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -44,6 +48,8 @@ it('tracks duration of async function', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -54,7 +60,7 @@ it('tracks duration of async function', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -65,6 +71,8 @@ it('tracks time to first token', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackTimeToFirstToken(1000); @@ -72,7 +80,7 @@ it('tracks time to first token', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:ttf', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -83,6 +91,8 @@ it('tracks positive feedback', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackFeedback({ kind: LDFeedbackKind.Positive }); @@ -90,7 +100,7 @@ it('tracks positive feedback', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:feedback:user:positive', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); @@ -101,6 +111,8 @@ it('tracks negative feedback', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackFeedback({ kind: LDFeedbackKind.Negative }); @@ -108,7 +120,7 @@ it('tracks negative feedback', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:feedback:user:negative', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); @@ -119,6 +131,8 @@ it('tracks success', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackSuccess(); @@ -126,14 +140,14 @@ it('tracks success', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); @@ -144,6 +158,8 @@ it('tracks OpenAI usage', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -163,21 +179,21 @@ it('tracks OpenAI usage', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -191,21 +207,21 @@ it('tracks OpenAI usage', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -216,6 +232,8 @@ it('tracks error when OpenAI metrics function throws', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -230,21 +248,21 @@ it('tracks error when OpenAI metrics function throws', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -262,6 +280,8 @@ it('tracks Bedrock conversation with successful response', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -284,14 +304,14 @@ it('tracks Bedrock conversation with successful response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -305,28 +325,28 @@ it('tracks Bedrock conversation with successful response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 500, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -337,6 +357,8 @@ it('tracks Bedrock conversation with error response', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -350,14 +372,14 @@ it('tracks Bedrock conversation with error response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -376,6 +398,8 @@ describe('Vercel AI SDK generateText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -395,21 +419,21 @@ describe('Vercel AI SDK generateText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -423,21 +447,21 @@ describe('Vercel AI SDK generateText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -448,6 +472,8 @@ describe('Vercel AI SDK generateText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -462,21 +488,21 @@ describe('Vercel AI SDK generateText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -496,6 +522,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -528,21 +556,21 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -556,21 +584,21 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -581,6 +609,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -595,21 +625,21 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -627,6 +657,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -640,21 +672,21 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -672,6 +704,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -685,21 +719,21 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -717,6 +751,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -731,21 +767,21 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -771,6 +807,8 @@ it('tracks tokens', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -787,21 +825,21 @@ it('tracks tokens', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -812,6 +850,8 @@ it('only tracks non-zero token counts', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -831,7 +871,7 @@ it('only tracks non-zero token counts', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 50, ); @@ -849,6 +889,8 @@ it('returns empty summary when no metrics tracked', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -863,6 +905,8 @@ it('summarizes tracked metrics', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -897,6 +941,8 @@ it('tracks duration when async function throws', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -911,7 +957,7 @@ it('tracks duration when async function throws', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -922,6 +968,8 @@ it('tracks error', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackError(); @@ -929,14 +977,14 @@ it('tracks error', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 331e74800..99822af47 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -77,6 +77,8 @@ export class LDAIClientImpl implements LDAIClient { value._ldMeta?.variationKey ?? '', // eslint-disable-next-line no-underscore-dangle value._ldMeta?.version ?? 1, + value.model?.name ?? '', + value.provider?.name ?? '', context, ); diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts index 73c4bdfbd..166dc9957 100644 --- a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts @@ -19,14 +19,18 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { private _configKey: string, private _variationKey: string, private _version: number, + private _modelName: string, + private _providerName: string, private _context: LDContext, ) {} - private _getTrackData(): { variationKey: string; configKey: string; version: number } { + private _getTrackData(): { variationKey: string; configKey: string; version: number; modelName: string; providerName: string } { return { variationKey: this._variationKey, configKey: this._configKey, version: this._version, + modelName: this._modelName, + providerName: this._providerName, }; } From 9fd0ab0ad0b35e9bc5422374e97b97b828461ba3 Mon Sep 17 00:00:00 2001 From: Kevin Freeman Date: Tue, 29 Jul 2025 10:34:29 -0400 Subject: [PATCH 2/2] fixed lint --- packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts index 166dc9957..95e1c5aa4 100644 --- a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts @@ -24,7 +24,13 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { private _context: LDContext, ) {} - private _getTrackData(): { variationKey: string; configKey: string; version: number; modelName: string; providerName: string } { + private _getTrackData(): { + variationKey: string; + configKey: string; + version: number; + modelName: string; + providerName: string; + } { return { variationKey: this._variationKey, configKey: this._configKey,