diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index 6b292e1ba..8d197f4ce 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -15,6 +15,8 @@ const testContext: LDContext = { kind: 'user', key: 'test-user' }; const configKey = 'test-config'; const variationKey = 'v1'; const version = 1; +const modelName = 'test-model'; +const providerName = 'test-provider'; beforeEach(() => { jest.clearAllMocks(); @@ -26,6 +28,8 @@ it('tracks duration', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackDuration(1000); @@ -33,7 +37,7 @@ it('tracks duration', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -44,6 +48,8 @@ it('tracks duration of async function', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -54,7 +60,7 @@ it('tracks duration of async function', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -65,6 +71,8 @@ it('tracks time to first token', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackTimeToFirstToken(1000); @@ -72,7 +80,7 @@ it('tracks time to first token', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:ttf', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -83,6 +91,8 @@ it('tracks positive feedback', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackFeedback({ kind: LDFeedbackKind.Positive }); @@ -90,7 +100,7 @@ it('tracks positive feedback', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:feedback:user:positive', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); @@ -101,6 +111,8 @@ it('tracks negative feedback', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackFeedback({ kind: LDFeedbackKind.Negative }); @@ -108,7 +120,7 @@ it('tracks negative feedback', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:feedback:user:negative', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); @@ -119,6 +131,8 @@ it('tracks success', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackSuccess(); @@ -126,7 +140,7 @@ it('tracks success', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); @@ -137,6 +151,8 @@ it('tracks OpenAI usage', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -156,14 +172,14 @@ it('tracks OpenAI usage', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -177,21 +193,21 @@ it('tracks OpenAI usage', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -202,6 +218,8 @@ it('tracks error when OpenAI metrics function throws', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -216,14 +234,14 @@ it('tracks error when OpenAI metrics function throws', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -241,6 +259,8 @@ it('tracks Bedrock conversation with successful response', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -263,7 +283,7 @@ it('tracks Bedrock conversation with successful response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -277,28 +297,28 @@ it('tracks Bedrock conversation with successful response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 500, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -309,6 +329,8 @@ it('tracks Bedrock conversation with error response', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -323,7 +345,7 @@ it('tracks Bedrock conversation with error response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -342,6 +364,8 @@ describe('Vercel AI SDK generateText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -361,14 +385,14 @@ describe('Vercel AI SDK generateText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -382,21 +406,21 @@ describe('Vercel AI SDK generateText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -407,6 +431,8 @@ describe('Vercel AI SDK generateText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -421,14 +447,14 @@ describe('Vercel AI SDK generateText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -448,6 +474,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -480,14 +508,14 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -501,21 +529,21 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -526,6 +554,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -540,14 +570,14 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -565,6 +595,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -578,14 +610,14 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -603,6 +635,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -616,14 +650,14 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -641,6 +675,8 @@ describe('Vercel AI SDK streamText', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -655,14 +691,14 @@ describe('Vercel AI SDK streamText', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:success', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); @@ -688,6 +724,8 @@ it('tracks tokens', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -704,21 +742,21 @@ it('tracks tokens', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, COMPLETION_TOKENS, ); }); @@ -729,6 +767,8 @@ it('only tracks non-zero token counts', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -748,7 +788,7 @@ it('only tracks non-zero token counts', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 50, ); @@ -766,6 +806,8 @@ it('returns empty summary when no metrics tracked', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -780,6 +822,8 @@ it('summarizes tracked metrics', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); @@ -814,6 +858,8 @@ it('tracks duration when async function throws', async () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); @@ -828,7 +874,7 @@ it('tracks duration when async function throws', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1000, ); }); @@ -839,6 +885,8 @@ it('tracks error', () => { configKey, variationKey, version, + modelName, + providerName, testContext, ); tracker.trackError(); @@ -846,7 +894,7 @@ it('tracks error', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey, version }, + { configKey, variationKey, version, modelName, providerName }, 1, ); }); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 331e74800..99822af47 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -77,6 +77,8 @@ export class LDAIClientImpl implements LDAIClient { value._ldMeta?.variationKey ?? '', // eslint-disable-next-line no-underscore-dangle value._ldMeta?.version ?? 1, + value.model?.name ?? '', + value.provider?.name ?? '', context, ); diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts index 6dd21bb51..ec8876b4c 100644 --- a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts @@ -19,14 +19,24 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { private _configKey: string, private _variationKey: string, private _version: number, + private _modelName: string, + private _providerName: string, private _context: LDContext, ) {} - private _getTrackData(): { variationKey: string; configKey: string; version: number } { + private _getTrackData(): { + variationKey: string; + configKey: string; + version: number; + modelName: string; + providerName: string; + } { return { variationKey: this._variationKey, configKey: this._configKey, version: this._version, + modelName: this._modelName, + providerName: this._providerName, }; }