From d93ded6e4f54c984ad237375740c776c9ed322c2 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Tue, 4 Feb 2025 11:47:31 -0500 Subject: [PATCH 01/17] Add support for token-based usage metrics. Token measurement is broken down by modaliy, with separate counters for image, audio, etc. Tests are in version 6.*, so this change also includes bumping update_responses.sh --- .../vertexai/src/methods/count-tokens.test.ts | 25 +++++++++++++++++++ .../src/methods/generate-content.test.ts | 24 ++++++++++++++++++ packages/vertexai/src/types/enums.ts | 13 ++++++++++ packages/vertexai/src/types/responses.ts | 16 +++++++++++- scripts/update_vertexai_responses.sh | 2 +- 5 files changed, 78 insertions(+), 2 deletions(-) diff --git a/packages/vertexai/src/methods/count-tokens.test.ts b/packages/vertexai/src/methods/count-tokens.test.ts index fd4b99e1e00..a16024099e8 100644 --- a/packages/vertexai/src/methods/count-tokens.test.ts +++ b/packages/vertexai/src/methods/count-tokens.test.ts @@ -66,6 +66,31 @@ describe('countTokens()', () => { undefined ); }); + it('total tokens with modality details', async () => { + const mockResponse = getMockResponse('unary-success-detailed-token-response.json'); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const result = await countTokens( + fakeApiSettings, + 'model', + fakeRequestParams + ); + expect(result.totalTokens).to.equal(1837); + expect(result.totalBillableCharacters).to.equal(117); + expect(result.promptTokensDetails?.[0].modality).to.equal('IMAGE'); + expect(result.promptTokensDetails?.[0].tokenCount).to.equal(1806); + expect(makeRequestStub).to.be.calledWith( + 'model', + Task.COUNT_TOKENS, + fakeApiSettings, + false, + match((value: string) => { + return value.includes('contents'); + }), + undefined + ); + }); it('total tokens no billable characters', async () => { const mockResponse = getMockResponse( 'unary-success-no-billable-characters.json' diff --git a/packages/vertexai/src/methods/generate-content.test.ts b/packages/vertexai/src/methods/generate-content.test.ts index c5a1d9e1e91..bcd0f372e32 100644 --- a/packages/vertexai/src/methods/generate-content.test.ts +++ b/packages/vertexai/src/methods/generate-content.test.ts @@ -102,6 +102,30 @@ describe('generateContent()', () => { match.any ); }); + it('long response with token details', async () => { + const mockResponse = getMockResponse('unary-success-basic-response-long-usage-metadata.json'); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams + ); + expect(result.response.usageMetadata?.totalTokenCount).to.equal(1913); + expect(result.response.usageMetadata?.candidatesTokenCount).to.equal(76); + expect(result.response.usageMetadata?.promptTokensDetails?.[0].modality).to.equal('IMAGE'); + expect(result.response.usageMetadata?.promptTokensDetails?.[0].tokenCount).to.equal(1806); + expect(result.response.usageMetadata?.candidatesTokensDetails?.[0].modality).to.equal('TEXT'); + expect(result.response.usageMetadata?.candidatesTokensDetails?.[0].tokenCount).to.equal(76); + expect(makeRequestStub).to.be.calledWith( + 'model', + Task.GENERATE_CONTENT, + fakeApiSettings, + false, + match.any + ); + }); it('citations', async () => { const mockResponse = getMockResponse('unary-success-citations.json'); const makeRequestStub = stub(request, 'makeRequest').resolves( diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index 3e66bacc612..99439bd3821 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -137,3 +137,16 @@ export enum FunctionCallingMode { // not passing any function declarations. NONE = 'NONE' } + +/** + * Type of content. + * @public + */ +export enum Modality { + UNSPECIFIED = 'UNSPECIFIED', + TEXT = 'TEXT', + IMAGE = 'IMAGE', + VIDEO = 'VIDEO', + AUDIO = 'AUDIO', + DOCUMENT = 'DOCUMENT' +} diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index 83cd4366f12..fe6671232f7 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -21,7 +21,8 @@ import { FinishReason, HarmCategory, HarmProbability, - HarmSeverity + HarmSeverity, + Modality } from './enums'; /** @@ -83,6 +84,17 @@ export interface UsageMetadata { promptTokenCount: number; candidatesTokenCount: number; totalTokenCount: number; + promptTokensDetails?: ModalityTokenCount[]; + candidatesTokensDetails?: ModalityTokenCount[]; +} + +/** + * The number of tokens used by a given content type. + * @public + */ +export interface ModalityTokenCount { + modality: Modality; + tokenCount: number; } /** @@ -213,4 +225,6 @@ export interface CountTokensResponse { * from the request. */ totalBillableCharacters?: number; + + promptTokensDetails?: ModalityTokenCount; } diff --git a/scripts/update_vertexai_responses.sh b/scripts/update_vertexai_responses.sh index 101eac90d9f..20b9082861e 100755 --- a/scripts/update_vertexai_responses.sh +++ b/scripts/update_vertexai_responses.sh @@ -17,7 +17,7 @@ # This script replaces mock response files for Vertex AI unit tests with a fresh # clone of the shared repository of Vertex AI test data. -RESPONSES_VERSION='v5.*' # The major version of mock responses to use +RESPONSES_VERSION='v6.*' # The major version of mock responses to use REPO_NAME="vertexai-sdk-test-data" REPO_LINK="https://github.com/FirebaseExtended/$REPO_NAME.git" From 41cfa7b9dad35383d3d85e3f7bf222bd3f13eed1 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Tue, 4 Feb 2025 13:55:41 -0500 Subject: [PATCH 02/17] Use right name for unspecified enum --- packages/vertexai/src/types/enums.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index 99439bd3821..68008f868c9 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -143,7 +143,7 @@ export enum FunctionCallingMode { * @public */ export enum Modality { - UNSPECIFIED = 'UNSPECIFIED', + MODALITY_UNSPECIFIED = 'MODALITY_UNSPECIFIED', TEXT = 'TEXT', IMAGE = 'IMAGE', VIDEO = 'VIDEO', From bb950ff17894dda77879fc4abfe71930d480ef7b Mon Sep 17 00:00:00 2001 From: rlazo Date: Tue, 4 Feb 2025 19:07:45 +0000 Subject: [PATCH 03/17] Update API reports --- common/api-review/vertexai.api.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 041bc62451f..892f7b92087 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -92,6 +92,8 @@ export interface CountTokensRequest { // @public export interface CountTokensResponse { + // (undocumented) + promptTokensDetails?: ModalityTokenCount; totalBillableCharacters?: number; totalTokens: number; } @@ -447,6 +449,30 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public +export enum Modality { + // (undocumented) + AUDIO = "AUDIO", + // (undocumented) + DOCUMENT = "DOCUMENT", + // (undocumented) + IMAGE = "IMAGE", + // (undocumented) + MODALITY_UNSPECIFIED = "MODALITY_UNSPECIFIED", + // (undocumented) + TEXT = "TEXT", + // (undocumented) + VIDEO = "VIDEO" +} + +// @public +export interface ModalityTokenCount { + // (undocumented) + modality: Modality; + // (undocumented) + tokenCount: number; +} + // @public export interface ModelParams extends BaseParams { // (undocumented) @@ -682,8 +708,12 @@ export interface UsageMetadata { // (undocumented) candidatesTokenCount: number; // (undocumented) + candidatesTokensDetails?: ModalityTokenCount[]; + // (undocumented) promptTokenCount: number; // (undocumented) + promptTokensDetails?: ModalityTokenCount[]; + // (undocumented) totalTokenCount: number; } From ca74f88c51af7ce843ed1534546f52287e63b1d5 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Tue, 4 Feb 2025 14:58:54 -0500 Subject: [PATCH 04/17] Make promptTokensDetails an array --- packages/vertexai/src/types/responses.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index fe6671232f7..8ccb4557402 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -226,5 +226,5 @@ export interface CountTokensResponse { */ totalBillableCharacters?: number; - promptTokensDetails?: ModalityTokenCount; + promptTokensDetails?: ModalityTokenCount[]; } From accb059d5de7097ae8b0a8c0efd11763a1129727 Mon Sep 17 00:00:00 2001 From: rlazo Date: Tue, 4 Feb 2025 20:10:15 +0000 Subject: [PATCH 05/17] Update API reports --- common/api-review/vertexai.api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 892f7b92087..c21d61e5ea3 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -93,7 +93,7 @@ export interface CountTokensRequest { // @public export interface CountTokensResponse { // (undocumented) - promptTokensDetails?: ModalityTokenCount; + promptTokensDetails?: ModalityTokenCount[]; totalBillableCharacters?: number; totalTokens: number; } From 61efb239051ab64969a230dcf664bc3480fba05b Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Fri, 7 Feb 2025 23:43:34 -0500 Subject: [PATCH 06/17] Add missing documentation --- packages/vertexai/src/types/enums.ts | 8 +++++++- packages/vertexai/src/types/responses.ts | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index 68008f868c9..56e9cd09e0f 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -139,14 +139,20 @@ export enum FunctionCallingMode { } /** - * Type of content. + * Content part modality. * @public */ export enum Modality { + // Unspecified modality. MODALITY_UNSPECIFIED = 'MODALITY_UNSPECIFIED', + // Plain text. TEXT = 'TEXT', + // Image. IMAGE = 'IMAGE', + // Video. VIDEO = 'VIDEO', + // Audio. AUDIO = 'AUDIO', + // Document, e.g. PDF. DOCUMENT = 'DOCUMENT' } diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index 8ccb4557402..ccfaf4ffe0c 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -89,11 +89,14 @@ export interface UsageMetadata { } /** - * The number of tokens used by a given content type. + * Represents token counting info for a single modality. + * * @public */ export interface ModalityTokenCount { + /** The modality associated with this token count. */ modality: Modality; + /** The number of tokens counted. */ tokenCount: number; } From 40914d7b464ab8e5f99d79dc9cb5c6fee2b9305d Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Fri, 7 Feb 2025 23:49:03 -0500 Subject: [PATCH 07/17] Add changeset entry --- .changeset/chilled-tips-judge.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/chilled-tips-judge.md diff --git a/.changeset/chilled-tips-judge.md b/.changeset/chilled-tips-judge.md new file mode 100644 index 00000000000..15d58806a9e --- /dev/null +++ b/.changeset/chilled-tips-judge.md @@ -0,0 +1,6 @@ +--- +'@firebase/vertexai': minor +'firebase': minor +--- + +Added support for modality-based token count. From 9b8a59fc3eabb714f04370854bc9c889675c6109 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Fri, 7 Feb 2025 23:52:03 -0500 Subject: [PATCH 08/17] Fix formatting errors --- .../vertexai/src/methods/count-tokens.test.ts | 4 +++- .../src/methods/generate-content.test.ts | 20 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/vertexai/src/methods/count-tokens.test.ts b/packages/vertexai/src/methods/count-tokens.test.ts index a16024099e8..2032e884fb4 100644 --- a/packages/vertexai/src/methods/count-tokens.test.ts +++ b/packages/vertexai/src/methods/count-tokens.test.ts @@ -67,7 +67,9 @@ describe('countTokens()', () => { ); }); it('total tokens with modality details', async () => { - const mockResponse = getMockResponse('unary-success-detailed-token-response.json'); + const mockResponse = getMockResponse( + 'unary-success-detailed-token-response.json' + ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); diff --git a/packages/vertexai/src/methods/generate-content.test.ts b/packages/vertexai/src/methods/generate-content.test.ts index bcd0f372e32..001fe12c9c8 100644 --- a/packages/vertexai/src/methods/generate-content.test.ts +++ b/packages/vertexai/src/methods/generate-content.test.ts @@ -103,7 +103,9 @@ describe('generateContent()', () => { ); }); it('long response with token details', async () => { - const mockResponse = getMockResponse('unary-success-basic-response-long-usage-metadata.json'); + const mockResponse = getMockResponse( + 'unary-success-basic-response-long-usage-metadata.json' + ); const makeRequestStub = stub(request, 'makeRequest').resolves( mockResponse as Response ); @@ -114,10 +116,18 @@ describe('generateContent()', () => { ); expect(result.response.usageMetadata?.totalTokenCount).to.equal(1913); expect(result.response.usageMetadata?.candidatesTokenCount).to.equal(76); - expect(result.response.usageMetadata?.promptTokensDetails?.[0].modality).to.equal('IMAGE'); - expect(result.response.usageMetadata?.promptTokensDetails?.[0].tokenCount).to.equal(1806); - expect(result.response.usageMetadata?.candidatesTokensDetails?.[0].modality).to.equal('TEXT'); - expect(result.response.usageMetadata?.candidatesTokensDetails?.[0].tokenCount).to.equal(76); + expect( + result.response.usageMetadata?.promptTokensDetails?.[0].modality + ).to.equal('IMAGE'); + expect( + result.response.usageMetadata?.promptTokensDetails?.[0].tokenCount + ).to.equal(1806); + expect( + result.response.usageMetadata?.candidatesTokensDetails?.[0].modality + ).to.equal('TEXT'); + expect( + result.response.usageMetadata?.candidatesTokensDetails?.[0].tokenCount + ).to.equal(76); expect(makeRequestStub).to.be.calledWith( 'model', Task.GENERATE_CONTENT, From 94bb8269253fd483e50b66ce325b5e1d75e869c8 Mon Sep 17 00:00:00 2001 From: rlazo Date: Sat, 8 Feb 2025 05:00:15 +0000 Subject: [PATCH 09/17] Update API reports --- common/api-review/vertexai.api.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index c21d61e5ea3..fa288e4914b 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -467,9 +467,7 @@ export enum Modality { // @public export interface ModalityTokenCount { - // (undocumented) modality: Modality; - // (undocumented) tokenCount: number; } From 7b5193a1c50ea2f096da0f7033c79d5a55e71bea Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Fri, 14 Feb 2025 10:05:09 -0500 Subject: [PATCH 10/17] Add missing refdoc --- packages/vertexai/src/types/responses.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/vertexai/src/types/responses.ts b/packages/vertexai/src/types/responses.ts index ccfaf4ffe0c..5685ed68ad6 100644 --- a/packages/vertexai/src/types/responses.ts +++ b/packages/vertexai/src/types/responses.ts @@ -228,6 +228,8 @@ export interface CountTokensResponse { * from the request. */ totalBillableCharacters?: number; - + /** + * The breakdown, by modality, of how many tokens are consumed by the prompt. + */ promptTokensDetails?: ModalityTokenCount[]; } From 6aaf68cab927c1975eda1a343648b97351149b70 Mon Sep 17 00:00:00 2001 From: rlazo Date: Fri, 14 Feb 2025 15:13:52 +0000 Subject: [PATCH 11/17] Update API reports --- common/api-review/vertexai.api.md | 1 - 1 file changed, 1 deletion(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index fa288e4914b..7098cfcb96a 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -92,7 +92,6 @@ export interface CountTokensRequest { // @public export interface CountTokensResponse { - // (undocumented) promptTokensDetails?: ModalityTokenCount[]; totalBillableCharacters?: number; totalTokens: number; From ad619f599d4780372da5a67c9b83824f2ebcadc8 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Tue, 18 Feb 2025 16:59:41 -0500 Subject: [PATCH 12/17] Use correct formatting for Modality documentation. --- packages/vertexai/src/types/enums.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index 56e9cd09e0f..b838987d4b4 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -143,16 +143,28 @@ export enum FunctionCallingMode { * @public */ export enum Modality { - // Unspecified modality. + /** + * Unspecified modality. + */ MODALITY_UNSPECIFIED = 'MODALITY_UNSPECIFIED', - // Plain text. + /** + * Plain text. + */ TEXT = 'TEXT', - // Image. + /** + * Image. + */ IMAGE = 'IMAGE', - // Video. + /** + * Video. + */ VIDEO = 'VIDEO', - // Audio. + /** + * Audio. + */ AUDIO = 'AUDIO', - // Document, e.g. PDF. + /** + * Document, e.g. PDF. + */ DOCUMENT = 'DOCUMENT' } From db47a2f038edae0bec8d94981d88cc18c25dea28 Mon Sep 17 00:00:00 2001 From: rlazo Date: Tue, 18 Feb 2025 22:08:34 +0000 Subject: [PATCH 13/17] Update API reports --- common/api-review/vertexai.api.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index 7098cfcb96a..76e5dccbced 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -450,17 +450,11 @@ export class IntegerSchema extends Schema { // @public export enum Modality { - // (undocumented) AUDIO = "AUDIO", - // (undocumented) DOCUMENT = "DOCUMENT", - // (undocumented) IMAGE = "IMAGE", - // (undocumented) MODALITY_UNSPECIFIED = "MODALITY_UNSPECIFIED", - // (undocumented) TEXT = "TEXT", - // (undocumented) VIDEO = "VIDEO" } From 2efd3faf8c3f058d2ed188624dcbc5cd410056fe Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Wed, 19 Feb 2025 18:25:18 -0500 Subject: [PATCH 14/17] Update docs --- docs-devsite/_toc.yaml | 2 ++ docs-devsite/vertexai.counttokensresponse.md | 11 +++++++++ docs-devsite/vertexai.md | 23 +++++++++++++++++++ docs-devsite/vertexai.usagemetadata.md | 18 +++++++++++++++ .../rules-unit-testing/api-extractor.json | 6 ++--- 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index bf0318389ba..c7bd631a2fb 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -540,6 +540,8 @@ toc: path: /docs/reference/js/vertexai.inlinedatapart.md - title: IntegerSchema path: /docs/reference/js/vertexai.integerschema.md + - title: ModalityTokenCount + path: /docs/reference/js/vertexai.modalitytokencount.md - title: ModelParams path: /docs/reference/js/vertexai.modelparams.md - title: NumberSchema diff --git a/docs-devsite/vertexai.counttokensresponse.md b/docs-devsite/vertexai.counttokensresponse.md index 2978f9a45bb..d67cc99fab2 100644 --- a/docs-devsite/vertexai.counttokensresponse.md +++ b/docs-devsite/vertexai.counttokensresponse.md @@ -22,9 +22,20 @@ export interface CountTokensResponse | Property | Type | Description | | --- | --- | --- | +| [promptTokensDetails](./vertexai.counttokensresponse.md#counttokensresponseprompttokensdetails) | [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface)\[\] | The breakdown, by modality, of how many tokens are consumed by the prompt. | | [totalBillableCharacters](./vertexai.counttokensresponse.md#counttokensresponsetotalbillablecharacters) | number | The total number of billable characters counted across all instances from the request. | | [totalTokens](./vertexai.counttokensresponse.md#counttokensresponsetotaltokens) | number | The total number of tokens counted across all instances from the request. | +## CountTokensResponse.promptTokensDetails + +The breakdown, by modality, of how many tokens are consumed by the prompt. + +Signature: + +```typescript +promptTokensDetails?: ModalityTokenCount[]; +``` + ## CountTokensResponse.totalBillableCharacters The total number of billable characters counted across all instances from the request. diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index d9e26eabc5d..b60a8d790b9 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -48,6 +48,7 @@ The Vertex AI in Firebase Web SDK. | [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | | [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | | [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | +| [Modality](./vertexai.md#modality) | Content part modality. | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -84,6 +85,7 @@ The Vertex AI in Firebase Web SDK. | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | +| [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | | [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | @@ -363,6 +365,27 @@ export declare enum HarmSeverity | HARM\_SEVERITY\_MEDIUM | "HARM_SEVERITY_MEDIUM" | | | HARM\_SEVERITY\_NEGLIGIBLE | "HARM_SEVERITY_NEGLIGIBLE" | | +## Modality + +Content part modality. + +Signature: + +```typescript +export declare enum Modality +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| AUDIO | "AUDIO" | Audio. | +| DOCUMENT | "DOCUMENT" | Document, e.g. PDF. | +| IMAGE | "IMAGE" | Image. | +| MODALITY\_UNSPECIFIED | "MODALITY_UNSPECIFIED" | Unspecified modality. | +| TEXT | "TEXT" | Plain text. | +| VIDEO | "VIDEO" | Video. | + ## SchemaType Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) diff --git a/docs-devsite/vertexai.usagemetadata.md b/docs-devsite/vertexai.usagemetadata.md index d254f34335f..5f886dd29f2 100644 --- a/docs-devsite/vertexai.usagemetadata.md +++ b/docs-devsite/vertexai.usagemetadata.md @@ -23,7 +23,9 @@ export interface UsageMetadata | Property | Type | Description | | --- | --- | --- | | [candidatesTokenCount](./vertexai.usagemetadata.md#usagemetadatacandidatestokencount) | number | | +| [candidatesTokensDetails](./vertexai.usagemetadata.md#usagemetadatacandidatestokensdetails) | [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface)\[\] | | | [promptTokenCount](./vertexai.usagemetadata.md#usagemetadataprompttokencount) | number | | +| [promptTokensDetails](./vertexai.usagemetadata.md#usagemetadataprompttokensdetails) | [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface)\[\] | | | [totalTokenCount](./vertexai.usagemetadata.md#usagemetadatatotaltokencount) | number | | ## UsageMetadata.candidatesTokenCount @@ -34,6 +36,14 @@ export interface UsageMetadata candidatesTokenCount: number; ``` +## UsageMetadata.candidatesTokensDetails + +Signature: + +```typescript +candidatesTokensDetails?: ModalityTokenCount[]; +``` + ## UsageMetadata.promptTokenCount Signature: @@ -42,6 +52,14 @@ candidatesTokenCount: number; promptTokenCount: number; ``` +## UsageMetadata.promptTokensDetails + +Signature: + +```typescript +promptTokensDetails?: ModalityTokenCount[]; +``` + ## UsageMetadata.totalTokenCount Signature: diff --git a/packages/rules-unit-testing/api-extractor.json b/packages/rules-unit-testing/api-extractor.json index 12ef61e843d..50acb5157ae 100644 --- a/packages/rules-unit-testing/api-extractor.json +++ b/packages/rules-unit-testing/api-extractor.json @@ -1,5 +1,5 @@ { "extends": "../../config/api-extractor.json", - // Point it to your entry point d.ts file. - "mainEntryPointFilePath": "/dist/rules-unit-testing/index.d.ts" -} \ No newline at end of file + // Point it to your entry point d.ts file. + "mainEntryPointFilePath": "/dist/index.d.ts" +} From 421ab29bee0a94a9dd0b97d95d05444fb2a73955 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Date: Mon, 24 Feb 2025 11:02:33 -0500 Subject: [PATCH 15/17] Update packages/vertexai/src/types/enums.ts Co-authored-by: rachelsaunders <52258509+rachelsaunders@users.noreply.github.com> --- packages/vertexai/src/types/enums.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index b838987d4b4..4a7d95c660c 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -164,7 +164,7 @@ export enum Modality { */ AUDIO = 'AUDIO', /** - * Document, e.g. PDF. + * Document (for example, PDF). */ DOCUMENT = 'DOCUMENT' } From 813f682b59b9f33e500c9af8823aabef36f6bf34 Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Mon, 24 Feb 2025 11:33:24 -0500 Subject: [PATCH 16/17] Update md files --- docs-devsite/vertexai.md | 553 ++++++++++++++++++++ docs-devsite/vertexai.modalitytokencount.md | 46 ++ 2 files changed, 599 insertions(+) create mode 100644 docs-devsite/vertexai.modalitytokencount.md diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index e69de29bb2d..d174bef7bcf 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -0,0 +1,553 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# vertexai package +The Vertex AI in Firebase Web SDK. + +## Functions + +| Function | Description | +| --- | --- | +| function(app, ...) | +| [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | +| function(vertexAI, ...) | +| [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | (Public Preview) Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen.Only Imagen 3 models (named imagen-3.0-*) are supported. | + +## Classes + +| Class | Description | +| --- | --- | +| [ArraySchema](./vertexai.arrayschema.md#arrayschema_class) | Schema class for "array" types. The items param should refer to the type of item that can be a member of the array. | +| [BooleanSchema](./vertexai.booleanschema.md#booleanschema_class) | Schema class for "boolean" types. | +| [ChatSession](./vertexai.chatsession.md#chatsession_class) | ChatSession class that enables sending chat messages and stores history of sent and received messages so far. | +| [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) | Class for generative model APIs. | +| [ImagenImageFormat](./vertexai.imagenimageformat.md#imagenimageformat_class) | (Public Preview) Defines the image format for images generated by Imagen.Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface). | +| [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) | (Public Preview) Class for Imagen model APIs.This class provides methods for generating images using the Imagen model. | +| [IntegerSchema](./vertexai.integerschema.md#integerschema_class) | Schema class for "integer" types. | +| [NumberSchema](./vertexai.numberschema.md#numberschema_class) | Schema class for "number" types. | +| [ObjectSchema](./vertexai.objectschema.md#objectschema_class) | Schema class for "object" types. The properties param must be a map of Schema objects. | +| [Schema](./vertexai.schema.md#schema_class) | Parent class encompassing all Schema types, with static methods that allow building specific Schema types. This class can be converted with JSON.stringify() into a JSON string accepted by Vertex AI REST endpoints. (This string conversion is automatically done when calling SDK methods.) | +| [StringSchema](./vertexai.stringschema.md#stringschema_class) | Schema class for "string" types. Can be used with or without enum values. | +| [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) | Error class for the Vertex AI in Firebase SDK. | +| [VertexAIModel](./vertexai.vertexaimodel.md#vertexaimodel_class) | Base class for Vertex AI in Firebase model APIs. | + +## Enumerations + +| Enumeration | Description | +| --- | --- | +| [BlockReason](./vertexai.md#blockreason) | Reason that a prompt was blocked. | +| [FinishReason](./vertexai.md#finishreason) | Reason that a candidate finished. | +| [FunctionCallingMode](./vertexai.md#functioncallingmode) | | +| [HarmBlockMethod](./vertexai.md#harmblockmethod) | | +| [HarmBlockThreshold](./vertexai.md#harmblockthreshold) | Threshold above which a prompt or candidate will be blocked. | +| [HarmCategory](./vertexai.md#harmcategory) | Harm categories that would cause prompts or candidates to be blocked. | +| [HarmProbability](./vertexai.md#harmprobability) | Probability that a prompt or candidate matches a harm category. | +| [HarmSeverity](./vertexai.md#harmseverity) | Harm severity levels. | +| [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface).See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | +| [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | +| [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | +| [Modality](./vertexai.md#modality) | Content part modality. | +| [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | +| [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | + +## Interfaces + +| Interface | Description | +| --- | --- | +| [BaseParams](./vertexai.baseparams.md#baseparams_interface) | Base parameters for a number of methods. | +| [Citation](./vertexai.citation.md#citation_interface) | A single citation. | +| [CitationMetadata](./vertexai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface). | +| [Content](./vertexai.content.md#content_interface) | Content type for both prompts and response candidates. | +| [CountTokensRequest](./vertexai.counttokensrequest.md#counttokensrequest_interface) | Params for calling [GenerativeModel.countTokens()](./vertexai.generativemodel.md#generativemodelcounttokens) | +| [CountTokensResponse](./vertexai.counttokensresponse.md#counttokensresponse_interface) | Response from calling [GenerativeModel.countTokens()](./vertexai.generativemodel.md#generativemodelcounttokens). | +| [CustomErrorData](./vertexai.customerrordata.md#customerrordata_interface) | Details object that contains data originating from a bad HTTP response. | +| [Date\_2](./vertexai.date_2.md#date_2_interface) | Protobuf google.type.Date | +| [EnhancedGenerateContentResponse](./vertexai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponse_interface) | Response object wrapped with helper methods. | +| [ErrorDetails](./vertexai.errordetails.md#errordetails_interface) | Details object that may be included in an error response. | +| [FileData](./vertexai.filedata.md#filedata_interface) | Data pointing to a file uploaded on Google Cloud Storage. | +| [FileDataPart](./vertexai.filedatapart.md#filedatapart_interface) | Content part interface if the part represents [FileData](./vertexai.filedata.md#filedata_interface) | +| [FunctionCall](./vertexai.functioncall.md#functioncall_interface) | A predicted [FunctionCall](./vertexai.functioncall.md#functioncall_interface) returned from the model that contains a string representing the [FunctionDeclaration.name](./vertexai.functiondeclaration.md#functiondeclarationname) and a structured JSON object containing the parameters and their values. | +| [FunctionCallingConfig](./vertexai.functioncallingconfig.md#functioncallingconfig_interface) | | +| [FunctionCallPart](./vertexai.functioncallpart.md#functioncallpart_interface) | Content part interface if the part represents a [FunctionCall](./vertexai.functioncall.md#functioncall_interface). | +| [FunctionDeclaration](./vertexai.functiondeclaration.md#functiondeclaration_interface) | Structured representation of a function declaration as defined by the [OpenAPI 3.0 specification](https://spec.openapis.org/oas/v3.0.3). Included in this declaration are the function name and parameters. This FunctionDeclaration is a representation of a block of code that can be used as a Tool by the model and executed by the client. | +| [FunctionDeclarationsTool](./vertexai.functiondeclarationstool.md#functiondeclarationstool_interface) | A FunctionDeclarationsTool is a piece of code that enables the system to interact with external systems to perform an action, or set of actions, outside of knowledge and scope of the model. | +| [FunctionResponse](./vertexai.functionresponse.md#functionresponse_interface) | The result output from a [FunctionCall](./vertexai.functioncall.md#functioncall_interface) that contains a string representing the [FunctionDeclaration.name](./vertexai.functiondeclaration.md#functiondeclarationname) and a structured JSON object containing any output from the function is used as context to the model. This should contain the result of a [FunctionCall](./vertexai.functioncall.md#functioncall_interface) made based on model prediction. | +| [FunctionResponsePart](./vertexai.functionresponsepart.md#functionresponsepart_interface) | Content part interface if the part represents [FunctionResponse](./vertexai.functionresponse.md#functionresponse_interface). | +| [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | A candidate returned as part of a [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface). | +| [GenerateContentRequest](./vertexai.generatecontentrequest.md#generatecontentrequest_interface) | Request sent through [GenerativeModel.generateContent()](./vertexai.generativemodel.md#generativemodelgeneratecontent) | +| [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface) | Individual response from [GenerativeModel.generateContent()](./vertexai.generativemodel.md#generativemodelgeneratecontent) and [GenerativeModel.generateContentStream()](./vertexai.generativemodel.md#generativemodelgeneratecontentstream). generateContentStream() will return one in each chunk until the stream is done. | +| [GenerateContentResult](./vertexai.generatecontentresult.md#generatecontentresult_interface) | Result object returned from [GenerativeModel.generateContent()](./vertexai.generativemodel.md#generativemodelgeneratecontent) call. | +| [GenerateContentStreamResult](./vertexai.generatecontentstreamresult.md#generatecontentstreamresult_interface) | Result object returned from [GenerativeModel.generateContentStream()](./vertexai.generativemodel.md#generativemodelgeneratecontentstream) call. Iterate over stream to get chunks as they come in and/or use the response promise to get the aggregated response when the stream is done. | +| [GenerationConfig](./vertexai.generationconfig.md#generationconfig_interface) | Config options for content-related requests | +| [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | +| [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | +| [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | +| [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | +| [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | +| [ImagenInlineImage](./vertexai.imageninlineimage.md#imageninlineimage_interface) | (Public Preview) An image generated by Imagen, represented as inline data. | +| [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | (Public Preview) Parameters for configuring an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class). | +| [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | +| [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | +| [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | +| [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | +| [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | +| [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | +| [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | +| [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | +| [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | +| [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | +| [SchemaInterface](./vertexai.schemainterface.md#schemainterface_interface) | Interface for [Schema](./vertexai.schema.md#schema_class) class. | +| [SchemaParams](./vertexai.schemaparams.md#schemaparams_interface) | Params passed to [Schema](./vertexai.schema.md#schema_class) static methods to create specific [Schema](./vertexai.schema.md#schema_class) classes. | +| [SchemaRequest](./vertexai.schemarequest.md#schemarequest_interface) | Final format for [Schema](./vertexai.schema.md#schema_class) params passed to backend requests. | +| [SchemaShared](./vertexai.schemashared.md#schemashared_interface) | Basic [Schema](./vertexai.schema.md#schema_class) properties shared across several Schema-related types. | +| [Segment](./vertexai.segment.md#segment_interface) | | +| [StartChatParams](./vertexai.startchatparams.md#startchatparams_interface) | Params for [GenerativeModel.startChat()](./vertexai.generativemodel.md#generativemodelstartchat). | +| [TextPart](./vertexai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. | +| [ToolConfig](./vertexai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | +| [UsageMetadata](./vertexai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./vertexai.generatecontentresponse.md#generatecontentresponse_interface). | +| [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| [VertexAIOptions](./vertexai.vertexaioptions.md#vertexaioptions_interface) | Options when initializing the Vertex AI in Firebase SDK. | +| [VideoMetadata](./vertexai.videometadata.md#videometadata_interface) | Describes the input video content. | +| [WebAttribution](./vertexai.webattribution.md#webattribution_interface) | | + +## Variables + +| Variable | Description | +| --- | --- | +| [POSSIBLE\_ROLES](./vertexai.md#possible_roles) | Possible roles. | + +## Type Aliases + +| Type Alias | Description | +| --- | --- | +| [Part](./vertexai.md#part) | Content part - includes text, image/video, or function call/response part types. | +| [Role](./vertexai.md#role) | Role is the producer of the content. | +| [Tool](./vertexai.md#tool) | Defines a tool that model can call to access external knowledge. | +| [TypedSchema](./vertexai.md#typedschema) | A type that includes all specific Schema types. | + +## function(app, ...) + +### getVertexAI(app, options) {:#getvertexai_04094cf} + +Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. + +Signature: + +```typescript +export declare function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions): VertexAI; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| app | [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) | The [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) to use. | +| options | [VertexAIOptions](./vertexai.vertexaioptions.md#vertexaioptions_interface) | | + +Returns: + +[VertexAI](./vertexai.vertexai.md#vertexai_interface) + +## function(vertexAI, ...) + +### getGenerativeModel(vertexAI, modelParams, requestOptions) {:#getgenerativemodel_e3037c9} + +Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. + +Signature: + +```typescript +export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | +| modelParams | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | + +Returns: + +[GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) + +### getImagenModel(vertexAI, modelParams, requestOptions) {:#getimagenmodel_812c375} + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen. + +Only Imagen 3 models (named `imagen-3.0-*`) are supported. + +Signature: + +```typescript +export declare function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | An instance of the Vertex AI in Firebase SDK. | +| modelParams | [ImagenModelParams](./vertexai.imagenmodelparams.md#imagenmodelparams_interface) | Parameters to use when making Imagen requests. | +| requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Additional options to use when making requests. | + +Returns: + +[ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) + +#### Exceptions + +If the `apiKey` or `projectId` fields are missing in your Firebase config. + +## POSSIBLE\_ROLES + +Possible roles. + +Signature: + +```typescript +POSSIBLE_ROLES: readonly ["user", "model", "function", "system"] +``` + +## Part + +Content part - includes text, image/video, or function call/response part types. + +Signature: + +```typescript +export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +``` + +## Role + +Role is the producer of the content. + +Signature: + +```typescript +export type Role = (typeof POSSIBLE_ROLES)[number]; +``` + +## Tool + +Defines a tool that model can call to access external knowledge. + +Signature: + +```typescript +export declare type Tool = FunctionDeclarationsTool; +``` + +## TypedSchema + +A type that includes all specific Schema types. + +Signature: + +```typescript +export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema; +``` + +## BlockReason + +Reason that a prompt was blocked. + +Signature: + +```typescript +export declare enum BlockReason +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| OTHER | "OTHER" | | +| SAFETY | "SAFETY" | | + +## FinishReason + +Reason that a candidate finished. + +Signature: + +```typescript +export declare enum FinishReason +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| MAX\_TOKENS | "MAX_TOKENS" | | +| OTHER | "OTHER" | | +| RECITATION | "RECITATION" | | +| SAFETY | "SAFETY" | | +| STOP | "STOP" | | + +## FunctionCallingMode + + +Signature: + +```typescript +export declare enum FunctionCallingMode +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ANY | "ANY" | | +| AUTO | "AUTO" | | +| NONE | "NONE" | | + +## HarmBlockMethod + + +Signature: + +```typescript +export declare enum HarmBlockMethod +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| PROBABILITY | "PROBABILITY" | | +| SEVERITY | "SEVERITY" | | + +## HarmBlockThreshold + +Threshold above which a prompt or candidate will be blocked. + +Signature: + +```typescript +export declare enum HarmBlockThreshold +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| BLOCK\_LOW\_AND\_ABOVE | "BLOCK_LOW_AND_ABOVE" | | +| BLOCK\_MEDIUM\_AND\_ABOVE | "BLOCK_MEDIUM_AND_ABOVE" | | +| BLOCK\_NONE | "BLOCK_NONE" | | +| BLOCK\_ONLY\_HIGH | "BLOCK_ONLY_HIGH" | | + +## HarmCategory + +Harm categories that would cause prompts or candidates to be blocked. + +Signature: + +```typescript +export declare enum HarmCategory +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| HARM\_CATEGORY\_DANGEROUS\_CONTENT | "HARM_CATEGORY_DANGEROUS_CONTENT" | | +| HARM\_CATEGORY\_HARASSMENT | "HARM_CATEGORY_HARASSMENT" | | +| HARM\_CATEGORY\_HATE\_SPEECH | "HARM_CATEGORY_HATE_SPEECH" | | +| HARM\_CATEGORY\_SEXUALLY\_EXPLICIT | "HARM_CATEGORY_SEXUALLY_EXPLICIT" | | + +## HarmProbability + +Probability that a prompt or candidate matches a harm category. + +Signature: + +```typescript +export declare enum HarmProbability +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| HIGH | "HIGH" | | +| LOW | "LOW" | | +| MEDIUM | "MEDIUM" | | +| NEGLIGIBLE | "NEGLIGIBLE" | | + +## HarmSeverity + +Harm severity levels. + +Signature: + +```typescript +export declare enum HarmSeverity +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| HARM\_SEVERITY\_HIGH | "HARM_SEVERITY_HIGH" | | +| HARM\_SEVERITY\_LOW | "HARM_SEVERITY_LOW" | | +| HARM\_SEVERITY\_MEDIUM | "HARM_SEVERITY_MEDIUM" | | +| HARM\_SEVERITY\_NEGLIGIBLE | "HARM_SEVERITY_NEGLIGIBLE" | | + +## ImagenAspectRatio + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +Aspect ratios for Imagen images. + +To specify an aspect ratio for generated images, set the `aspectRatio` property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface). + +See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. + +Signature: + +```typescript +export declare enum ImagenAspectRatio +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| LANDSCAPE\_16x9 | "16:9" | (Public Preview) Landscape (16:9) aspect ratio. | +| LANDSCAPE\_3x4 | "3:4" | (Public Preview) Landscape (3:4) aspect ratio. | +| PORTRAIT\_4x3 | "4:3" | (Public Preview) Portrait (4:3) aspect ratio. | +| PORTRAIT\_9x16 | "9:16" | (Public Preview) Portrait (9:16) aspect ratio. | +| SQUARE | "1:1" | (Public Preview) Square (1:1) aspect ratio. | + +## ImagenPersonFilterLevel + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling whether generation of images containing people or faces is allowed. + +See the personGeneration documentation for more details. + +Signature: + +```typescript +export declare enum ImagenPersonFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ALLOW\_ADULT | "allow_adult" | (Public Preview) Allow generation of images containing adults only; images of children are filtered out.Generation of images containing people or faces may require your use case to be reviewed and approved by Cloud support; see the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) for more details. | +| ALLOW\_ALL | "allow_all" | (Public Preview) Allow generation of images containing adults only; images of children are filtered out.Generation of images containing people or faces may require your use case to be reviewed and approved by Cloud support; see the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#person-face-gen) for more details. | +| BLOCK\_ALL | "dont_allow" | (Public Preview) Disallow generation of images containing people or faces; images of people are filtered out. | + +## ImagenSafetyFilterLevel + +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + +A filter level controlling how aggressively to filter sensitive content. + +Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, `violence`, `sexual`, `derogatory`, and `toxic`). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. + +Signature: + +```typescript +export declare enum ImagenSafetyFilterLevel +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| BLOCK\_LOW\_AND\_ABOVE | "block_low_and_above" | (Public Preview) The most aggressive filtering level; most strict blocking. | +| BLOCK\_MEDIUM\_AND\_ABOVE | "block_medium_and_above" | (Public Preview) Blocks some sensitive prompts and responses. | +| BLOCK\_NONE | "block_none" | (Public Preview) The least aggressive filtering level; blocks very few sensitive prompts and responses.Access to this feature is restricted and may require your case to be reviewed and approved by Cloud support. | +| BLOCK\_ONLY\_HIGH | "block_only_high" | (Public Preview) Blocks few sensitive prompts and responses. | + +## Modality + +Content part modality. + +Signature: + +```typescript +export declare enum Modality +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| AUDIO | "AUDIO" | Audio. | +| DOCUMENT | "DOCUMENT" | Document (for example, PDF). | +| IMAGE | "IMAGE" | Image. | +| MODALITY\_UNSPECIFIED | "MODALITY_UNSPECIFIED" | Unspecified modality. | +| TEXT | "TEXT" | Plain text. | +| VIDEO | "VIDEO" | Video. | + +## SchemaType + +Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) + +Signature: + +```typescript +export declare enum SchemaType +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ARRAY | "array" | Array type. | +| BOOLEAN | "boolean" | Boolean type. | +| INTEGER | "integer" | Integer type. | +| NUMBER | "number" | Number type. | +| OBJECT | "object" | Object type. | +| STRING | "string" | String type. | + +## VertexAIErrorCode + +Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. + +Signature: + +```typescript +export declare const enum VertexAIErrorCode +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| API\_NOT\_ENABLED | "api-not-enabled" | An error due to the Firebase API not being enabled in the Console. | +| ERROR | "error" | A generic error occurred. | +| FETCH\_ERROR | "fetch-error" | An error occurred while performing a fetch. | +| INVALID\_CONTENT | "invalid-content" | An error associated with a Content object. | +| INVALID\_SCHEMA | "invalid-schema" | An error due to invalid Schema input. | +| NO\_API\_KEY | "no-api-key" | An error occurred due to a missing Firebase API key. | +| NO\_MODEL | "no-model" | An error occurred due to a model name not being specified during initialization. | +| NO\_PROJECT\_ID | "no-project-id" | An error occurred due to a missing project ID. | +| PARSE\_FAILED | "parse-failed" | An error occurred while parsing. | +| REQUEST\_ERROR | "request-error" | An error occurred in a request. | +| RESPONSE\_ERROR | "response-error" | An error occurred in a response. | + diff --git a/docs-devsite/vertexai.modalitytokencount.md b/docs-devsite/vertexai.modalitytokencount.md new file mode 100644 index 00000000000..d710b51fba6 --- /dev/null +++ b/docs-devsite/vertexai.modalitytokencount.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ModalityTokenCount interface +Represents token counting info for a single modality. + +Signature: + +```typescript +export interface ModalityTokenCount +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [modality](./vertexai.modalitytokencount.md#modalitytokencountmodality) | [Modality](./vertexai.md#modality) | The modality associated with this token count. | +| [tokenCount](./vertexai.modalitytokencount.md#modalitytokencounttokencount) | number | The number of tokens counted. | + +## ModalityTokenCount.modality + +The modality associated with this token count. + +Signature: + +```typescript +modality: Modality; +``` + +## ModalityTokenCount.tokenCount + +The number of tokens counted. + +Signature: + +```typescript +tokenCount: number; +``` From 6f90b2ec5f3b2f7a128b96ebb359207f83a40cad Mon Sep 17 00:00:00 2001 From: Rodrigo Lazo Paz Date: Mon, 24 Feb 2025 11:37:27 -0500 Subject: [PATCH 17/17] Revert changes to api-extractor.json --- packages/rules-unit-testing/api-extractor.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/rules-unit-testing/api-extractor.json b/packages/rules-unit-testing/api-extractor.json index 50acb5157ae..12ef61e843d 100644 --- a/packages/rules-unit-testing/api-extractor.json +++ b/packages/rules-unit-testing/api-extractor.json @@ -1,5 +1,5 @@ { "extends": "../../config/api-extractor.json", - // Point it to your entry point d.ts file. - "mainEntryPointFilePath": "/dist/index.d.ts" -} + // Point it to your entry point d.ts file. + "mainEntryPointFilePath": "/dist/rules-unit-testing/index.d.ts" +} \ No newline at end of file