VinF Hybrid Inference: support structured output (#9009)

Author: erikeldridge

common/api-review/vertexai.api.md

@@ -553,8 +553,7 @@ export enum HarmSeverity {
 export interface HybridParams {
     inCloudParams?: ModelParams;
     mode: InferenceMode;
-    // Warning: (ae-forgotten-export) The symbol "LanguageModelCreateOptions" needs to be exported by the entry point index.d.ts
-    onDeviceParams?: LanguageModelCreateOptions;
+    onDeviceParams?: OnDeviceParams;
 }
 
 // @beta
@@ -718,6 +717,18 @@ export interface ObjectSchemaInterface extends SchemaInterface {
     type: SchemaType.OBJECT;
 }
 
+// @public
+export interface OnDeviceParams {
+    // Warning: (ae-forgotten-export) The symbol "LanguageModelCreateOptions" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    createOptions?: LanguageModelCreateOptions;
+    // Warning: (ae-forgotten-export) The symbol "LanguageModelPromptOptions" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    promptOptions?: LanguageModelPromptOptions;
+}
+
 // @public
 export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart;
 

docs-devsite/_toc.yaml

@@ -562,6 +562,8 @@ toc:
     path: /docs/reference/js/vertexai.objectschema.md
   - title: ObjectSchemaInterface
     path: /docs/reference/js/vertexai.objectschemainterface.md
+  - title: OnDeviceParams
+    path: /docs/reference/js/vertexai.ondeviceparams.md
   - title: PromptFeedback
     path: /docs/reference/js/vertexai.promptfeedback.md
   - title: RequestOptions

docs-devsite/vertexai.hybridparams.md

@@ -24,7 +24,7 @@ export interface HybridParams
 |  --- | --- | --- |
 |  [inCloudParams](./vertexai.hybridparams.md#hybridparamsincloudparams) | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Optional. Specifies advanced params for in-cloud inference. |
 |  [mode](./vertexai.hybridparams.md#hybridparamsmode) | [InferenceMode](./vertexai.md#inferencemode) | Specifies on-device or in-cloud inference. Defaults to prefer on-device. |
-|  [onDeviceParams](./vertexai.hybridparams.md#hybridparamsondeviceparams) | LanguageModelCreateOptions | Optional. Specifies advanced params for on-device inference. |
+|  [onDeviceParams](./vertexai.hybridparams.md#hybridparamsondeviceparams) | [OnDeviceParams](./vertexai.ondeviceparams.md#ondeviceparams_interface) | Optional. Specifies advanced params for on-device inference. |
 
 ## HybridParams.inCloudParams
 
@@ -53,5 +53,5 @@ Optional. Specifies advanced params for on-device inference.
 <b>Signature:</b>
 
 ```typescript
-onDeviceParams?: LanguageModelCreateOptions;
+onDeviceParams?: OnDeviceParams;
 ```

docs-devsite/vertexai.md

@@ -108,6 +108,7 @@ The Firebase AI Web SDK.
 |  [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. |
 |  [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_c63f46a)<!-- -->. |
 |  [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. |
+|  [OnDeviceParams](./vertexai.ondeviceparams.md#ondeviceparams_interface) | Encapsulates configuration for on-device inference. |
 |  [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with <code>blockReason</code> and the relevant <code>safetyRatings</code>. |
 |  [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_c63f46a)<!-- -->. |
 |  [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) |  |

docs-devsite/vertexai.ondeviceparams.md

@@ -0,0 +1,42 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# OnDeviceParams interface
+Encapsulates configuration for on-device inference.
+
+<b>Signature:</b>
+
+```typescript
+export interface OnDeviceParams 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [createOptions](./vertexai.ondeviceparams.md#ondeviceparamscreateoptions) | LanguageModelCreateOptions |  |
+|  [promptOptions](./vertexai.ondeviceparams.md#ondeviceparamspromptoptions) | LanguageModelPromptOptions |  |
+
+## OnDeviceParams.createOptions
+
+<b>Signature:</b>
+
+```typescript
+createOptions?: LanguageModelCreateOptions;
+```
+
+## OnDeviceParams.promptOptions
+
+<b>Signature:</b>
+
+```typescript
+promptOptions?: LanguageModelPromptOptions;
+```

e2e/sample-apps/modular.js

@@ -58,7 +58,7 @@ import {
   onValue,
   off
 } from 'firebase/database';
-import { getGenerativeModel, getVertexAI } from 'firebase/vertexai';
+import { getGenerativeModel, getVertexAI, Schema } from 'firebase/vertexai';
 import { getDataConnect, DataConnect } from 'firebase/data-connect';
 
 /**
@@ -313,23 +313,43 @@ function callPerformance(app) {
 async function callVertexAI(app) {
   console.log('[VERTEXAI] start');
   const vertexAI = getVertexAI(app);
-  const model = getGenerativeModel(vertexAI, {
-    mode: 'prefer_on_device'
+
+  const jsonSchema = Schema.object({
+    properties: {
+      characters: Schema.array({
+        items: Schema.object({
+          properties: {
+            name: Schema.string(),
+            accessory: Schema.string(),
+            age: Schema.number(),
+            species: Schema.string()
+          },
+          optionalProperties: ['accessory']
+        })
+      })
+    }
   });
-  const singleResult = await model.generateContent([
-    { text: 'describe this 20 x 20 px image in two words' },
-    {
-      inlineData: {
-        mimeType: 'image/heic',
-        data: 'AAAAGGZ0eXBoZWljAAAAAGhlaWNtaWYxAAAB7G1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAHBpY3QAAAAAAAAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAADnBpdG0AAAAAAAEAAAA4aWluZgAAAAAAAgAAABVpbmZlAgAAAAABAABodmMxAAAAABVpbmZlAgAAAQACAABFeGlmAAAAABppcmVmAAAAAAAAAA5jZHNjAAIAAQABAAABD2lwcnAAAADtaXBjbwAAABNjb2xybmNseAACAAIABoAAAAAMY2xsaQDLAEAAAAAUaXNwZQAAAAAAAAAUAAAADgAAAChjbGFwAAAAFAAAAAEAAAANAAAAAQAAAAAAAAAB/8AAAACAAAAAAAAJaXJvdAAAAAAQcGl4aQAAAAADCAgIAAAAcWh2Y0MBA3AAAACwAAAAAAAe8AD8/fj4AAALA6AAAQAXQAEMAf//A3AAAAMAsAAAAwAAAwAecCShAAEAI0IBAQNwAAADALAAAAMAAAMAHqAUIEHAjw1iHuRZVNwICBgCogABAAlEAcBhcshAUyQAAAAaaXBtYQAAAAAAAAABAAEHgQIDhIUGhwAAACxpbG9jAAAAAEQAAAIAAQAAAAEAAAJsAAABDAACAAAAAQAAAhQAAABYAAAAAW1kYXQAAAAAAAABdAAAAAZFeGlmAABNTQAqAAAACAAEARIAAwAAAAEAAQAAARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAAAAAAAAAAEgAAAABAAAASAAAAAEAAAEIKAGvoR8wDimTiRYUbALiHkU3ZdZ8DXAcSrRB9GARtVQHvnCE0LEyBGAyb5P4eYr6JAK5UxNX10WNlARq3ZpcGeVD+Xom6LodYasuZKKtDHCz/xnswOtC/ksZzVKhtWQqGvkXcsJnLYqWevNkacnccQ95jbHJBg9nXub69jAAN3xhNOXxjGSxaG9QvES5R7sYICEojRjLF5OB5K3v+okQAwfgWpz/u21ayideOgOZQLAyBkKOv7ymLNCagiPWTlHAuy/3qR1Q7m2ERFaxKIAbLSkIVO/P8m8+anKxhzhC//L8NMAUoF+Sf3aEH9O41fwLc+PlcbrDrjgY2EboD3cn9DyN32Rum2Ym'
+
+  const model = getGenerativeModel(vertexAI, {
+    // mode: 'prefer_on_device',
+    mode: 'only_in_cloud',
+    inCloudParams: {
+      generationConfig: {
+        responseMimeType: 'application/json',
+        responseSchema: jsonSchema
+      }
+    },
+    onDeviceParams: {
+      promptOptions: {
+        responseConstraint: jsonSchema
       }
     }
-  ]);
-  console.log(`Generated text: ${singleResult.response.text()}`);
-  const chat = model.startChat();
-  let chatResult = await chat.sendMessage('describe red in two words');
-  chatResult = await chat.sendMessage('describe blue');
-  console.log('Chat history:', await chat.getHistory());
+  });
+
+  const singleResult = await model.generateContent(
+    "For use in a children's card game, generate 10 animal-based characters."
+  );
+  console.log(`Generated text:`, JSON.parse(singleResult.response.text()));
   console.log(`[VERTEXAI] end`);
 }
 

e2e/webpack.config.js

@@ -88,7 +88,7 @@ module.exports = [
     stats: {
       colors: true
     },
-    devtool: 'source-map',
+    devtool: 'eval-source-map',
     devServer: {
       static: './build'
     }