feat(ui)/clip skip

invokeai/frontend/web/public/locales/en.json

@@ -770,6 +770,7 @@
         "allPrompts": "All Prompts",
         "cfgScale": "CFG scale",
         "cfgRescaleMultiplier": "$t(parameters.cfgRescaleMultiplier)",
+        "clipSkip": "$t(parameters.clipSkip)",
         "createdBy": "Created By",
         "generationMode": "Generation Mode",
         "guidance": "Guidance",
@@ -1290,6 +1291,7 @@
         "remixImage": "Remix Image",
         "usePrompt": "Use Prompt",
         "useSeed": "Use Seed",
+        "useClipSkip": "Use CLIP Skip",
         "width": "Width",
         "gaussianBlur": "Gaussian Blur",
         "boxBlur": "Box Blur",

invokeai/frontend/web/src/features/controlLayers/store/paramsSlice.ts

@@ -107,14 +107,7 @@ const slice = createSlice({
         return;
       }
 
-      // Clamp CLIP skip layer count to the bounds of the new model
-      if (model.base === 'sdxl') {
-        // We don't support user-defined CLIP skip for SDXL because it doesn't do anything useful
-        state.clipSkip = 0;
-      } else {
-        const { maxClip } = CLIP_SKIP_MAP[model.base];
-        state.clipSkip = clamp(state.clipSkip, 0, maxClip);
-      }
+      applyClipSkip(state, model, state.clipSkip);
     },
     vaeSelected: (state, action: PayloadAction<ParameterVAEModel | null>) => {
       // null is a valid VAE!
@@ -170,7 +163,7 @@ const slice = createSlice({
       state.vaePrecision = action.payload;
     },
     setClipSkip: (state, action: PayloadAction<number>) => {
-      state.clipSkip = action.payload;
+      applyClipSkip(state, state.model, action.payload);
     },
     shouldUseCpuNoiseChanged: (state, action: PayloadAction<boolean>) => {
       state.shouldUseCpuNoise = action.payload;
@@ -366,6 +359,33 @@ const slice = createSlice({
   },
 });
 
+const applyClipSkip = (state: { clipSkip: number }, model: ParameterModel | null, clipSkip: number) => {
+  if (model === null) {
+    return;
+  }
+
+  const maxClip = getModelMaxClipSkip(model);
+
+  state.clipSkip = clamp(clipSkip, 0, maxClip);
+};
+
+const hasModelClipSkip = (model: ParameterModel | null) => {
+  if (model === null) {
+    return false;
+  }
+
+  return getModelMaxClipSkip(model) > 0;
+};
+
+const getModelMaxClipSkip = (model: ParameterModel) => {
+  if (model.base === 'sdxl') {
+    // We don't support user-defined CLIP skip for SDXL because it doesn't do anything useful
+    return 0;
+  }
+
+  return CLIP_SKIP_MAP[model.base].maxClip;
+};
+
 const resetState = (state: ParamsState): ParamsState => {
   // When a new session is requested, we need to keep the current model selections, plus dependent state
   // like VAE precision. Everything else gets reset to default.
@@ -484,7 +504,8 @@ export const selectCFGScale = createParamsSelector((params) => params.cfgScale);
 export const selectGuidance = createParamsSelector((params) => params.guidance);
 export const selectSteps = createParamsSelector((params) => params.steps);
 export const selectCFGRescaleMultiplier = createParamsSelector((params) => params.cfgRescaleMultiplier);
-export const selectCLIPSKip = createParamsSelector((params) => params.clipSkip);
+export const selectCLIPSkip = createParamsSelector((params) => params.clipSkip);
+export const selectHasModelCLIPSkip = createParamsSelector((params) => hasModelClipSkip(params.model));
 export const selectCanvasCoherenceEdgeSize = createParamsSelector((params) => params.canvasCoherenceEdgeSize);
 export const selectCanvasCoherenceMinDenoise = createParamsSelector((params) => params.canvasCoherenceMinDenoise);
 export const selectCanvasCoherenceMode = createParamsSelector((params) => params.canvasCoherenceMode);

invokeai/frontend/web/src/features/gallery/components/ImageContextMenu/ImageMenuItemMetadataRecallActionsCanvasGenerateTabs.tsx

@@ -2,6 +2,7 @@ import { Menu, MenuButton, MenuItem, MenuList } from '@invoke-ai/ui-library';
 import { SubMenuButtonContent, useSubMenu } from 'common/hooks/useSubMenu';
 import { useImageDTOContext } from 'features/gallery/contexts/ImageDTOContext';
 import { useRecallAll } from 'features/gallery/hooks/useRecallAll';
+import { useRecallCLIPSkip } from 'features/gallery/hooks/useRecallCLIPSkip';
 import { useRecallDimensions } from 'features/gallery/hooks/useRecallDimensions';
 import { useRecallPrompts } from 'features/gallery/hooks/useRecallPrompts';
 import { useRecallRemix } from 'features/gallery/hooks/useRecallRemix';
@@ -28,6 +29,7 @@ export const ImageMenuItemMetadataRecallActionsCanvasGenerateTabs = memo(() => {
   const recallPrompts = useRecallPrompts(imageDTO);
   const recallSeed = useRecallSeed(imageDTO);
   const recallDimensions = useRecallDimensions(imageDTO);
+  const recallCLIPSkip = useRecallCLIPSkip(imageDTO);
 
   return (
     <MenuItem {...subMenu.parentMenuItemProps} icon={<PiArrowBendUpLeftBold />}>
@@ -55,6 +57,9 @@ export const ImageMenuItemMetadataRecallActionsCanvasGenerateTabs = memo(() => {
           <MenuItem icon={<PiRulerBold />} onClick={recallDimensions.recall} isDisabled={!recallDimensions.isEnabled}>
             {t('parameters.useSize')}
           </MenuItem>
+          <MenuItem icon={<PiRulerBold />} onClick={recallCLIPSkip.recall} isDisabled={!recallCLIPSkip.isEnabled}>
+            {t('parameters.useClipSkip')}
+          </MenuItem>
         </MenuList>
       </Menu>
     </MenuItem>

invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx

@@ -40,6 +40,7 @@ const ImageMetadataActions = (props: Props) => {
       <SingleMetadataDatum metadata={metadata} handler={MetadataHandlers.Seed} />
       <SingleMetadataDatum metadata={metadata} handler={MetadataHandlers.Steps} />
       <SingleMetadataDatum metadata={metadata} handler={MetadataHandlers.Scheduler} />
+      <SingleMetadataDatum metadata={metadata} handler={MetadataHandlers.CLIPSkip} />
       <SingleMetadataDatum metadata={metadata} handler={MetadataHandlers.CFGScale} />
       <SingleMetadataDatum metadata={metadata} handler={MetadataHandlers.CFGRescaleMultiplier} />
       <SingleMetadataDatum metadata={metadata} handler={MetadataHandlers.Guidance} />

invokeai/frontend/web/src/features/gallery/hooks/useRecallCLIPSkip.ts

@@ -0,0 +1,72 @@
+import { useAppSelector, useAppStore } from 'app/store/storeHooks';
+import { selectHasModelCLIPSkip } from 'features/controlLayers/store/paramsSlice';
+import { MetadataHandlers, MetadataUtils } from 'features/metadata/parsing';
+import { selectActiveTab } from 'features/ui/store/uiSelectors';
+import type { TabName } from 'features/ui/store/uiTypes';
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useDebouncedMetadata } from 'services/api/hooks/useDebouncedMetadata';
+import type { ImageDTO } from 'services/api/types';
+
+const ALLOWED_TABS: TabName[] = ['canvas', 'generate', 'upscaling'];
+
+export const useRecallCLIPSkip = (imageDTO: ImageDTO) => {
+  const store = useAppStore();
+  const hasModelCLIPSkip = useAppSelector(selectHasModelCLIPSkip);
+  const tab = useAppSelector(selectActiveTab);
+  const [hasCLIPSkip, setHasCLIPSkip] = useState(false);
+
+  const { metadata, isLoading } = useDebouncedMetadata(imageDTO.image_name);
+
+  useEffect(() => {
+    const parse = async () => {
+      try {
+        await MetadataHandlers.CLIPSkip.parse(metadata, store);
+        setHasCLIPSkip(true);
+      } catch {
+        setHasCLIPSkip(false);
+      }
+    };
+
+    if (!hasModelCLIPSkip) {
+      setHasCLIPSkip(false);
+      return;
+    }
+
+    parse();
+  }, [metadata, store, hasModelCLIPSkip]);
+
+  const isEnabled = useMemo(() => {
+    if (isLoading) {
+      return false;
+    }
+
+    if (!ALLOWED_TABS.includes(tab)) {
+      return false;
+    }
+
+    if (!metadata) {
+      return false;
+    }
+
+    if (!hasCLIPSkip) {
+      return false;
+    }
+
+    return true;
+  }, [hasCLIPSkip, isLoading, metadata, tab]);
+
+  const recall = useCallback(() => {
+    if (!metadata) {
+      return;
+    }
+    if (!isEnabled) {
+      return;
+    }
+    MetadataUtils.recallByHandler({ metadata, handler: MetadataHandlers.CLIPSkip, store });
+  }, [metadata, isEnabled, store]);
+
+  return {
+    recall,
+    isEnabled,
+  };
+};

invokeai/frontend/web/src/features/metadata/parsing.tsx

@@ -15,6 +15,7 @@ import {
   selectBase,
   setCfgRescaleMultiplier,
   setCfgScale,
+  setClipSkip,
   setGuidance,
   setImg2imgStrength,
   setRefinerCFGScale,
@@ -41,6 +42,7 @@ import { modelSelected } from 'features/parameters/store/actions';
 import type {
   ParameterCFGRescaleMultiplier,
   ParameterCFGScale,
+  ParameterCLIPSkip,
   ParameterGuidance,
   ParameterHeight,
   ParameterModel,
@@ -63,6 +65,7 @@ import {
   zLoRAWeight,
   zParameterCFGRescaleMultiplier,
   zParameterCFGScale,
+  zParameterCLIPSkip,
   zParameterGuidance,
   zParameterImageDimension,
   zParameterNegativePrompt,
@@ -321,6 +324,24 @@ const CFGRescaleMultiplier: SingleMetadataHandler<ParameterCFGRescaleMultiplier>
 };
 //#endregion CFG Rescale Multiplier
 
+//#region CLIP Skip
+const CLIPSkip: SingleMetadataHandler<ParameterCLIPSkip> = {
+  [SingleMetadataKey]: true,
+  type: 'CLIPSkip',
+  parse: (metadata, _store) => {
+    const raw = getProperty(metadata, 'clip_skip');
+    const parsed = zParameterCLIPSkip.parse(raw);
+    return Promise.resolve(parsed);
+  },
+  recall: (value, store) => {
+    store.dispatch(setClipSkip(value));
+  },
+  i18nKey: 'metadata.clipSkip',
+  LabelComponent: MetadataLabel,
+  ValueComponent: ({ value }: SingleMetadataValueProps<ParameterCLIPSkip>) => <MetadataPrimitiveValue value={value} />,
+};
+//#endregion CLIP Skip
+
 //#region Guidance
 const Guidance: SingleMetadataHandler<ParameterGuidance> = {
   [SingleMetadataKey]: true,
@@ -883,6 +904,7 @@ export const MetadataHandlers = {
   NegativePrompt,
   CFGScale,
   CFGRescaleMultiplier,
+  CLIPSkip,
   Guidance,
   Scheduler,
   Width,

invokeai/frontend/web/src/features/parameters/components/Advanced/ParamClipSkip.tsx

@@ -1,14 +1,14 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { InformationalPopover } from 'common/components/InformationalPopover/InformationalPopover';
-import { selectCLIPSKip, selectModel, setClipSkip } from 'features/controlLayers/store/paramsSlice';
+import { selectCLIPSkip, selectModel, setClipSkip } from 'features/controlLayers/store/paramsSlice';
 import { CLIP_SKIP_MAP } from 'features/parameters/types/constants';
 import { selectCLIPSkipConfig } from 'features/system/store/configSlice';
 import { memo, useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 
 const ParamClipSkip = () => {
-  const clipSkip = useAppSelector(selectCLIPSKip);
+  const clipSkip = useAppSelector(selectCLIPSkip);
   const config = useAppSelector(selectCLIPSkipConfig);
   const model = useAppSelector(selectModel);
 

invokeai/frontend/web/src/features/parameters/types/parameterSchemas.ts

@@ -193,3 +193,8 @@ export type ParameterCanvasCoherenceMode = z.infer<typeof zParameterCanvasCohere
 export const [zLoRAWeight, isParameterLoRAWeight] = buildParameter(z.number());
 export type ParameterLoRAWeight = z.infer<typeof zLoRAWeight>;
 // #endregion
+
+// #region CLIP skip
+export const [zParameterCLIPSkip, isParameterCLIPSkip] = buildParameter(z.number().int().min(0));
+export type ParameterCLIPSkip = z.infer<typeof zParameterCLIPSkip>;
+// #endregion