feat: control adapters in control layers canvas

invokeai/app/invocations/controlnet_image_processors.py

@@ -35,15 +35,16 @@
 from invokeai.app.invocations.primitives import ImageOutput
 from invokeai.app.invocations.util import validate_begin_end_step, validate_weights
 from invokeai.app.services.shared.invocation_context import InvocationContext
-from invokeai.app.util.controlnet_utils import CONTROLNET_MODE_VALUES, CONTROLNET_RESIZE_VALUES
+from invokeai.app.util.controlnet_utils import CONTROLNET_MODE_VALUES, CONTROLNET_RESIZE_VALUES, heuristic_resize
 from invokeai.backend.image_util.canny import get_canny_edges
 from invokeai.backend.image_util.depth_anything import DepthAnythingDetector
 from invokeai.backend.image_util.dw_openpose import DWOpenposeDetector
 from invokeai.backend.image_util.hed import HEDProcessor
 from invokeai.backend.image_util.lineart import LineartProcessor
 from invokeai.backend.image_util.lineart_anime import LineartAnimeProcessor
+from invokeai.backend.image_util.util import np_to_pil, pil_to_np
 
-from .baseinvocation import BaseInvocation, BaseInvocationOutput, invocation, invocation_output
+from .baseinvocation import BaseInvocation, BaseInvocationOutput, Classification, invocation, invocation_output
 
 
 class ControlField(BaseModel):
@@ -634,3 +635,27 @@ def run_processor(self, image: Image.Image):
             resolution=self.image_resolution,
         )
         return processed_image
+
+
+@invocation(
+    "heuristic_resize",
+    title="Heuristic Resize",
+    tags=["image, controlnet"],
+    category="image",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class HeuristicResizeInvocation(BaseInvocation):
+    """Resize an image using a heuristic method. Preserves edge maps."""
+
+    image: ImageField = InputField(description="The image to resize")
+    width: int = InputField(default=512, gt=0, description="The width to resize to (px)")
+    height: int = InputField(default=512, gt=0, description="The height to resize to (px)")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.images.get_pil(self.image.image_name, "RGB")
+        np_img = pil_to_np(image)
+        np_resized = heuristic_resize(np_img, (self.width, self.height))
+        resized = np_to_pil(np_resized)
+        image_dto = context.images.save(image=resized)
+        return ImageOutput.build(image_dto)

invokeai/backend/image_util/util.py

@@ -144,10 +144,8 @@ def resize_image_to_resolution(input_image: np.ndarray, resolution: int) -> np.n
     h = float(input_image.shape[0])
     w = float(input_image.shape[1])
     scaling_factor = float(resolution) / min(h, w)
-    h *= scaling_factor
-    w *= scaling_factor
-    h = int(np.round(h / 64.0)) * 64
-    w = int(np.round(w / 64.0)) * 64
+    h = int(h * scaling_factor)
+    w = int(w * scaling_factor)
     if scaling_factor > 1:
         return cv2.resize(input_image, (w, h), interpolation=cv2.INTER_LANCZOS4)
     else:

invokeai/frontend/web/package.json

@@ -101,6 +101,7 @@
     "serialize-error": "^11.0.3",
     "socket.io-client": "^4.7.5",
     "use-debounce": "^10.0.0",
+    "use-device-pixel-ratio": "^1.1.2",
     "use-image": "^1.1.1",
     "uuid": "^9.0.1",
     "zod": "^3.22.4",

invokeai/frontend/web/public/locales/en.json

@@ -156,6 +156,7 @@
         "balanced": "Balanced",
         "base": "Base",
         "beginEndStepPercent": "Begin / End Step Percentage",
+        "beginEndStepPercentShort": "Begin/End %",
         "bgth": "bg_th",
         "canny": "Canny",
         "cannyDescription": "Canny edge detection",
@@ -227,7 +228,8 @@
         "scribble": "scribble",
         "selectModel": "Select a model",
         "selectCLIPVisionModel": "Select a CLIP Vision model",
-        "setControlImageDimensions": "Set Control Image Dimensions To W/H",
+        "setControlImageDimensions": "Copy size to W/H (optimize for model)",
+        "setControlImageDimensionsForce": "Copy size to W/H (ignore model)",
         "showAdvanced": "Show Advanced",
         "small": "Small",
         "toggleControlNet": "Toggle this ControlNet",
@@ -1511,16 +1513,15 @@
     "app": {
         "storeNotInitialized": "Store is not initialized"
     },
-    "regionalPrompts": {
+    "controlLayers": {
         "deleteAll": "Delete All",
         "addLayer": "Add Layer",
         "moveToFront": "Move to Front",
         "moveToBack": "Move to Back",
         "moveForward": "Move Forward",
         "moveBackward": "Move Backward",
         "brushSize": "Brush Size",
-        "regionalControl": "Regional Control (ALPHA)",
-        "enableRegionalPrompts": "Enable $t(regionalPrompts.regionalPrompts)",
+        "controlLayers": "Control Layers (BETA)",
         "globalMaskOpacity": "Global Mask Opacity",
         "autoNegative": "Auto Negative",
         "toggleVisibility": "Toggle Layer Visibility",
@@ -1531,6 +1532,16 @@
         "maskPreviewColor": "Mask Preview Color",
         "addPositivePrompt": "Add $t(common.positivePrompt)",
         "addNegativePrompt": "Add $t(common.negativePrompt)",
-        "addIPAdapter": "Add $t(common.ipAdapter)"
+        "addIPAdapter": "Add $t(common.ipAdapter)",
+        "regionalGuidance": "Regional Guidance",
+        "regionalGuidanceLayer": "$t(controlLayers.regionalGuidance) $t(unifiedCanvas.layer)",
+        "controlNetLayer": "$t(common.controlNet) $t(unifiedCanvas.layer)",
+        "ipAdapterLayer": "$t(common.ipAdapter) $t(unifiedCanvas.layer)",
+        "opacity": "Opacity",
+        "globalControlAdapter": "Global $t(controlnet.controlAdapter_one)",
+        "globalControlAdapterLayer": "Global $t(controlnet.controlAdapter_one) $t(unifiedCanvas.layer)",
+        "globalIPAdapter": "Global $t(common.ipAdapter)",
+        "globalIPAdapterLayer": "Global $t(common.ipAdapter) $t(unifiedCanvas.layer)",
+        "opacityFilter": "Opacity Filter"
     }
 }

invokeai/frontend/web/src/app/logging/logger.ts

@@ -28,7 +28,7 @@ export type LoggerNamespace =
   | 'session'
   | 'queue'
   | 'dnd'
-  | 'regionalPrompts';
+  | 'controlLayers';
 
 export const logger = (namespace: LoggerNamespace) => $logger.get().child({ namespace });
 

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/index.ts

@@ -16,6 +16,7 @@ import { addCanvasMaskSavedToGalleryListener } from 'app/store/middleware/listen
 import { addCanvasMaskToControlNetListener } from 'app/store/middleware/listenerMiddleware/listeners/canvasMaskToControlNet';
 import { addCanvasMergedListener } from 'app/store/middleware/listenerMiddleware/listeners/canvasMerged';
 import { addCanvasSavedToGalleryListener } from 'app/store/middleware/listenerMiddleware/listeners/canvasSavedToGallery';
+import { addControlLayersToControlAdapterBridge } from 'app/store/middleware/listenerMiddleware/listeners/controlLayersToControlAdapterBridge';
 import { addControlNetAutoProcessListener } from 'app/store/middleware/listenerMiddleware/listeners/controlNetAutoProcess';
 import { addControlNetImageProcessedListener } from 'app/store/middleware/listenerMiddleware/listeners/controlNetImageProcessed';
 import { addEnqueueRequestedCanvasListener } from 'app/store/middleware/listenerMiddleware/listeners/enqueueRequestedCanvas';
@@ -157,3 +158,5 @@ addUpscaleRequestedListener(startAppListening);
 addDynamicPromptsListener(startAppListening);
 
 addSetDefaultSettingsListener(startAppListening);
+
+addControlLayersToControlAdapterBridge(startAppListening);

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasImageToControlNet.ts

@@ -48,12 +48,10 @@ export const addCanvasImageToControlNetListener = (startAppListening: AppStartLi
         })
       ).unwrap();
 
-      const { image_name } = imageDTO;
-
       dispatch(
         controlAdapterImageChanged({
           id,
-          controlImage: image_name,
+          controlImage: imageDTO,
         })
       );
     },

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/canvasMaskToControlNet.ts

@@ -58,12 +58,10 @@ export const addCanvasMaskToControlNetListener = (startAppListening: AppStartLis
         })
       ).unwrap();
 
-      const { image_name } = imageDTO;
-
       dispatch(
         controlAdapterImageChanged({
           id,
-          controlImage: image_name,
+          controlImage: imageDTO,
         })
       );
     },

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlLayersToControlAdapterBridge.ts

@@ -0,0 +1,144 @@
+import { createAction } from '@reduxjs/toolkit';
+import type { AppStartListening } from 'app/store/middleware/listenerMiddleware';
+import { CONTROLNET_PROCESSORS } from 'features/controlAdapters/store/constants';
+import { controlAdapterAdded, controlAdapterRemoved } from 'features/controlAdapters/store/controlAdaptersSlice';
+import type { ControlNetConfig, IPAdapterConfig } from 'features/controlAdapters/store/types';
+import { isControlAdapterProcessorType } from 'features/controlAdapters/store/types';
+import {
+  controlAdapterLayerAdded,
+  ipAdapterLayerAdded,
+  layerDeleted,
+  maskLayerIPAdapterAdded,
+  maskLayerIPAdapterDeleted,
+  regionalGuidanceLayerAdded,
+} from 'features/controlLayers/store/controlLayersSlice';
+import type { Layer } from 'features/controlLayers/store/types';
+import { modelConfigsAdapterSelectors, modelsApi } from 'services/api/endpoints/models';
+import { isControlNetModelConfig, isIPAdapterModelConfig } from 'services/api/types';
+import { assert } from 'tsafe';
+import { v4 as uuidv4 } from 'uuid';
+
+export const guidanceLayerAdded = createAction<Layer['type']>('controlLayers/guidanceLayerAdded');
+export const guidanceLayerDeleted = createAction<string>('controlLayers/guidanceLayerDeleted');
+export const allLayersDeleted = createAction('controlLayers/allLayersDeleted');
+export const guidanceLayerIPAdapterAdded = createAction<string>('controlLayers/guidanceLayerIPAdapterAdded');
+export const guidanceLayerIPAdapterDeleted = createAction<{ layerId: string; ipAdapterId: string }>(
+  'controlLayers/guidanceLayerIPAdapterDeleted'
+);
+
+export const addControlLayersToControlAdapterBridge = (startAppListening: AppStartListening) => {
+  startAppListening({
+    actionCreator: guidanceLayerAdded,
+    effect: (action, { dispatch, getState }) => {
+      const type = action.payload;
+      const layerId = uuidv4();
+      if (type === 'regional_guidance_layer') {
+        dispatch(regionalGuidanceLayerAdded({ layerId }));
+        return;
+      }
+
+      const state = getState();
+      const baseModel = state.generation.model?.base;
+      const modelConfigs = modelsApi.endpoints.getModelConfigs.select(undefined)(state).data;
+
+      if (type === 'ip_adapter_layer') {
+        const ipAdapterId = uuidv4();
+        const overrides: Partial<IPAdapterConfig> = {
+          id: ipAdapterId,
+        };
+
+        // Find and select the first matching model
+        if (modelConfigs) {
+          const models = modelConfigsAdapterSelectors.selectAll(modelConfigs).filter(isIPAdapterModelConfig);
+          overrides.model = models.find((m) => m.base === baseModel) ?? null;
+        }
+        dispatch(controlAdapterAdded({ type: 'ip_adapter', overrides }));
+        dispatch(ipAdapterLayerAdded({ layerId, ipAdapterId }));
+        return;
+      }
+
+      if (type === 'control_adapter_layer') {
+        const controlNetId = uuidv4();
+        const overrides: Partial<ControlNetConfig> = {
+          id: controlNetId,
+        };
+
+        // Find and select the first matching model
+        if (modelConfigs) {
+          const models = modelConfigsAdapterSelectors.selectAll(modelConfigs).filter(isControlNetModelConfig);
+          const model = models.find((m) => m.base === baseModel) ?? null;
+          overrides.model = model;
+          const defaultPreprocessor = model?.default_settings?.preprocessor;
+          overrides.processorType = isControlAdapterProcessorType(defaultPreprocessor) ? defaultPreprocessor : 'none';
+          overrides.processorNode = CONTROLNET_PROCESSORS[overrides.processorType].buildDefaults(baseModel);
+        }
+        dispatch(controlAdapterAdded({ type: 'controlnet', overrides }));
+        dispatch(controlAdapterLayerAdded({ layerId, controlNetId }));
+        return;
+      }
+    },
+  });
+
+  startAppListening({
+    actionCreator: guidanceLayerDeleted,
+    effect: (action, { getState, dispatch }) => {
+      const layerId = action.payload;
+      const state = getState();
+      const layer = state.controlLayers.present.layers.find((l) => l.id === layerId);
+      assert(layer, `Layer ${layerId} not found`);
+
+      if (layer.type === 'ip_adapter_layer') {
+        dispatch(controlAdapterRemoved({ id: layer.ipAdapterId }));
+      } else if (layer.type === 'control_adapter_layer') {
+        dispatch(controlAdapterRemoved({ id: layer.controlNetId }));
+      } else if (layer.type === 'regional_guidance_layer') {
+        for (const ipAdapterId of layer.ipAdapterIds) {
+          dispatch(controlAdapterRemoved({ id: ipAdapterId }));
+        }
+      }
+      dispatch(layerDeleted(layerId));
+    },
+  });
+
+  startAppListening({
+    actionCreator: allLayersDeleted,
+    effect: (action, { dispatch, getOriginalState }) => {
+      const state = getOriginalState();
+      for (const layer of state.controlLayers.present.layers) {
+        dispatch(guidanceLayerDeleted(layer.id));
+      }
+    },
+  });
+
+  startAppListening({
+    actionCreator: guidanceLayerIPAdapterAdded,
+    effect: (action, { dispatch, getState }) => {
+      const layerId = action.payload;
+      const ipAdapterId = uuidv4();
+      const overrides: Partial<IPAdapterConfig> = {
+        id: ipAdapterId,
+      };
+
+      // Find and select the first matching model
+      const state = getState();
+      const baseModel = state.generation.model?.base;
+      const modelConfigs = modelsApi.endpoints.getModelConfigs.select(undefined)(state).data;
+      if (modelConfigs) {
+        const models = modelConfigsAdapterSelectors.selectAll(modelConfigs).filter(isIPAdapterModelConfig);
+        overrides.model = models.find((m) => m.base === baseModel) ?? null;
+      }
+
+      dispatch(controlAdapterAdded({ type: 'ip_adapter', overrides }));
+      dispatch(maskLayerIPAdapterAdded({ layerId, ipAdapterId }));
+    },
+  });
+
+  startAppListening({
+    actionCreator: guidanceLayerIPAdapterDeleted,
+    effect: (action, { dispatch }) => {
+      const { layerId, ipAdapterId } = action.payload;
+      dispatch(controlAdapterRemoved({ id: ipAdapterId }));
+      dispatch(maskLayerIPAdapterDeleted({ layerId, ipAdapterId }));
+    },
+  });
+};

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlNetAutoProcess.ts

@@ -12,6 +12,7 @@ import {
   selectControlAdapterById,
 } from 'features/controlAdapters/store/controlAdaptersSlice';
 import { isControlNetOrT2IAdapter } from 'features/controlAdapters/store/types';
+import { isEqual } from 'lodash-es';
 
 type AnyControlAdapterParamChangeAction =
   | ReturnType<typeof controlAdapterProcessorParamsChanged>
@@ -52,6 +53,11 @@ const predicate: AnyListenerPredicate<RootState> = (action, state, prevState) =>
     return false;
   }
 
+  if (prevCA.controlImage === ca.controlImage && isEqual(prevCA.processorNode, ca.processorNode)) {
+    // Don't re-process if the processor hasn't changed
+    return false;
+  }
+
   const isProcessorSelected = processorType !== 'none';
 
   const hasControlImage = Boolean(controlImage);

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/controlNetImageProcessed.ts

@@ -91,7 +91,7 @@ export const addControlNetImageProcessedListener = (startAppListening: AppStartL
           dispatch(
             controlAdapterProcessedImageChanged({
               id,
-              processedControlImage: processedControlImage.image_name,
+              processedControlImage,
             })
           );
         }

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageDropped.ts

@@ -71,7 +71,7 @@ export const addImageDroppedListener = (startAppListening: AppStartListening) =>
         dispatch(
           controlAdapterImageChanged({
             id,
-            controlImage: activeData.payload.imageDTO.image_name,
+            controlImage: activeData.payload.imageDTO,
           })
         );
         dispatch(

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageUploaded.ts

@@ -96,7 +96,7 @@ export const addImageUploadedFulfilledListener = (startAppListening: AppStartLis
         dispatch(
           controlAdapterImageChanged({
             id,
-            controlImage: imageDTO.image_name,
+            controlImage: imageDTO,
           })
         );
         dispatch(