Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(ui, worker): update processed image resolution based on base model type #5986

Merged
merged 8 commits into from
Mar 19, 2024
Merged

feat(ui, worker): update processed image resolution based on base model type #5986

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5fa19ca
typegen
Mar 18, 2024
367e10b
feat(worker): add image_resolution as option for all cnet procesors
maryhipp Mar 18, 2024
b651ca8
'feat(ui): update processor constants to calculate default resolution…
Mar 18, 2024
23933bd
fix(worker): remove resolution from zoe as it seems to break it
maryhipp Mar 18, 2024
1f8ecad
fix(ui): remove image_resolution from colormap
Mar 18, 2024
2a54fe6
tsc
Mar 18, 2024
b4fae5c
ruff
maryhipp Mar 18, 2024
f1961ff
prettier
Mar 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions invokeai/app/invocations/controlnet_image_processors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput:
class CannyImageProcessorInvocation(ImageProcessorInvocation):
"""Canny edge detection for ControlNet"""

image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
low_threshold: int = InputField(
default=100, ge=0, le=255, description="The low threshold of the Canny pixel gradient (0-255)"
)
Expand All @@ -189,7 +190,12 @@ def load_image(self, context: InvocationContext) -> Image.Image:

def run_processor(self, image):
canny_processor = CannyDetector()
processed_image = canny_processor(image, self.low_threshold, self.high_threshold)
processed_image = canny_processor(
image,
self.low_threshold,
self.high_threshold,
image_resolution=self.image_resolution,
)
return processed_image


Expand Down Expand Up @@ -279,6 +285,7 @@ class MidasDepthImageProcessorInvocation(ImageProcessorInvocation):

a_mult: float = InputField(default=2.0, ge=0, description="Midas parameter `a_mult` (a = a_mult * PI)")
bg_th: float = InputField(default=0.1, ge=0, description="Midas parameter `bg_th`")
image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)
# depth_and_normal not supported in controlnet_aux v0.0.3
# depth_and_normal: bool = InputField(default=False, description="whether to use depth and normal mode")

Expand All @@ -288,6 +295,7 @@ def run_processor(self, image):
image,
a=np.pi * self.a_mult,
bg_th=self.bg_th,
image_resolution=self.image_resolution,
# dept_and_normal not supported in controlnet_aux v0.0.3
# depth_and_normal=self.depth_and_normal,
)
Expand Down Expand Up @@ -419,10 +427,13 @@ class MediapipeFaceProcessorInvocation(ImageProcessorInvocation):

max_faces: int = InputField(default=1, ge=1, description="Maximum number of faces to detect")
min_confidence: float = InputField(default=0.5, ge=0, le=1, description="Minimum confidence for face detection")
image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)

def run_processor(self, image):
mediapipe_face_processor = MediapipeFaceDetector()
processed_image = mediapipe_face_processor(image, max_faces=self.max_faces, min_confidence=self.min_confidence)
processed_image = mediapipe_face_processor(
image, max_faces=self.max_faces, min_confidence=self.min_confidence, image_resolution=self.image_resolution
)
return processed_image


Expand Down Expand Up @@ -505,13 +516,15 @@ def run_processor(self, img):
class SegmentAnythingProcessorInvocation(ImageProcessorInvocation):
"""Applies segment anything processing to image"""

image_resolution: int = InputField(default=512, ge=0, description=FieldDescriptions.image_res)

def run_processor(self, image):
# segment_anything_processor = SamDetector.from_pretrained("ybelkada/segment-anything", subfolder="checkpoints")
segment_anything_processor = SamDetectorReproducibleColors.from_pretrained(
"ybelkada/segment-anything", subfolder="checkpoints"
)
np_img = np.array(image, dtype=np.uint8)
processed_image = segment_anything_processor(np_img)
processed_image = segment_anything_processor(np_img, image_resolution=self.image_resolution)
return processed_image


Expand Down
42 changes: 34 additions & 8 deletions invokeai/frontend/web/src/features/controlAdapters/components/processors/CannyProcessor.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
import { useProcessorNodeChanged } from 'features/controlAdapters/components/hooks/useProcessorNodeChanged';
import { CONTROLNET_PROCESSORS } from 'features/controlAdapters/store/constants';
import { useGetDefaultForControlnetProcessor } from 'features/controlAdapters/hooks/useGetDefaultForControlnetProcessor';
import type { RequiredCannyImageProcessorInvocation } from 'features/controlAdapters/store/types';
import { memo, useCallback } from 'react';
import { useTranslation } from 'react-i18next';

import ProcessorWrapper from './common/ProcessorWrapper';

const DEFAULTS = CONTROLNET_PROCESSORS.canny_image_processor.default as RequiredCannyImageProcessorInvocation;

type CannyProcessorProps = {
controlNetId: string;
processorNode: RequiredCannyImageProcessorInvocation;
Expand All @@ -17,9 +15,12 @@ type CannyProcessorProps = {

const CannyProcessor = (props: CannyProcessorProps) => {
const { controlNetId, processorNode, isEnabled } = props;
const { low_threshold, high_threshold } = processorNode;
const { low_threshold, high_threshold, image_resolution } = processorNode;
const processorChanged = useProcessorNodeChanged();
const { t } = useTranslation();
const defaults = useGetDefaultForControlnetProcessor(
'canny_image_processor'
) as RequiredCannyImageProcessorInvocation;

const handleLowThresholdChanged = useCallback(
(v: number) => {
Expand All @@ -35,21 +36,28 @@ const CannyProcessor = (props: CannyProcessorProps) => {
[controlNetId, processorChanged]
);

const handleImageResolutionChanged = useCallback(
(v: number) => {
processorChanged(controlNetId, { image_resolution: v });
},
[controlNetId, processorChanged]
);

return (
<ProcessorWrapper>
<FormControl isDisabled={!isEnabled}>
<FormLabel>{t('controlnet.lowThreshold')}</FormLabel>
<CompositeSlider
value={low_threshold}
onChange={handleLowThresholdChanged}
defaultValue={DEFAULTS.low_threshold}
defaultValue={defaults.low_threshold}
min={0}
max={255}
/>
<CompositeNumberInput
value={low_threshold}
onChange={handleLowThresholdChanged}
defaultValue={DEFAULTS.low_threshold}
defaultValue={defaults.low_threshold}
min={0}
max={255}
/>
Expand All @@ -59,18 +67,36 @@ const CannyProcessor = (props: CannyProcessorProps) => {
<CompositeSlider
value={high_threshold}
onChange={handleHighThresholdChanged}
defaultValue={DEFAULTS.high_threshold}
defaultValue={defaults.high_threshold}
min={0}
max={255}
/>
<CompositeNumberInput
value={high_threshold}
onChange={handleHighThresholdChanged}
defaultValue={DEFAULTS.high_threshold}
defaultValue={defaults.high_threshold}
min={0}
max={255}
/>
</FormControl>
<FormControl isDisabled={!isEnabled}>
<FormLabel>{t('controlnet.imageResolution')}</FormLabel>
<CompositeSlider
value={image_resolution}
onChange={handleImageResolutionChanged}
defaultValue={defaults.image_resolution}
min={0}
max={4096}
marks
/>
<CompositeNumberInput
value={image_resolution}
onChange={handleImageResolutionChanged}
defaultValue={defaults.image_resolution}
min={0}
max={4096}
/>
</FormControl>
</ProcessorWrapper>
);
};
Expand Down
11 changes: 6 additions & 5 deletions ...eai/frontend/web/src/features/controlAdapters/components/processors/ColorMapProcessor.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
import { useProcessorNodeChanged } from 'features/controlAdapters/components/hooks/useProcessorNodeChanged';
import { CONTROLNET_PROCESSORS } from 'features/controlAdapters/store/constants';
import { useGetDefaultForControlnetProcessor } from 'features/controlAdapters/hooks/useGetDefaultForControlnetProcessor';
import type { RequiredColorMapImageProcessorInvocation } from 'features/controlAdapters/store/types';
import { memo, useCallback } from 'react';
import { useTranslation } from 'react-i18next';

import ProcessorWrapper from './common/ProcessorWrapper';

const DEFAULTS = CONTROLNET_PROCESSORS.color_map_image_processor.default as RequiredColorMapImageProcessorInvocation;

type ColorMapProcessorProps = {
controlNetId: string;
processorNode: RequiredColorMapImageProcessorInvocation;
Expand All @@ -20,6 +18,9 @@ const ColorMapProcessor = (props: ColorMapProcessorProps) => {
const { color_map_tile_size } = processorNode;
const processorChanged = useProcessorNodeChanged();
const { t } = useTranslation();
const defaults = useGetDefaultForControlnetProcessor(
'color_map_image_processor'
) as RequiredColorMapImageProcessorInvocation;

const handleColorMapTileSizeChanged = useCallback(
(v: number) => {
Expand All @@ -34,7 +35,7 @@ const ColorMapProcessor = (props: ColorMapProcessorProps) => {
<FormLabel>{t('controlnet.colorMapTileSize')}</FormLabel>
<CompositeSlider
value={color_map_tile_size}
defaultValue={DEFAULTS.color_map_tile_size}
defaultValue={defaults.color_map_tile_size}
onChange={handleColorMapTileSizeChanged}
min={1}
max={256}
Expand All @@ -43,7 +44,7 @@ const ColorMapProcessor = (props: ColorMapProcessorProps) => {
/>
<CompositeNumberInput
value={color_map_tile_size}
defaultValue={DEFAULTS.color_map_tile_size}
defaultValue={defaults.color_map_tile_size}
onChange={handleColorMapTileSizeChanged}
min={1}
max={4096}
Expand Down
29 changes: 15 additions & 14 deletions ...ontend/web/src/features/controlAdapters/components/processors/ContentShuffleProcessor.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
import { useProcessorNodeChanged } from 'features/controlAdapters/components/hooks/useProcessorNodeChanged';
import { CONTROLNET_PROCESSORS } from 'features/controlAdapters/store/constants';
import { useGetDefaultForControlnetProcessor } from 'features/controlAdapters/hooks/useGetDefaultForControlnetProcessor';
import type { RequiredContentShuffleImageProcessorInvocation } from 'features/controlAdapters/store/types';
import { memo, useCallback } from 'react';
import { useTranslation } from 'react-i18next';

import ProcessorWrapper from './common/ProcessorWrapper';

const DEFAULTS = CONTROLNET_PROCESSORS.content_shuffle_image_processor
.default as RequiredContentShuffleImageProcessorInvocation;

type Props = {
controlNetId: string;
processorNode: RequiredContentShuffleImageProcessorInvocation;
Expand All @@ -22,6 +19,10 @@ const ContentShuffleProcessor = (props: Props) => {
const processorChanged = useProcessorNodeChanged();
const { t } = useTranslation();

const defaults = useGetDefaultForControlnetProcessor(
'content_shuffle_image_processor'
) as RequiredContentShuffleImageProcessorInvocation;

const handleDetectResolutionChanged = useCallback(
(v: number) => {
processorChanged(controlNetId, { detect_resolution: v });
Expand Down Expand Up @@ -63,15 +64,15 @@ const ContentShuffleProcessor = (props: Props) => {
<FormLabel>{t('controlnet.detectResolution')}</FormLabel>
<CompositeSlider
value={detect_resolution}
defaultValue={DEFAULTS.detect_resolution}
defaultValue={defaults.detect_resolution}
onChange={handleDetectResolutionChanged}
min={0}
max={4096}
marks
/>
<CompositeNumberInput
value={detect_resolution}
defaultValue={DEFAULTS.detect_resolution}
defaultValue={defaults.detect_resolution}
onChange={handleDetectResolutionChanged}
min={0}
max={4096}
Expand All @@ -81,34 +82,34 @@ const ContentShuffleProcessor = (props: Props) => {
<FormLabel>{t('controlnet.imageResolution')}</FormLabel>
<CompositeSlider
value={image_resolution}
defaultValue={DEFAULTS.image_resolution}
defaultValue={defaults.image_resolution}
onChange={handleImageResolutionChanged}
min={0}
max={4096}
marks
/>
<CompositeNumberInput
value={image_resolution}
defaultValue={DEFAULTS.image_resolution}
defaultValue={defaults.image_resolution}
onChange={handleImageResolutionChanged}
min={0}
max={4096}
/>
</FormControl>
<FormControl isDisabled={!isEnabled}>
<FormLabel>{t('controlnet.w')}</FormLabel>
<CompositeSlider value={w} defaultValue={DEFAULTS.w} onChange={handleWChanged} min={0} max={4096} marks />
<CompositeNumberInput value={w} defaultValue={DEFAULTS.w} onChange={handleWChanged} min={0} max={4096} />
<CompositeSlider value={w} defaultValue={defaults.w} onChange={handleWChanged} min={0} max={4096} marks />
<CompositeNumberInput value={w} defaultValue={defaults.w} onChange={handleWChanged} min={0} max={4096} />
</FormControl>
<FormControl isDisabled={!isEnabled}>
<FormLabel>{t('controlnet.h')}</FormLabel>
<CompositeSlider value={h} defaultValue={DEFAULTS.h} onChange={handleHChanged} min={0} max={4096} marks />
<CompositeNumberInput value={h} defaultValue={DEFAULTS.h} onChange={handleHChanged} min={0} max={4096} />
<CompositeSlider value={h} defaultValue={defaults.h} onChange={handleHChanged} min={0} max={4096} marks />
<CompositeNumberInput value={h} defaultValue={defaults.h} onChange={handleHChanged} min={0} max={4096} />
</FormControl>
<FormControl isDisabled={!isEnabled}>
<FormLabel>{t('controlnet.f')}</FormLabel>
<CompositeSlider value={f} defaultValue={DEFAULTS.f} onChange={handleFChanged} min={0} max={4096} marks />
<CompositeNumberInput value={f} defaultValue={DEFAULTS.f} onChange={handleFChanged} min={0} max={4096} />
<CompositeSlider value={f} defaultValue={defaults.f} onChange={handleFChanged} min={0} max={4096} marks />
<CompositeNumberInput value={f} defaultValue={defaults.f} onChange={handleFChanged} min={0} max={4096} />
</FormControl>
</ProcessorWrapper>
);
Expand Down
19 changes: 10 additions & 9 deletions ...i/frontend/web/src/features/controlAdapters/components/processors/DWOpenposeProcessor.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
import { CompositeNumberInput, CompositeSlider, Flex, FormControl, FormLabel, Switch } from '@invoke-ai/ui-library';
import { useProcessorNodeChanged } from 'features/controlAdapters/components/hooks/useProcessorNodeChanged';
import { CONTROLNET_PROCESSORS } from 'features/controlAdapters/store/constants';
import { useGetDefaultForControlnetProcessor } from 'features/controlAdapters/hooks/useGetDefaultForControlnetProcessor';
import type { RequiredDWOpenposeImageProcessorInvocation } from 'features/controlAdapters/store/types';
import type { ChangeEvent } from 'react';
import { memo, useCallback } from 'react';
import { useTranslation } from 'react-i18next';

import ProcessorWrapper from './common/ProcessorWrapper';

const DEFAULTS = CONTROLNET_PROCESSORS.dw_openpose_image_processor
.default as RequiredDWOpenposeImageProcessorInvocation;

type Props = {
controlNetId: string;
processorNode: RequiredDWOpenposeImageProcessorInvocation;
Expand All @@ -23,6 +20,10 @@ const DWOpenposeProcessor = (props: Props) => {
const processorChanged = useProcessorNodeChanged();
const { t } = useTranslation();

const defaults = useGetDefaultForControlnetProcessor(
'dw_openpose_image_processor'
) as RequiredDWOpenposeImageProcessorInvocation;

const handleDrawBodyChanged = useCallback(
(e: ChangeEvent<HTMLInputElement>) => {
processorChanged(controlNetId, { draw_body: e.target.checked });
Expand Down Expand Up @@ -56,31 +57,31 @@ const DWOpenposeProcessor = (props: Props) => {
<Flex sx={{ flexDir: 'row', gap: 6 }}>
<FormControl isDisabled={!isEnabled} w="max-content">
<FormLabel>{t('controlnet.body')}</FormLabel>
<Switch defaultChecked={DEFAULTS.draw_body} isChecked={draw_body} onChange={handleDrawBodyChanged} />
<Switch defaultChecked={defaults.draw_body} isChecked={draw_body} onChange={handleDrawBodyChanged} />
</FormControl>
<FormControl isDisabled={!isEnabled} w="max-content">
<FormLabel>{t('controlnet.face')}</FormLabel>
<Switch defaultChecked={DEFAULTS.draw_face} isChecked={draw_face} onChange={handleDrawFaceChanged} />
<Switch defaultChecked={defaults.draw_face} isChecked={draw_face} onChange={handleDrawFaceChanged} />
</FormControl>
<FormControl isDisabled={!isEnabled} w="max-content">
<FormLabel>{t('controlnet.hands')}</FormLabel>
<Switch defaultChecked={DEFAULTS.draw_hands} isChecked={draw_hands} onChange={handleDrawHandsChanged} />
<Switch defaultChecked={defaults.draw_hands} isChecked={draw_hands} onChange={handleDrawHandsChanged} />
</FormControl>
</Flex>
<FormControl isDisabled={!isEnabled}>
<FormLabel>{t('controlnet.imageResolution')}</FormLabel>
<CompositeSlider
value={image_resolution}
onChange={handleImageResolutionChanged}
defaultValue={DEFAULTS.image_resolution}
defaultValue={defaults.image_resolution}
min={0}
max={4096}
marks
/>
<CompositeNumberInput
value={image_resolution}
onChange={handleImageResolutionChanged}
defaultValue={DEFAULTS.image_resolution}
defaultValue={defaults.image_resolution}
min={0}
max={4096}
/>
Expand Down
Loading
Loading