diff --git a/invokeai/app/invocations/controlnet_image_processors.py b/invokeai/app/invocations/controlnet_image_processors.py index c0b332f27b6..e8698f98acd 100644 --- a/invokeai/app/invocations/controlnet_image_processors.py +++ b/invokeai/app/invocations/controlnet_image_processors.py @@ -1,11 +1,13 @@ # Invocations for ControlNet image preprocessors # initial implementation by Gregg Helt, 2023 # heavily leverages controlnet_aux package: https://github.com/patrickvonplaten/controlnet_aux +import random from builtins import bool, float from pathlib import Path -from typing import Dict, List, Literal, Union +from typing import Any, Dict, List, Literal, Union import cv2 +import cv2.ximgproc import numpy as np from controlnet_aux import ( ContentShuffleDetector, @@ -39,6 +41,7 @@ from invokeai.backend.image_util.canny import get_canny_edges from invokeai.backend.image_util.depth_anything import DEPTH_ANYTHING_MODELS, DepthAnythingDetector from invokeai.backend.image_util.dw_openpose import DWPOSE_MODELS, DWOpenposeDetector +from invokeai.backend.image_util.fast_guided_filter.fast_guided_filter import FastGuidedFilter from invokeai.backend.image_util.hed import HEDProcessor from invokeai.backend.image_util.lineart import LineartProcessor from invokeai.backend.image_util.lineart_anime import LineartAnimeProcessor @@ -476,37 +479,86 @@ def run_processor(self, image: Image.Image) -> Image.Image: title="Tile Resample Processor", tags=["controlnet", "tile"], category="controlnet", - version="1.2.3", + version="1.3.0", ) class TileResamplerProcessorInvocation(ImageProcessorInvocation): """Tile resampler processor""" # res: int = InputField(default=512, ge=0, le=1024, description="The pixel resolution for each tile") down_sampling_rate: float = InputField(default=1.0, ge=1.0, le=8.0, description="Down sampling rate") + mode: Literal["regular", "blur", "super"] = InputField( + default="regular", description="The Tile ControlNet pre-processing mode to use." + ) - # tile_resample copied from sd-webui-controlnet/scripts/processor.py - def tile_resample( - self, - np_img: np.ndarray, - res=512, # never used? - down_sampling_rate=1.0, - ): + # referenced from + # https://huggingface.co/TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic/blob/37f1c4575b543fb2036e39f5763d082fdd135318/TTP_tile_preprocessor_v5.py + def _apply_gaussian_blur(self, image_np: np.ndarray[Any, Any], ksize: int = 5, sigma_x: float = 1.0): + if ksize % 2 == 0: + ksize += 1 # ksize must be odd + blurred_image = cv2.GaussianBlur(image_np, (ksize, ksize), sigmaX=sigma_x) + return blurred_image + + # referenced from + # https://huggingface.co/TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic/blob/37f1c4575b543fb2036e39f5763d082fdd135318/TTP_tile_preprocessor_v5.py + def _apply_guided_filter(self, image_np: np.ndarray[Any, Any], radius: int, eps: float, scale: int): + filter = FastGuidedFilter(image_np, radius, eps, scale) + return filter.filter(image_np) + + def _regular_resample(self, np_img: np.ndarray[Any, Any]): + height, width, _ = np_img.shape np_img = HWC3(np_img) - if down_sampling_rate < 1.1: + if self.down_sampling_rate < 1.1: return np_img - H, W, C = np_img.shape - H = int(float(H) / float(down_sampling_rate)) - W = int(float(W) / float(down_sampling_rate)) - np_img = cv2.resize(np_img, (W, H), interpolation=cv2.INTER_AREA) + + new_height = int(float(height) / float(self.down_sampling_rate)) + new_width = int(float(width) / float(self.down_sampling_rate)) + np_img = cv2.resize(np_img, (new_width, new_height), interpolation=cv2.INTER_AREA) + return np_img + + # referenced from + # https://huggingface.co/TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic/blob/37f1c4575b543fb2036e39f5763d082fdd135318/TTP_tile_preprocessor_v5.py + def _blur_resample(self, np_img: np.ndarray[Any, Any]): + height, width, _ = np_img.shape + ratio = np.sqrt(1024.0 * 1024.0 / (width * height)) + resize_w, resize_h = int(width * ratio), int(height * ratio) + np_img = cv2.resize(np_img, (resize_w, resize_h)) + + blur_strength = random.sample([i / 10.0 for i in range(10, 201, 2)], k=1)[0] + radius = random.sample([i for i in range(1, 40, 2)], k=1)[0] # noqa: C416 + eps = random.sample([i / 1000.0 for i in range(1, 101, 2)], k=1)[0] + scale_factor = random.sample([i / 10.0 for i in range(10, 181, 5)], k=1)[0] + + if random.random() > 0.5: + np_img = self._apply_gaussian_blur(np_img, ksize=int(blur_strength), sigma_x=blur_strength / 2) + + if random.random() > 0.5: + np_img = self._apply_guided_filter(np_img, radius, eps, int(scale_factor)) + + np_img = cv2.resize( + np_img, (int(resize_w / scale_factor), int(resize_h / scale_factor)), interpolation=cv2.INTER_AREA + ) + np_img = cv2.resize(np_img, (resize_w, resize_h), interpolation=cv2.INTER_CUBIC) + return np_img + + def _super_resample(self, np_img: np.ndarray[Any, Any]): + height, width, _ = np_img.shape + ratio = np.sqrt(1024.0 * 1024.0 / (width * height)) + resize_w, resize_h = int(width * ratio) // 48 * 48, int(height * ratio) // 48 * 48 + np_img = cv2.resize(np_img, (resize_w, resize_h)) return np_img def run_processor(self, image: Image.Image) -> Image.Image: np_img = np.array(image, dtype=np.uint8) - processed_np_image = self.tile_resample( - np_img, - # res=self.tile_size, - down_sampling_rate=self.down_sampling_rate, - ) + + if self.mode == "regular": + processed_np_image = self._regular_resample(np_img) + elif self.mode == "blur": + processed_np_image = self._blur_resample(np_img) + elif self.mode == "super": + processed_np_image = self._super_resample(np_img) + else: + raise ValueError(f"Invalid mode: {self.mode}") + processed_image = Image.fromarray(processed_np_image) return processed_image diff --git a/invokeai/backend/image_util/fast_guided_filter/fast_guided_filter.py b/invokeai/backend/image_util/fast_guided_filter/fast_guided_filter.py new file mode 100644 index 00000000000..2073bf87b84 --- /dev/null +++ b/invokeai/backend/image_util/fast_guided_filter/fast_guided_filter.py @@ -0,0 +1,283 @@ +# ruff: noqa: E741 +# -*- coding: utf-8 -*- +## @package guided_filter.core.filters +# +# Implementation of guided filter. +# * GuidedFilter: Original guided filter. +# * FastGuidedFilter: Fast version of the guided filter. +# @author tody +# @date 2015/08/26 + + +import cv2 +import numpy as np + + +## Convert image into float32 type. +def to32F(img): + if img.dtype == np.float32: + return img + return (1.0 / 255.0) * np.float32(img) + + +## Convert image into uint8 type. +def to8U(img): + if img.dtype == np.uint8: + return img + return np.clip(np.uint8(255.0 * img), 0, 255) + + +## Return if the input image is gray or not. +def _isGray(I): + return len(I.shape) == 2 + + +## Return down sampled image. +# @param scale (w/s, h/s) image will be created. +# @param shape I.shape[:2]=(h, w). numpy friendly size parameter. +def _downSample(I, scale=4, shape=None): + if shape is not None: + h, w = shape + return cv2.resize(I, (w, h), interpolation=cv2.INTER_NEAREST) + + h, w = I.shape[:2] + return cv2.resize(I, (int(w / scale), int(h / scale)), interpolation=cv2.INTER_NEAREST) + + +## Return up sampled image. +# @param scale (w*s, h*s) image will be created. +# @param shape I.shape[:2]=(h, w). numpy friendly size parameter. +def _upSample(I, scale=2, shape=None): + if shape is not None: + h, w = shape + return cv2.resize(I, (w, h), interpolation=cv2.INTER_LINEAR) + + h, w = I.shape[:2] + return cv2.resize(I, (int(w * scale), int(h * scale)), interpolation=cv2.INTER_LINEAR) + + +## Fast guide filter. +class FastGuidedFilter: + ## Constructor. + # @param I Input guidance image. Color or gray. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + # @param scale Down sampled scale. + def __init__(self, I, radius=5, epsilon=0.4, scale=4): + I_32F = to32F(I) + self._I = I_32F + h, w = I.shape[:2] + + I_sub = _downSample(I_32F, scale) + + self._I_sub = I_sub + radius = int(radius / scale) + + if _isGray(I): + self._guided_filter = GuidedFilterGray(I_sub, radius, epsilon) + else: + self._guided_filter = GuidedFilterColor(I_sub, radius, epsilon) + + ## Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + p_32F = to32F(p) + shape_original = p.shape[:2] + + p_sub = _downSample(p_32F, shape=self._I_sub.shape[:2]) + + if _isGray(p_sub): + return self._filterGray(p_sub, shape_original) + + cs = p.shape[2] + q = np.array(p_32F) + + for ci in range(cs): + q[:, :, ci] = self._filterGray(p_sub[:, :, ci], shape_original) + return to8U(q) + + def _filterGray(self, p_sub, shape_original): + ab_sub = self._guided_filter._computeCoefficients(p_sub) + ab = [_upSample(abi, shape=shape_original) for abi in ab_sub] + return self._guided_filter._computeOutput(ab, self._I) + + +## Guide filter. +class GuidedFilter: + ## Constructor. + # @param I Input guidance image. Color or gray. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + def __init__(self, I, radius=5, epsilon=0.4): + I_32F = to32F(I) + + if _isGray(I): + self._guided_filter = GuidedFilterGray(I_32F, radius, epsilon) + else: + self._guided_filter = GuidedFilterColor(I_32F, radius, epsilon) + + ## Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + return to8U(self._guided_filter.filter(p)) + + +## Common parts of guided filter. +# +# This class is used by guided_filter class. GuidedFilterGray and GuidedFilterColor. +# Based on guided_filter._computeCoefficients, guided_filter._computeOutput, +# GuidedFilterCommon.filter computes filtered image for color and gray. +class GuidedFilterCommon: + def __init__(self, guided_filter): + self._guided_filter = guided_filter + + ## Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + p_32F = to32F(p) + if _isGray(p_32F): + return self._filterGray(p_32F) + + cs = p.shape[2] + q = np.array(p_32F) + + for ci in range(cs): + q[:, :, ci] = self._filterGray(p_32F[:, :, ci]) + return q + + def _filterGray(self, p): + ab = self._guided_filter._computeCoefficients(p) + return self._guided_filter._computeOutput(ab, self._guided_filter._I) + + +## Guided filter for gray guidance image. +class GuidedFilterGray: + # @param I Input gray guidance image. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + def __init__(self, I, radius=5, epsilon=0.4): + self._radius = 2 * radius + 1 + self._epsilon = epsilon + self._I = to32F(I) + self._initFilter() + self._filter_common = GuidedFilterCommon(self) + + ## Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + return self._filter_common.filter(p) + + def _initFilter(self): + I = self._I + r = self._radius + self._I_mean = cv2.blur(I, (r, r)) + I_mean_sq = cv2.blur(I**2, (r, r)) + self._I_var = I_mean_sq - self._I_mean**2 + + def _computeCoefficients(self, p): + r = self._radius + p_mean = cv2.blur(p, (r, r)) + p_cov = p_mean - self._I_mean * p_mean + a = p_cov / (self._I_var + self._epsilon) + b = p_mean - a * self._I_mean + a_mean = cv2.blur(a, (r, r)) + b_mean = cv2.blur(b, (r, r)) + return a_mean, b_mean + + def _computeOutput(self, ab, I): + a_mean, b_mean = ab + return a_mean * I + b_mean + + +## Guided filter for color guidance image. +class GuidedFilterColor: + # @param I Input color guidance image. + # @param radius Radius of Guided Filter. + # @param epsilon Regularization term of Guided Filter. + def __init__(self, I, radius=5, epsilon=0.2): + self._radius = 2 * radius + 1 + self._epsilon = epsilon + self._I = to32F(I) + self._initFilter() + self._filter_common = GuidedFilterCommon(self) + + ## Apply filter for the input image. + # @param p Input image for the filtering. + def filter(self, p): + return self._filter_common.filter(p) + + def _initFilter(self): + I = self._I + r = self._radius + eps = self._epsilon + + Ir, Ig, Ib = I[:, :, 0], I[:, :, 1], I[:, :, 2] + + self._Ir_mean = cv2.blur(Ir, (r, r)) + self._Ig_mean = cv2.blur(Ig, (r, r)) + self._Ib_mean = cv2.blur(Ib, (r, r)) + + Irr_var = cv2.blur(Ir**2, (r, r)) - self._Ir_mean**2 + eps + Irg_var = cv2.blur(Ir * Ig, (r, r)) - self._Ir_mean * self._Ig_mean + Irb_var = cv2.blur(Ir * Ib, (r, r)) - self._Ir_mean * self._Ib_mean + Igg_var = cv2.blur(Ig * Ig, (r, r)) - self._Ig_mean * self._Ig_mean + eps + Igb_var = cv2.blur(Ig * Ib, (r, r)) - self._Ig_mean * self._Ib_mean + Ibb_var = cv2.blur(Ib * Ib, (r, r)) - self._Ib_mean * self._Ib_mean + eps + + Irr_inv = Igg_var * Ibb_var - Igb_var * Igb_var + Irg_inv = Igb_var * Irb_var - Irg_var * Ibb_var + Irb_inv = Irg_var * Igb_var - Igg_var * Irb_var + Igg_inv = Irr_var * Ibb_var - Irb_var * Irb_var + Igb_inv = Irb_var * Irg_var - Irr_var * Igb_var + Ibb_inv = Irr_var * Igg_var - Irg_var * Irg_var + + I_cov = Irr_inv * Irr_var + Irg_inv * Irg_var + Irb_inv * Irb_var + Irr_inv /= I_cov + Irg_inv /= I_cov + Irb_inv /= I_cov + Igg_inv /= I_cov + Igb_inv /= I_cov + Ibb_inv /= I_cov + + self._Irr_inv = Irr_inv + self._Irg_inv = Irg_inv + self._Irb_inv = Irb_inv + self._Igg_inv = Igg_inv + self._Igb_inv = Igb_inv + self._Ibb_inv = Ibb_inv + + def _computeCoefficients(self, p): + r = self._radius + I = self._I + Ir, Ig, Ib = I[:, :, 0], I[:, :, 1], I[:, :, 2] + + p_mean = cv2.blur(p, (r, r)) + + Ipr_mean = cv2.blur(Ir * p, (r, r)) + Ipg_mean = cv2.blur(Ig * p, (r, r)) + Ipb_mean = cv2.blur(Ib * p, (r, r)) + + Ipr_cov = Ipr_mean - self._Ir_mean * p_mean + Ipg_cov = Ipg_mean - self._Ig_mean * p_mean + Ipb_cov = Ipb_mean - self._Ib_mean * p_mean + + ar = self._Irr_inv * Ipr_cov + self._Irg_inv * Ipg_cov + self._Irb_inv * Ipb_cov + ag = self._Irg_inv * Ipr_cov + self._Igg_inv * Ipg_cov + self._Igb_inv * Ipb_cov + ab = self._Irb_inv * Ipr_cov + self._Igb_inv * Ipg_cov + self._Ibb_inv * Ipb_cov + b = p_mean - ar * self._Ir_mean - ag * self._Ig_mean - ab * self._Ib_mean + + ar_mean = cv2.blur(ar, (r, r)) + ag_mean = cv2.blur(ag, (r, r)) + ab_mean = cv2.blur(ab, (r, r)) + b_mean = cv2.blur(b, (r, r)) + + return ar_mean, ag_mean, ab_mean, b_mean + + def _computeOutput(self, ab, I): + ar_mean, ag_mean, ab_mean, b_mean = ab + + Ir, Ig, Ib = I[:, :, 0], I[:, :, 1], I[:, :, 2] + + q = ar_mean * Ir + ag_mean * Ig + ab_mean * Ib + b_mean + + return q diff --git a/invokeai/backend/model_manager/probe.py b/invokeai/backend/model_manager/probe.py index 2f18f1a8a60..286eb3617c8 100644 --- a/invokeai/backend/model_manager/probe.py +++ b/invokeai/backend/model_manager/probe.py @@ -312,9 +312,11 @@ def _get_checkpoint_config_path( config_file = ( "stable-diffusion/v1-inference.yaml" if base_type is BaseModelType.StableDiffusion1 - else "stable-diffusion/sd_xl_base.yaml" - if base_type is BaseModelType.StableDiffusionXL - else "stable-diffusion/v2-inference.yaml" + else ( + "stable-diffusion/sd_xl_base.yaml" + if base_type is BaseModelType.StableDiffusionXL + else "stable-diffusion/v2-inference.yaml" + ) ) else: raise InvalidModelConfigException( @@ -361,6 +363,7 @@ def _scan_model(cls, model_name: str, checkpoint: Path) -> None: "pose": "dw_openpose_image_processor", "mediapipe": "mediapipe_face_processor", "pidi": "pidi_image_processor", + "tile": "tile_image_processor", "zoe": "zoe_depth_image_processor", "color": "color_map_image_processor", } diff --git a/invokeai/frontend/web/public/locales/en.json b/invokeai/frontend/web/public/locales/en.json index 6d3829bc718..4b2fc5326f7 100644 --- a/invokeai/frontend/web/public/locales/en.json +++ b/invokeai/frontend/web/public/locales/en.json @@ -231,6 +231,13 @@ "dwOpenposeDescription": "Human pose estimation using DW Openpose", "pidi": "PIDI", "pidiDescription": "PIDI image processing", + "tile": "Tile", + "TileDescription": "Tile Resampling", + "mode": "Mode", + "downsamplingRate": "Downsampling Rate", + "regular": "Regular", + "blur": "Blur", + "super": "Super", "processor": "Processor", "prompt": "Prompt", "resetControlImage": "Reset Control Image", diff --git a/invokeai/frontend/web/src/features/controlAdapters/store/constants.ts b/invokeai/frontend/web/src/features/controlAdapters/store/constants.ts index 152e977e5c6..1a40be4cdc3 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/store/constants.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/store/constants.ts @@ -245,6 +245,21 @@ export const CONTROLNET_PROCESSORS: ControlNetProcessorsDict = { safe: false, }), }, + tile_image_processor: { + type: 'tile_image_processor', + get label() { + return i18n.t('controlnet.tile'); + }, + get description() { + return i18n.t('controlnet.tileDescription'); + }, + buildDefaults: () => ({ + id: 'tile_image_processor', + type: 'tile_image_processor', + down_sampling_rate: 1.0, + mode: 'blur', + }), + }, zoe_depth_image_processor: { type: 'zoe_depth_image_processor', get label() { diff --git a/invokeai/frontend/web/src/features/controlAdapters/store/types.ts b/invokeai/frontend/web/src/features/controlAdapters/store/types.ts index b76a729263e..1e70a610bf2 100644 --- a/invokeai/frontend/web/src/features/controlAdapters/store/types.ts +++ b/invokeai/frontend/web/src/features/controlAdapters/store/types.ts @@ -26,6 +26,7 @@ export type ControlAdapterProcessorNode = | Invocation<'normalbae_image_processor'> | Invocation<'dw_openpose_image_processor'> | Invocation<'pidi_image_processor'> + | Invocation<'tile_image_processor'> | Invocation<'zoe_depth_image_processor'>; /** @@ -46,6 +47,7 @@ export const zControlAdapterProcessorType = z.enum([ 'normalbae_image_processor', 'dw_openpose_image_processor', 'pidi_image_processor', + 'tile_image_processor', 'zoe_depth_image_processor', 'none', ]); @@ -161,6 +163,14 @@ export type RequiredPidiImageProcessorInvocation = O.Required< 'type' | 'detect_resolution' | 'image_resolution' | 'safe' | 'scribble' >; +/** + * The Tile processor node, with parameters flagged as required + */ +export type RequiredTileImageProcessorInvocation = O.Required< + Invocation<'tile_image_processor'>, + 'type' | 'down_sampling_rate' | 'mode' +>; + /** * The ZoeDepth processor node, with parameters flagged as required */ @@ -184,6 +194,7 @@ export type RequiredControlAdapterProcessorNode = | RequiredNormalbaeImageProcessorInvocation | RequiredDWOpenposeImageProcessorInvocation | RequiredPidiImageProcessorInvocation + | RequiredTileImageProcessorInvocation | RequiredZoeDepthImageProcessorInvocation, 'id' > @@ -199,6 +210,10 @@ const zIPMethod = z.enum(['full', 'style', 'composition']); export type IPMethod = z.infer; export const isIPMethod = (v: unknown): v is IPMethod => zIPMethod.safeParse(v).success; +const zTileProcessorMode = z.enum(['regular', 'blur', 'var', 'super']); +export type TileProcessorMode = z.infer; +export const isTileProcessorMode = (v: unknown): v is TileProcessorMode => zTileProcessorMode.safeParse(v).success; + export type ControlNetConfig = { type: 'controlnet'; id: string; diff --git a/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterProcessorConfig.tsx b/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterProcessorConfig.tsx index 034dc5454e7..83b2371c8ef 100644 --- a/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterProcessorConfig.tsx +++ b/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/ControlAdapterProcessorConfig.tsx @@ -12,6 +12,7 @@ import { MediapipeFaceProcessor } from './processors/MediapipeFaceProcessor'; import { MidasDepthProcessor } from './processors/MidasDepthProcessor'; import { MlsdImageProcessor } from './processors/MlsdImageProcessor'; import { PidiProcessor } from './processors/PidiProcessor'; +import { TileProcessor } from './processors/TileProcessor'; type Props = { config: ProcessorConfig | null; @@ -77,6 +78,10 @@ export const ControlAdapterProcessorConfig = memo(({ config, onChange }: Props) return ; } + if (config.type === 'tile_image_processor') { + return ; + } + if (config.type === 'zoe_depth_image_processor') { return null; } diff --git a/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/TileProcessor.tsx b/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/TileProcessor.tsx new file mode 100644 index 00000000000..82818a28365 --- /dev/null +++ b/invokeai/frontend/web/src/features/controlLayers/components/ControlAndIPAdapter/processors/TileProcessor.tsx @@ -0,0 +1,75 @@ +import type { ComboboxOnChange } from '@invoke-ai/ui-library'; +import { Combobox, CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library'; +import { isTileProcessorMode, type TileProcessorMode } from 'features/controlAdapters/store/types'; +import type { ProcessorComponentProps } from 'features/controlLayers/components/ControlAndIPAdapter/processors/types'; +import { CA_PROCESSOR_DATA, type TileProcessorConfig } from 'features/controlLayers/util/controlAdapters'; +import { memo, useCallback, useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; + +import ProcessorWrapper from './ProcessorWrapper'; + +type Props = ProcessorComponentProps; +const DEFAULTS = CA_PROCESSOR_DATA['tile_image_processor'].buildDefaults(); + +export const TileProcessor = memo(({ onChange, config }: Props) => { + const { t } = useTranslation(); + + const tileModeOptions: { label: string; value: TileProcessorMode }[] = useMemo( + () => [ + { label: t('controlnet.regular'), value: 'regular' }, + { label: t('controlnet.blur'), value: 'blur' }, + { label: t('controlnet.super'), value: 'super' }, + ], + [t] + ); + + const tileModeValue = useMemo(() => tileModeOptions.find((o) => o.value === config.mode), [tileModeOptions, config]); + + const handleTileModeChange = useCallback( + (v) => { + if (!isTileProcessorMode(v?.value)) { + return; + } + onChange({ ...config, mode: v.value }); + }, + [config, onChange] + ); + + const handleDownSamplingRateChanged = useCallback( + (v: number) => { + onChange({ ...config, down_sampling_rate: v }); + }, + [config, onChange] + ); + + return ( + + + {t('controlnet.mode')} + + + + {t('controlnet.downsamplingRate')} + + + + + ); +}); + +TileProcessor.displayName = 'TileProcessor'; diff --git a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.test.ts b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.test.ts index 22f54d622cd..ebdc7b5b6dc 100644 --- a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.test.ts +++ b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.test.ts @@ -23,6 +23,7 @@ import type { PidiProcessorConfig, ProcessorConfig, ProcessorTypeV2, + TileProcessorConfig, ZoeDepthProcessorConfig, } from './controlAdapters'; @@ -58,6 +59,7 @@ describe('Control Adapter Types', () => { assert>(); assert>(); assert>(); + assert>(); assert>(); }); }); @@ -90,4 +92,7 @@ type _DWOpenposeProcessorConfig = Required< Pick, 'id' | 'type' | 'draw_body' | 'draw_face' | 'draw_hands'> >; type _PidiProcessorConfig = Required, 'id' | 'type' | 'safe' | 'scribble'>>; +type _TileProcessorConfig = Required< + Pick, 'id' | 'type' | 'down_sampling_rate' | 'mode'> +>; type _ZoeDepthProcessorConfig = Required, 'id' | 'type'>>; diff --git a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts index 13435bdb7cf..2715fa3ee8b 100644 --- a/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts +++ b/invokeai/frontend/web/src/features/controlLayers/util/controlAdapters.ts @@ -114,6 +114,14 @@ const zPidiProcessorConfig = z.object({ }); export type PidiProcessorConfig = z.infer; +const zTileProcessorConfig = z.object({ + id: zId, + type: z.literal('tile_image_processor'), + down_sampling_rate: z.number().gte(0), + mode: z.enum(['regular', 'blur', 'var', 'super']), +}); +export type TileProcessorConfig = z.infer; + const zZoeDepthProcessorConfig = z.object({ id: zId, type: z.literal('zoe_depth_image_processor'), @@ -134,6 +142,7 @@ const zProcessorConfig = z.discriminatedUnion('type', [ zNormalbaeProcessorConfig, zDWOpenposeProcessorConfig, zPidiProcessorConfig, + zTileProcessorConfig, zZoeDepthProcessorConfig, ]); export type ProcessorConfig = z.infer; @@ -212,6 +221,7 @@ const zProcessorTypeV2 = z.enum([ 'normalbae_image_processor', 'dw_openpose_image_processor', 'pidi_image_processor', + 'tile_image_processor', 'zoe_depth_image_processor', ]); export type ProcessorTypeV2 = z.infer; @@ -453,6 +463,22 @@ export const CA_PROCESSOR_DATA: CAProcessorsData = { image_resolution: minDim(image), }), }, + tile_image_processor: { + type: 'tile_image_processor', + labelTKey: 'controlnet.tile', + descriptionTKey: 'controlnet.tileDescription', + buildDefaults: () => ({ + id: 'tile_image_processor', + type: 'tile_image_processor', + down_sampling_rate: 1.0, + mode: 'regular', + }), + buildNode: (image, config) => ({ + ...config, + type: 'tile_image_processor', + image: { image_name: image.name }, + }), + }, zoe_depth_image_processor: { type: 'zoe_depth_image_processor', labelTKey: 'controlnet.depthZoe', diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/DefaultPreprocessor.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/DefaultPreprocessor.tsx index b2284336bf4..a7fae3062c1 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/DefaultPreprocessor.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/DefaultPreprocessor.tsx @@ -15,6 +15,7 @@ const OPTIONS = [ { label: 'Depth Anything', value: 'depth_anything_image_processor' }, { label: 'Normal BAE', value: 'normalbae_image_processor' }, { label: 'Pidi', value: 'pidi_image_processor' }, + { label: 'Tile', value: 'tile_image_processor' }, { label: 'Lineart', value: 'lineart_image_processor' }, { label: 'Lineart Anime', value: 'lineart_anime_image_processor' }, { label: 'HED', value: 'hed_image_processor' }, diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index 07f1cdb34b3..8aac236aa87 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -7293,145 +7293,145 @@ export type components = { project_id: string | null; }; InvocationOutputMap: { - img_channel_offset: components["schemas"]["ImageOutput"]; - create_denoise_mask: components["schemas"]["DenoiseMaskOutput"]; - round_float: components["schemas"]["FloatOutput"]; - img_blur: components["schemas"]["ImageOutput"]; - sdxl_model_loader: components["schemas"]["SDXLModelLoaderOutput"]; - lblend: components["schemas"]["LatentsOutput"]; - img_lerp: components["schemas"]["ImageOutput"]; - zoe_depth_image_processor: components["schemas"]["ImageOutput"]; + image_mask_to_tensor: components["schemas"]["MaskOutput"]; range_of_size: components["schemas"]["IntegerCollectionOutput"]; - freeu: components["schemas"]["UNetOutput"]; - alpha_mask_to_tensor: components["schemas"]["MaskOutput"]; - latents: components["schemas"]["LatentsOutput"]; - controlnet: components["schemas"]["ControlOutput"]; - canny_image_processor: components["schemas"]["ImageOutput"]; - color_map_image_processor: components["schemas"]["ImageOutput"]; - l2i: components["schemas"]["ImageOutput"]; - string_join_three: components["schemas"]["StringOutput"]; - crop_latents: components["schemas"]["LatentsOutput"]; - img_resize: components["schemas"]["ImageOutput"]; - img_crop: components["schemas"]["ImageOutput"]; - face_off: components["schemas"]["FaceOffOutput"]; - seamless: components["schemas"]["SeamlessModeOutput"]; - string: components["schemas"]["StringOutput"]; - img_watermark: components["schemas"]["ImageOutput"]; - img_paste: components["schemas"]["ImageOutput"]; + calculate_image_tiles_even_split: components["schemas"]["CalculateImageTilesOutput"]; ideal_size: components["schemas"]["IdealSizeOutput"]; - img_scale: components["schemas"]["ImageOutput"]; - mul: components["schemas"]["IntegerOutput"]; - conditioning: components["schemas"]["ConditioningOutput"]; - add: components["schemas"]["IntegerOutput"]; + infill_tile: components["schemas"]["ImageOutput"]; + img_mul: components["schemas"]["ImageOutput"]; + img_blur: components["schemas"]["ImageOutput"]; + create_denoise_mask: components["schemas"]["DenoiseMaskOutput"]; + freeu: components["schemas"]["UNetOutput"]; + mediapipe_face_processor: components["schemas"]["ImageOutput"]; + dw_openpose_image_processor: components["schemas"]["ImageOutput"]; conditioning_collection: components["schemas"]["ConditioningCollectionOutput"]; - create_gradient_mask: components["schemas"]["GradientMaskOutput"]; - float: components["schemas"]["FloatOutput"]; - sdxl_refiner_compel_prompt: components["schemas"]["ConditioningOutput"]; - string_collection: components["schemas"]["StringCollectionOutput"]; - image_mask_to_tensor: components["schemas"]["MaskOutput"]; - infill_cv2: components["schemas"]["ImageOutput"]; - boolean_collection: components["schemas"]["BooleanCollectionOutput"]; - color: components["schemas"]["ColorOutput"]; - lresize: components["schemas"]["LatentsOutput"]; + round_float: components["schemas"]["FloatOutput"]; + img_paste: components["schemas"]["ImageOutput"]; + integer_math: components["schemas"]["IntegerOutput"]; + invert_tensor_mask: components["schemas"]["MaskOutput"]; + img_channel_offset: components["schemas"]["ImageOutput"]; + infill_patchmatch: components["schemas"]["ImageOutput"]; image: components["schemas"]["ImageOutput"]; + range: components["schemas"]["IntegerCollectionOutput"]; + sub: components["schemas"]["IntegerOutput"]; + mask_from_id: components["schemas"]["ImageOutput"]; + latents_collection: components["schemas"]["LatentsCollectionOutput"]; + merge_tiles_to_image: components["schemas"]["ImageOutput"]; + hed_image_processor: components["schemas"]["ImageOutput"]; + img_nsfw: components["schemas"]["ImageOutput"]; + integer_collection: components["schemas"]["IntegerCollectionOutput"]; + face_mask_detection: components["schemas"]["FaceMaskOutput"]; + content_shuffle_image_processor: components["schemas"]["ImageOutput"]; + img_ilerp: components["schemas"]["ImageOutput"]; esrgan: components["schemas"]["ImageOutput"]; - image_collection: components["schemas"]["ImageCollectionOutput"]; - metadata: components["schemas"]["MetadataOutput"]; - scheduler: components["schemas"]["SchedulerOutput"]; - img_pad_crop: components["schemas"]["ImageOutput"]; - integer: components["schemas"]["IntegerOutput"]; - boolean: components["schemas"]["BooleanOutput"]; + main_model_loader: components["schemas"]["ModelLoaderOutput"]; + string_join: components["schemas"]["StringOutput"]; + boolean_collection: components["schemas"]["BooleanCollectionOutput"]; float_math: components["schemas"]["FloatOutput"]; - sdxl_compel_prompt: components["schemas"]["ConditioningOutput"]; + sdxl_lora_loader: components["schemas"]["SDXLLoRALoaderOutput"]; + segment_anything_processor: components["schemas"]["ImageOutput"]; + step_param_easing: components["schemas"]["FloatCollectionOutput"]; + save_image: components["schemas"]["ImageOutput"]; + float_collection: components["schemas"]["FloatCollectionOutput"]; + leres_image_processor: components["schemas"]["ImageOutput"]; tile_image_processor: components["schemas"]["ImageOutput"]; - img_channel_multiply: components["schemas"]["ImageOutput"]; - tomask: components["schemas"]["ImageOutput"]; - sub: components["schemas"]["IntegerOutput"]; - img_mul: components["schemas"]["ImageOutput"]; - string_split: components["schemas"]["String2Output"]; - model_identifier: components["schemas"]["ModelIdentifierOutput"]; + mask_edge: components["schemas"]["ImageOutput"]; + boolean: components["schemas"]["BooleanOutput"]; + rand_float: components["schemas"]["FloatOutput"]; pair_tile_image: components["schemas"]["PairTileImageOutput"]; - heuristic_resize: components["schemas"]["ImageOutput"]; - infill_lama: components["schemas"]["ImageOutput"]; - lineart_anime_image_processor: components["schemas"]["ImageOutput"]; - float_range: components["schemas"]["FloatCollectionOutput"]; - infill_patchmatch: components["schemas"]["ImageOutput"]; - mediapipe_face_processor: components["schemas"]["ImageOutput"]; - cv_inpaint: components["schemas"]["ImageOutput"]; - denoise_latents: components["schemas"]["LatentsOutput"]; - range: components["schemas"]["IntegerCollectionOutput"]; - content_shuffle_image_processor: components["schemas"]["ImageOutput"]; - compel: components["schemas"]["ConditioningOutput"]; - leres_image_processor: components["schemas"]["ImageOutput"]; - prompt_from_file: components["schemas"]["StringCollectionOutput"]; - noise: components["schemas"]["NoiseOutput"]; - i2l: components["schemas"]["LatentsOutput"]; - pidi_image_processor: components["schemas"]["ImageOutput"]; - merge_metadata: components["schemas"]["MetadataOutput"]; - img_conv: components["schemas"]["ImageOutput"]; - ip_adapter: components["schemas"]["IPAdapterOutput"]; - div: components["schemas"]["IntegerOutput"]; - dw_openpose_image_processor: components["schemas"]["ImageOutput"]; - rand_int: components["schemas"]["IntegerOutput"]; - calculate_image_tiles_even_split: components["schemas"]["CalculateImageTilesOutput"]; - show_image: components["schemas"]["ImageOutput"]; - calculate_image_tiles_min_overlap: components["schemas"]["CalculateImageTilesOutput"]; - float_to_int: components["schemas"]["IntegerOutput"]; - infill_rgba: components["schemas"]["ImageOutput"]; - face_mask_detection: components["schemas"]["FaceMaskOutput"]; - sdxl_lora_collection_loader: components["schemas"]["SDXLLoRALoaderOutput"]; - iterate: components["schemas"]["IterateInvocationOutput"]; - rectangle_mask: components["schemas"]["MaskOutput"]; - merge_tiles_to_image: components["schemas"]["ImageOutput"]; + model_identifier: components["schemas"]["ModelIdentifierOutput"]; lineart_image_processor: components["schemas"]["ImageOutput"]; - lora_collection_loader: components["schemas"]["LoRALoaderOutput"]; - lora_selector: components["schemas"]["LoRASelectorOutput"]; midas_depth_image_processor: components["schemas"]["ImageOutput"]; - face_identifier: components["schemas"]["ImageOutput"]; - save_image: components["schemas"]["ImageOutput"]; - unsharp_mask: components["schemas"]["ImageOutput"]; + tiled_multi_diffusion_denoise_latents: components["schemas"]["LatentsOutput"]; + img_conv: components["schemas"]["ImageOutput"]; + color: components["schemas"]["ColorOutput"]; string_split_neg: components["schemas"]["StringPosNegOutput"]; - img_chan: components["schemas"]["ImageOutput"]; - float_collection: components["schemas"]["FloatCollectionOutput"]; - lora_loader: components["schemas"]["LoRALoaderOutput"]; - random_range: components["schemas"]["IntegerCollectionOutput"]; - invert_tensor_mask: components["schemas"]["MaskOutput"]; - clip_skip: components["schemas"]["CLIPSkipInvocationOutput"]; - sdxl_lora_loader: components["schemas"]["SDXLLoRALoaderOutput"]; - string_join: components["schemas"]["StringOutput"]; - img_hue_adjust: components["schemas"]["ImageOutput"]; - img_nsfw: components["schemas"]["ImageOutput"]; - tile_to_properties: components["schemas"]["TileToPropertiesOutput"]; - calculate_image_tiles: components["schemas"]["CalculateImageTilesOutput"]; - mask_combine: components["schemas"]["ImageOutput"]; - main_model_loader: components["schemas"]["ModelLoaderOutput"]; - img_ilerp: components["schemas"]["ImageOutput"]; - string_replace: components["schemas"]["StringOutput"]; + noise: components["schemas"]["NoiseOutput"]; + compel: components["schemas"]["ConditioningOutput"]; + cv_inpaint: components["schemas"]["ImageOutput"]; + scheduler: components["schemas"]["SchedulerOutput"]; + normalbae_image_processor: components["schemas"]["ImageOutput"]; + unsharp_mask: components["schemas"]["ImageOutput"]; + lora_collection_loader: components["schemas"]["LoRALoaderOutput"]; + i2l: components["schemas"]["LatentsOutput"]; sdxl_refiner_model_loader: components["schemas"]["SDXLRefinerModelLoaderOutput"]; - hed_image_processor: components["schemas"]["ImageOutput"]; - latents_collection: components["schemas"]["LatentsCollectionOutput"]; - infill_tile: components["schemas"]["ImageOutput"]; - vae_loader: components["schemas"]["VAEOutput"]; - depth_anything_image_processor: components["schemas"]["ImageOutput"]; + calculate_image_tiles_min_overlap: components["schemas"]["CalculateImageTilesOutput"]; + img_watermark: components["schemas"]["ImageOutput"]; + string: components["schemas"]["StringOutput"]; lscale: components["schemas"]["LatentsOutput"]; - t2i_adapter: components["schemas"]["T2IAdapterOutput"]; + sdxl_lora_collection_loader: components["schemas"]["SDXLLoRALoaderOutput"]; + pidi_image_processor: components["schemas"]["ImageOutput"]; + metadata: components["schemas"]["MetadataOutput"]; + l2i: components["schemas"]["ImageOutput"]; + alpha_mask_to_tensor: components["schemas"]["MaskOutput"]; + clip_skip: components["schemas"]["CLIPSkipInvocationOutput"]; + string_join_three: components["schemas"]["StringOutput"]; + show_image: components["schemas"]["ImageOutput"]; metadata_item: components["schemas"]["MetadataItemOutput"]; - blank_image: components["schemas"]["ImageOutput"]; - tiled_multi_diffusion_denoise_latents: components["schemas"]["LatentsOutput"]; - canvas_paste_back: components["schemas"]["ImageOutput"]; - rand_float: components["schemas"]["FloatOutput"]; - mask_from_id: components["schemas"]["ImageOutput"]; - segment_anything_processor: components["schemas"]["ImageOutput"]; - normalbae_image_processor: components["schemas"]["ImageOutput"]; - mask_edge: components["schemas"]["ImageOutput"]; + infill_lama: components["schemas"]["ImageOutput"]; dynamic_prompt: components["schemas"]["StringCollectionOutput"]; + mask_combine: components["schemas"]["ImageOutput"]; + create_gradient_mask: components["schemas"]["GradientMaskOutput"]; + face_identifier: components["schemas"]["ImageOutput"]; + rand_int: components["schemas"]["IntegerOutput"]; + tile_to_properties: components["schemas"]["TileToPropertiesOutput"]; + controlnet: components["schemas"]["ControlOutput"]; + blank_image: components["schemas"]["ImageOutput"]; + latents: components["schemas"]["LatentsOutput"]; + div: components["schemas"]["IntegerOutput"]; + lresize: components["schemas"]["LatentsOutput"]; + seamless: components["schemas"]["SeamlessModeOutput"]; + img_crop: components["schemas"]["ImageOutput"]; + float: components["schemas"]["FloatOutput"]; + mul: components["schemas"]["IntegerOutput"]; + heuristic_resize: components["schemas"]["ImageOutput"]; + infill_rgba: components["schemas"]["ImageOutput"]; + lblend: components["schemas"]["LatentsOutput"]; + string_collection: components["schemas"]["StringCollectionOutput"]; + prompt_from_file: components["schemas"]["StringCollectionOutput"]; + t2i_adapter: components["schemas"]["T2IAdapterOutput"]; + denoise_latents: components["schemas"]["LatentsOutput"]; + lineart_anime_image_processor: components["schemas"]["ImageOutput"]; color_correct: components["schemas"]["ImageOutput"]; - integer_math: components["schemas"]["IntegerOutput"]; + canvas_paste_back: components["schemas"]["ImageOutput"]; + lora_selector: components["schemas"]["LoRASelectorOutput"]; + sdxl_refiner_compel_prompt: components["schemas"]["ConditioningOutput"]; + float_to_int: components["schemas"]["IntegerOutput"]; + img_lerp: components["schemas"]["ImageOutput"]; + rectangle_mask: components["schemas"]["MaskOutput"]; core_metadata: components["schemas"]["MetadataOutput"]; - integer_collection: components["schemas"]["IntegerCollectionOutput"]; + infill_cv2: components["schemas"]["ImageOutput"]; + sdxl_compel_prompt: components["schemas"]["ConditioningOutput"]; + vae_loader: components["schemas"]["VAEOutput"]; + zoe_depth_image_processor: components["schemas"]["ImageOutput"]; + depth_anything_image_processor: components["schemas"]["ImageOutput"]; + img_resize: components["schemas"]["ImageOutput"]; + integer: components["schemas"]["IntegerOutput"]; + tomask: components["schemas"]["ImageOutput"]; + float_range: components["schemas"]["FloatCollectionOutput"]; + random_range: components["schemas"]["IntegerCollectionOutput"]; collect: components["schemas"]["CollectInvocationOutput"]; + image_collection: components["schemas"]["ImageCollectionOutput"]; + string_split: components["schemas"]["String2Output"]; + calculate_image_tiles: components["schemas"]["CalculateImageTilesOutput"]; + img_pad_crop: components["schemas"]["ImageOutput"]; + img_channel_multiply: components["schemas"]["ImageOutput"]; + sdxl_model_loader: components["schemas"]["SDXLModelLoaderOutput"]; mlsd_image_processor: components["schemas"]["ImageOutput"]; - step_param_easing: components["schemas"]["FloatCollectionOutput"]; + img_hue_adjust: components["schemas"]["ImageOutput"]; + merge_metadata: components["schemas"]["MetadataOutput"]; + crop_latents: components["schemas"]["LatentsOutput"]; + canny_image_processor: components["schemas"]["ImageOutput"]; + lora_loader: components["schemas"]["LoRALoaderOutput"]; + add: components["schemas"]["IntegerOutput"]; + string_replace: components["schemas"]["StringOutput"]; + color_map_image_processor: components["schemas"]["ImageOutput"]; + img_scale: components["schemas"]["ImageOutput"]; + face_off: components["schemas"]["FaceOffOutput"]; + iterate: components["schemas"]["IterateInvocationOutput"]; + conditioning: components["schemas"]["ConditioningOutput"]; + ip_adapter: components["schemas"]["IPAdapterOutput"]; + img_chan: components["schemas"]["ImageOutput"]; }; /** * InvocationStartedEvent @@ -12823,6 +12823,13 @@ export type components = { * @default 1 */ down_sampling_rate?: number; + /** + * Mode + * @description The Tile ControlNet pre-processing mode to use. + * @default regular + * @enum {string} + */ + mode?: "regular" | "blur" | "super"; /** * type * @default tile_image_processor