custom_nodes/ComfyUI_smZNodes/modules/sd_hijack_unet.py

import torch
from packaging import version

from . import devices
from .sd_hijack_utils import CondFunc
from torch.nn.functional import silu
import comfy
from comfy import ldm
import contextlib

class TorchHijackForUnet:
    """
    This is torch, but with cat that resizes tensors to appropriate dimensions if they do not match;
    this makes it possible to create pictures with dimensions that are multiples of 8 rather than 64
    """

    def __getattr__(self, item):
        if item == 'cat':
            return self.cat

        if hasattr(torch, item):
            return getattr(torch, item)

        raise AttributeError(f"'{type(self).__name__}' object has no attribute '{item}'")

    def cat(self, tensors, *args, **kwargs):
        if len(tensors) == 2:
            a, b = tensors
            if a.shape[-2:] != b.shape[-2:]:
                a = torch.nn.functional.interpolate(a, b.shape[-2:], mode="nearest")

            tensors = (a, b)

        return torch.cat(tensors, *args, **kwargs)

th = TorchHijackForUnet()

from . import sd_hijack_optimizations
from comfy.model_base import BaseModel
from functools import wraps
sdp_no_mem = sd_hijack_optimizations.SdOptimizationSdpNoMem()
BaseModel.apply_model_orig = BaseModel.apply_model

# @contextmanager
class ApplyOptimizationsContext:
    def __init__(self):
        self.nonlinearity = ldm.modules.diffusionmodules.model.nonlinearity
        self.th = ldm.modules.diffusionmodules.openaimodel.th
        ldm.modules.diffusionmodules.model.nonlinearity = silu
        ldm.modules.diffusionmodules.openaimodel.th = th
        sdp_no_mem.apply()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        ldm.modules.diffusionmodules.model.nonlinearity = self.nonlinearity
        ldm.modules.diffusionmodules.openaimodel.th = self.th
        sd_hijack_optimizations.undo()



def ApplyOptimizationsContext3(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        with ApplyOptimizationsContext():
            return func(*args, **kwargs)
    return wrapper

precision_scope_null = lambda a, dtype=None: contextlib.nullcontext(a)

# def apply_model(orig_func, self, x_noisy, t, c_concat=None, c_crossattn=None, c_adm=None, control=None, transformer_options={}, *args, **kwargs):
def apply_model(orig_func, self, *args, **kwargs):
    transformer_options = kwargs['transformer_options'] if 'transformer_options' in kwargs else {}
    c_crossattn = kwargs['c_crossattn'] if 'c_crossattn' in kwargs else args[3]
    x_noisy = kwargs['x_noisy'] if 'x_noisy' in kwargs else args[0]
    if not transformer_options.get('from_smZ', False):
        return self.apply_model_orig(*args, **kwargs)

    cond=c_crossattn
    if isinstance(cond, dict):
        for y in cond.keys():
            if isinstance(cond[y], list):
                cond[y] = [x.to(devices.dtype_unet) if isinstance(x, torch.Tensor) else x for x in cond[y]]
            else:
                cond[y] = cond[y].to(devices.dtype_unet) if isinstance(cond[y], torch.Tensor) else cond[y]

    if x_noisy.dtype != torch.float32:
        precision_scope = torch.autocast
    else:
        precision_scope = precision_scope_null

    with precision_scope(comfy.model_management.get_autocast_device(x_noisy.device), dtype=x_noisy.dtype): # , torch.float32):
    # with devices.autocast():
        out = orig_func(self, *args, **kwargs).float()
    return out

class GELUHijack(torch.nn.GELU, torch.nn.Module):
    def __init__(self, *args, **kwargs):
        torch.nn.GELU.__init__(self, *args, **kwargs)
    def forward(self, x):
        if devices.unet_needs_upcast:
            return torch.nn.GELU.forward(self.float(), x.float()).to(devices.dtype_unet)
        else:
            return torch.nn.GELU.forward(self, x)

ddpm_edit_hijack = None
def hijack_ddpm_edit():
    global ddpm_edit_hijack
    if not ddpm_edit_hijack:
        CondFunc('modules.models.diffusion.ddpm_edit.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond)
        CondFunc('modules.models.diffusion.ddpm_edit.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond)
        ddpm_edit_hijack = CondFunc('modules.models.diffusion.ddpm_edit.LatentDiffusion.apply_model', apply_model, unet_needs_upcast)


unet_needs_upcast = lambda *args, **kwargs: devices.unet_needs_upcast
# CondFunc('comfy.model_base.BaseModel.apply_model', apply_model, unet_needs_upcast)
# CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast)
# CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast)
# if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available():
#     CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast)
#     CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast)
#     try:
#         CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU)
#     except:
#         CondFunc('comfy.t2i_adapter.adapter.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU)
first_stage_cond = lambda _, self, *args, **kwargs: devices.unet_needs_upcast and self.model.diffusion_model.dtype == torch.float16
first_stage_sub = lambda orig_func, self, x, **kwargs: orig_func(self, x.to(devices.dtype_vae), **kwargs)
# CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond)
# CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond)
# CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.get_first_stage_encoding', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).float(), first_stage_cond)

# CondFunc('sgm.modules.diffusionmodules.wrappers.OpenAIWrapper.forward', apply_model, unet_needs_upcast)
# CondFunc('sgm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast)