import spaces import json import yaml import os from PIL import Image import numpy as np import torch import torchvision.utils as vutils import gradio as gr from huggingface_hub import hf_hub_download from model.pipeline import UJiTModel, JiTConfig from model.config import ClassContextConfig MODEL_REPO = os.environ.get("MODEL_REPO", "p1atdev/JiT-AnimeFace-experiment") MODEL_PATH = os.environ.get( "MODEL_PATH", "ujit-b512-p32-cls-derf/jit-anime_00020e_311840s.safetensors", ) LABEL2ID_PATH = os.environ.get("LABEL2ID_PATH", "ujit-b512-p32-cls-derf/label2id.json") CONFIG_PATH = os.environ.get( "CONFIG_PATH", "ujit-b512-p32-cls-derf/config.yml", ) DEVICE = ( torch.device("cuda") if torch.cuda.is_available() else torch.device("mps") if torch.backends.mps.is_available() else torch.device("cpu") ) DTYPE = torch.bfloat16 if DEVICE.type in ["cuda"] else torch.float16 MAX_TOKEN_LENGTH = 32 MAX_IMAGE_SIZE = 1024 MIN_IMAGE_SIZE = 256 COMMON_NEGATIVE_PROMPT = ( "retro artstyle, speech bubble, doujinshi, comic, title, cover, " "logo, watermark, signature, bad anatomy, sketch, abstract, unfinished, multiple views, " ) model_map: dict[str, UJiTModel] = {} # {model_path: model} label2id_map: dict[str, dict] = {} # {label2id_path: label2id} def images_to_tensor( images: list[Image.Image], dtype: torch.dtype, device: torch.device, ) -> torch.Tensor: # 0~255 -> -1~1 return torch.stack( [ torch.tensor(np.array(image), dtype=dtype, device=device).permute(2, 0, 1) / 127.5 - 1.0 for image in images ] ) @torch.no_grad() def images_to_grid_image( images: list[Image.Image] | torch.Tensor, padding: int = 2, ) -> Image.Image: if isinstance(images, list): tensor_images = images_to_tensor( images, dtype=torch.float16, device=torch.device("cpu"), ) grid = vutils.make_grid( tensor_images, nrow=int(len(tensor_images) ** 0.5), padding=padding, normalize=True, ) # TensorをPIL画像に変換して保存 # (C, H, W) -> (H, W, C) への変換なども自動化できる image = Image.fromarray( grid.mul(255) .add_(0.5) .clamp_(0, 255) .permute(1, 2, 0) .to("cpu", torch.uint8) .numpy() ) return image def get_file_path(repo: str, path: str) -> str: """Hugging Face Hub からファイルを取得""" return hf_hub_download(repo, path) def load_label2id(label2id_path: str) -> dict: """label2id.json を読み込む""" with open(label2id_path, "r") as f: return json.load(f) def load_config(config_path: str) -> JiTConfig: """設定ファイルを読み込む""" with open(config_path, "r") as f: if config_path.endswith(".json"): config_dict = json.load(f) elif config_path.endswith((".yaml", ".yml")): config_dict = yaml.safe_load(f) else: raise ValueError("Unsupported config file format. Use .json or .yaml/.yml") return JiTConfig.model_validate(config_dict) def load_model( model_path: str, label2id_path: str, config_path: str, device: torch.device, dtype: torch.dtype = DTYPE, ) -> tuple[UJiTModel, dict]: """モデルを読み込む""" if model_path in model_map: # use cache model = model_map[model_path] label2id = label2id_map[label2id_path] return model, label2id config = load_config(get_file_path(MODEL_REPO, config_path)) if isinstance(config.context_encoder, ClassContextConfig): config.context_encoder.label2id_map_path = get_file_path( MODEL_REPO, label2id_path ) model = UJiTModel.from_pretrained( config=config, checkpoint_path=get_file_path(MODEL_REPO, model_path), ) model.eval() model.requires_grad_(False) model.to(device=device, dtype=dtype) model_map[model_path] = model # cache label2id = load_label2id(get_file_path(MODEL_REPO, label2id_path)) label2id_map[label2id_path] = label2id # cache return model, label2id @spaces.GPU(duration=15) def generate_images( prompt: str, negative_prompt: str, num_steps: int, cfg_scale: float, batch_size: int, height: int, width: int, seed: int, # model_path: str = MODEL_PATH, label2id_path: str = LABEL2ID_PATH, config_path: str = CONFIG_PATH, progress=gr.Progress(track_tqdm=True), ): model, _label2id = load_model( model_path=model_path, label2id_path=label2id_path, config_path=config_path, device=DEVICE, dtype=DTYPE, ) with torch.inference_mode(), torch.autocast(device_type=DEVICE.type, dtype=DTYPE): images = model.generate( prompt=[prompt] * batch_size, negative_prompt=negative_prompt, num_inference_steps=num_steps, cfg_scale=cfg_scale, height=height, width=width, max_token_length=MAX_TOKEN_LENGTH, cfg_time_range=[0.1, 1.0], seed=seed if seed >= 0 else None, device=DEVICE, execution_dtype=DTYPE, ) return [images, images_to_grid_image(images, padding=4)] LABEL2ID_URL = f"https://huggingface.co/{MODEL_REPO}/blob/main/{LABEL2ID_PATH}" def demo(): with gr.Blocks() as ui: gr.Markdown(f""" # JiT-Anime Demo Pixel-space x-prediction flow-matching 380M parameter model for anime image generation, trained from scratch. - See full supported tags: [label2id.json]({LABEL2ID_URL}). 対応しているタグ一覧は [こちら]({LABEL2ID_URL}) から確認できます。ここに載っていないタグは反応しません。 - Current model: [{MODEL_PATH}](https://huggingface.co/{MODEL_REPO}/blob/main/{MODEL_PATH}) """) with gr.Row(): with gr.Column(): prompt = gr.TextArea( label="Prompt", info=f"Comma-separated tags. Not all of danbooru tags are supported. See [the full supported tags]({LABEL2ID_URL}). カンマで区切ってください。", value="1girl, solo, cowboy shot, looking at viewer, blue hair, short hair, colored inner hair, hair intakes, cat ears, animal ears, red eyes, white background, collared shirt, long sleeves", placeholder="e.g.: general, 1girl, solo, portrait, looking at viewer", ) negative_prompt = gr.TextArea( label="Negative Prompt", info="Comma-separated negative tags to avoid in generation. カンマで区切ってください。", value=COMMON_NEGATIVE_PROMPT, lines=2, placeholder="e.g.: retro artstyle, 1990s (style), sketch", ) num_steps = gr.Slider( minimum=1, maximum=100, value=32, step=1, label="Number of Steps", info="Recommended: more than 20 steps for better quality.", ) cfg_scale = gr.Slider( minimum=1.0, maximum=15.0, value=7.5, step=0.25, label="CFG Scale", info="Recommended: more than 2.0 for better adherence to the prompt.", ) batch_size = gr.Slider( minimum=1, maximum=36, value=16, step=1, label="Batch Size", info="Number of images to generate in one batch.", ) with gr.Row(): height = gr.Slider( minimum=MIN_IMAGE_SIZE, maximum=MAX_IMAGE_SIZE, value=640, step=32, label="Image Height", ) width = gr.Slider( minimum=MIN_IMAGE_SIZE, maximum=MAX_IMAGE_SIZE, value=448, step=32, label="Image Width", ) seed = gr.Number( value=-1, label="Seed (-1 for random)", ) with gr.Column(scale=2): generate_button = gr.Button("Generate Images", variant="primary") output_gallery = gr.Gallery( label="Generated Images", columns=5, height="768px", object_fit="contain", preview=False, show_label=False, ) with gr.Accordion("Grid", open=False): grid_image = gr.Image( label="Grid Image", type="pil", show_label=False, ) gr.Examples( examples=[ [ "1girl, solo, cowboy shot, looking at viewer, blue hair, short hair, colored inner hair, hair intakes, cat ears, animal ears, red eyes, white background, collared shirt, long sleeves", COMMON_NEGATIVE_PROMPT, ], [ "1girl, solo, original, portrait, upper body, looking at viewer, long hair, blue ribbon, hair ornament, hairclip, depth of field, head tilt, collared shirt, white shirt, simple background", COMMON_NEGATIVE_PROMPT, ], [ "1girl, aqua eyes, baseball cap, blonde hair, closed mouth, earrings, green background, hat, jewelry, looking at viewer, shirt, short hair, simple background, solo, portrait, yellow shirt", COMMON_NEGATIVE_PROMPT, ], ], inputs=[prompt, negative_prompt], label="Examples", examples_per_page=20, ) gr.on( triggers=[generate_button.click, prompt.submit], fn=generate_images, inputs=[ prompt, negative_prompt, num_steps, cfg_scale, batch_size, height, width, seed, ], outputs=[output_gallery, grid_image], ) return ui if __name__ == "__main__": load_model( model_path=MODEL_PATH, label2id_path=LABEL2ID_PATH, config_path=CONFIG_PATH, device=DEVICE, dtype=DTYPE, ) demo().launch()