Upload 4 files

README.md

@@ -1,21 +1,13 @@
 ---
-title: SUPIR Image Upscaler
+title: LTX Video Fast
+emoji: 🎥
+colorFrom: yellow
+colorTo: pink
 sdk: gradio
-emoji: 📷
-sdk_version: 4.38.1
+sdk_version: 5.29.1
 app_file: app.py
-license: mit
-colorFrom: blue
-colorTo: pink
-tags:
-  - Upscaling
-  - Restoring
-  - Image-to-Image
-  - Image-2-Image
-  - Img-to-Img
-  - Img-2-Img
-  - language models
-  - LLMs
-short_description: Restore blurred or small images with prompt
-suggested_hardware: zero-a10g
----
+pinned: false
+short_description: ultra-fast video model, LTX 0.9.7 13B distilled
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,864 +1,486 @@
-import os
-import gradio as gr
-import argparse
-import numpy as np
-import torch
-import einops
-import copy
-import math
-import time
-import random
-import spaces
-import re
-import uuid
-
-from gradio_imageslider import ImageSlider
-from PIL import Image
-from SUPIR.util import HWC3, upscale_image, fix_resize, convert_dtype, create_SUPIR_model, load_QF_ckpt
-from huggingface_hub import hf_hub_download
-from pillow_heif import register_heif_opener
-
-register_heif_opener()
-
-max_64_bit_int = np.iinfo(np.int32).max
-
-hf_hub_download(repo_id="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", filename="open_clip_pytorch_model.bin", local_dir="laion_CLIP-ViT-bigG-14-laion2B-39B-b160k")
-hf_hub_download(repo_id="camenduru/SUPIR", filename="sd_xl_base_1.0_0.9vae.safetensors", local_dir="yushan777_SUPIR")
-hf_hub_download(repo_id="camenduru/SUPIR", filename="SUPIR-v0F.ckpt", local_dir="yushan777_SUPIR")
-hf_hub_download(repo_id="camenduru/SUPIR", filename="SUPIR-v0Q.ckpt", local_dir="yushan777_SUPIR")
-hf_hub_download(repo_id="RunDiffusion/Juggernaut-XL-Lightning", filename="Juggernaut_RunDiffusionPhoto2_Lightning_4Steps.safetensors", local_dir="RunDiffusion_Juggernaut-XL-Lightning")
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--opt", type=str, default='options/SUPIR_v0.yaml')
-parser.add_argument("--ip", type=str, default='127.0.0.1')
-parser.add_argument("--port", type=int, default='6688')
-parser.add_argument("--no_llava", action='store_true', default=True)#False
-parser.add_argument("--use_image_slider", action='store_true', default=False)#False
-parser.add_argument("--log_history", action='store_true', default=False)
-parser.add_argument("--loading_half_params", action='store_true', default=False)#False
-parser.add_argument("--use_tile_vae", action='store_true', default=True)#False
-parser.add_argument("--encoder_tile_size", type=int, default=512)
-parser.add_argument("--decoder_tile_size", type=int, default=64)
-parser.add_argument("--load_8bit_llava", action='store_true', default=False)
-args = parser.parse_args()
-
-if torch.cuda.device_count() > 0:
-    SUPIR_device = 'cuda:0'
-
-    # Load SUPIR
-    model, default_setting = create_SUPIR_model(args.opt, SUPIR_sign='Q', load_default_setting=True)
-    if args.loading_half_params:
-        model = model.half()
-    if args.use_tile_vae:
-        model.init_tile_vae(encoder_tile_size=args.encoder_tile_size, decoder_tile_size=args.decoder_tile_size)
-    model = model.to(SUPIR_device)
-    model.first_stage_model.denoise_encoder_s1 = copy.deepcopy(model.first_stage_model.denoise_encoder)
-    model.current_model = 'v0-Q'
-    ckpt_Q, ckpt_F = load_QF_ckpt(args.opt)
-
-def check_upload(input_image):
-    if input_image is None:
-        raise gr.Error("Please provide an image to restore.")
-    return gr.update(visible = True)
-
-def update_seed(is_randomize_seed, seed):
-    if is_randomize_seed:
-        return random.randint(0, max_64_bit_int)
-    return seed
-
-def reset():
-    return [
-        None,
-        0,
-        None,
-        None,
-        "Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.",
-        "painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, pixel, unsharp, weird textures, ugly, dirty, messy, worst quality, low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth",
-        1,
-        1024,
-        1,
-        2,
-        50,
-        -1.0,
-        1.,
-        default_setting.s_cfg_Quality if torch.cuda.device_count() > 0 else 1.0,
-        True,
-        random.randint(0, max_64_bit_int),
-        5,
-        1.003,
-        "Wavelet",
-        "fp32",
-        "fp32",
-        1.0,
-        True,
-        False,
-        default_setting.spt_linear_CFG_Quality if torch.cuda.device_count() > 0 else 1.0,
-        0.,
-        "v0-Q",
-        "input",
-        179
-    ]
-
-def check_and_update(input_image):
-    if input_image is None:
-        raise gr.Error("Please provide an image to restore.")
-    return gr.update(visible = True)
-
-@spaces.GPU(duration=420)
-def stage1_process(
-    input_image,
-    gamma_correction,
-    diff_dtype,
-    ae_dtype
-):
-    print('stage1_process ==>>')
-    if torch.cuda.device_count() == 0:
-        gr.Warning('Set this space to GPU config to make it work.')
-        return None, None
-    torch.cuda.set_device(SUPIR_device)
-    LQ = HWC3(np.array(Image.open(input_image)))
-    LQ = fix_resize(LQ, 512)
-    # stage1
-    LQ = np.array(LQ) / 255 * 2 - 1
-    LQ = torch.tensor(LQ, dtype=torch.float32).permute(2, 0, 1).unsqueeze(0).to(SUPIR_device)[:, :3, :, :]
-
-    model.ae_dtype = convert_dtype(ae_dtype)
-    model.model.dtype = convert_dtype(diff_dtype)
-
-    LQ = model.batchify_denoise(LQ, is_stage1=True)
-    LQ = (LQ[0].permute(1, 2, 0) * 127.5 + 127.5).cpu().numpy().round().clip(0, 255).astype(np.uint8)
-    # gamma correction
-    LQ = LQ / 255.0
-    LQ = np.power(LQ, gamma_correction)
-    LQ *= 255.0
-    LQ = LQ.round().clip(0, 255).astype(np.uint8)
-    print('<<== stage1_process')
-    return LQ, gr.update(visible = True)
-
-def stage2_process(*args, **kwargs):
-    try:
-        return restore_in_Xmin(*args, **kwargs)
-    except Exception as e:
-        # NO_GPU_MESSAGE_INQUEUE
-        print("gradio.exceptions.Error 'No GPU is currently available for you after 60s'")
-        print('str(type(e)): ' + str(type(e))) # <class 'gradio.exceptions.Error'>
-        print('str(e): ' + str(e)) # You have exceeded your GPU quota...
-        try:
-            print('e.message: ' + e.message) # No GPU is currently available for you after 60s
-        except Exception as e2:
-            print('Failure')
-        if str(e).startswith("No GPU is currently available for you after 60s"):
-            print('Exception identified!!!')
-        #if str(type(e)) == "<class 'gradio.exceptions.Error'>":
-            #print('Exception of name ' + type(e).__name__)
-        raise e
-
-def restore_in_Xmin(
-    noisy_image,
-    rotation,
-    denoise_image,
-    prompt,
-    a_prompt,
-    n_prompt,
-    num_samples,
-    min_size,
-    downscale,
-    upscale,
-    edm_steps,
-    s_stage1,
-    s_stage2,
-    s_cfg,
-    randomize_seed,
-    seed,
-    s_churn,
-    s_noise,
-    color_fix_type,
-    diff_dtype,
-    ae_dtype,
-    gamma_correction,
-    linear_CFG,
-    linear_s_stage2,
-    spt_linear_CFG,
-    spt_linear_s_stage2,
-    model_select,
-    output_format,
-    allocation
-):
-    print("noisy_image:\n" + str(noisy_image))
-    print("denoise_image:\n" + str(denoise_image))
-    print("rotation: " + str(rotation))
-    print("prompt: " + str(prompt))
-    print("a_prompt: " + str(a_prompt))
-    print("n_prompt: " + str(n_prompt))
-    print("num_samples: " + str(num_samples))
-    print("min_size: " + str(min_size))
-    print("downscale: " + str(downscale))
-    print("upscale: " + str(upscale))
-    print("edm_steps: " + str(edm_steps))
-    print("s_stage1: " + str(s_stage1))
-    print("s_stage2: " + str(s_stage2))
-    print("s_cfg: " + str(s_cfg))
-    print("randomize_seed: " + str(randomize_seed))
-    print("seed: " + str(seed))
-    print("s_churn: " + str(s_churn))
-    print("s_noise: " + str(s_noise))
-    print("color_fix_type: " + str(color_fix_type))
-    print("diff_dtype: " + str(diff_dtype))
-    print("ae_dtype: " + str(ae_dtype))
-    print("gamma_correction: " + str(gamma_correction))
-    print("linear_CFG: " + str(linear_CFG))
-    print("linear_s_stage2: " + str(linear_s_stage2))
-    print("spt_linear_CFG: " + str(spt_linear_CFG))
-    print("spt_linear_s_stage2: " + str(spt_linear_s_stage2))
-    print("model_select: " + str(model_select))
-    print("GPU time allocation: " + str(allocation) + " min")
-    print("output_format: " + str(output_format))
-
-    input_format = re.sub(r"^.*\.([^\.]+)$", r"\1", noisy_image)
-
-    if input_format not in ['png', 'webp', 'jpg', 'jpeg', 'gif', 'bmp', 'heic']:
-        gr.Warning('Invalid image format. Please first convert into *.png, *.webp, *.jpg, *.jpeg, *.gif, *.bmp or *.heic.')
-        return None, None, None, None
-
-    if output_format == "input":
-        if noisy_image is None:
-            output_format = "png"
-        else:
-            output_format = input_format
-    print("final output_format: " + str(output_format))
-
-    if prompt is None:
-        prompt = ""
-
-    if a_prompt is None:
-        a_prompt = ""
-
-    if n_prompt is None:
-        n_prompt = ""
-
-    if prompt != "" and a_prompt != "":
-        a_prompt = prompt + ", " + a_prompt
-    else:
-        a_prompt = prompt + a_prompt
-    print("Final prompt: " + str(a_prompt))
-
-    denoise_image = np.array(Image.open(noisy_image if denoise_image is None else denoise_image))
-
-    if rotation == 90:
-        denoise_image = np.array(list(zip(*denoise_image[::-1])))
-    elif rotation == 180:
-        denoise_image = np.array(list(zip(*denoise_image[::-1])))
-        denoise_image = np.array(list(zip(*denoise_image[::-1])))
-    elif rotation == -90:
-        denoise_image = np.array(list(zip(*denoise_image))[::-1])
-
-    if 1 < downscale:
-        input_height, input_width, input_channel = denoise_image.shape
-        denoise_image = np.array(Image.fromarray(denoise_image).resize((input_width // downscale, input_height // downscale), Image.LANCZOS))
-
-    denoise_image = HWC3(denoise_image)
-
-    if torch.cuda.device_count() == 0:
-        gr.Warning('Set this space to GPU config to make it work.')
-        return [noisy_image, denoise_image], gr.update(label="Downloadable results in *." + output_format + " format", format = output_format, value = [denoise_image]), None, gr.update(visible=True)
-
-    if model_select != model.current_model:
-        print('load ' + model_select)
-        if model_select == 'v0-Q':
-            model.load_state_dict(ckpt_Q, strict=False)
-        elif model_select == 'v0-F':
-            model.load_state_dict(ckpt_F, strict=False)
-        model.current_model = model_select
-
-    model.ae_dtype = convert_dtype(ae_dtype)
-    model.model.dtype = convert_dtype(diff_dtype)
-
-    return restore_on_gpu(
-        noisy_image, denoise_image, prompt, a_prompt, n_prompt, num_samples, min_size, downscale, upscale, edm_steps, s_stage1, s_stage2, s_cfg, randomize_seed, seed, s_churn, s_noise, color_fix_type, diff_dtype, ae_dtype, gamma_correction, linear_CFG, linear_s_stage2, spt_linear_CFG, spt_linear_s_stage2, model_select, output_format, allocation
-    )
-
-def get_duration(
-    noisy_image,
-    input_image,
-    prompt,
-    a_prompt,
-    n_prompt,
-    num_samples,
-    min_size,
-    downscale,
-    upscale,
-    edm_steps,
-    s_stage1,
-    s_stage2,
-    s_cfg,
-    randomize_seed,
-    seed,
-    s_churn,
-    s_noise,
-    color_fix_type,
-    diff_dtype,
-    ae_dtype,
-    gamma_correction,
-    linear_CFG,
-    linear_s_stage2,
-    spt_linear_CFG,
-    spt_linear_s_stage2,
-    model_select,
-    output_format,
-    allocation
-):
-    return allocation
-
-@spaces.GPU(duration=get_duration)
-def restore_on_gpu(
-    noisy_image,
-    input_image,
-    prompt,
-    a_prompt,
-    n_prompt,
-    num_samples,
-    min_size,
-    downscale,
-    upscale,
-    edm_steps,
-    s_stage1,
-    s_stage2,
-    s_cfg,
-    randomize_seed,
-    seed,
-    s_churn,
-    s_noise,
-    color_fix_type,
-    diff_dtype,
-    ae_dtype,
-    gamma_correction,
-    linear_CFG,
-    linear_s_stage2,
-    spt_linear_CFG,
-    spt_linear_s_stage2,
-    model_select,
-    output_format,
-    allocation
-):
-    start = time.time()
-    print('restore ==>>')
-
-    torch.cuda.set_device(SUPIR_device)
-
-    with torch.no_grad():
-        input_image = upscale_image(input_image, upscale, unit_resolution=32, min_size=min_size)
-        LQ = np.array(input_image) / 255.0
-        LQ = np.power(LQ, gamma_correction)
-        LQ *= 255.0
-        LQ = LQ.round().clip(0, 255).astype(np.uint8)
-        LQ = LQ / 255 * 2 - 1
-        LQ = torch.tensor(LQ, dtype=torch.float32).permute(2, 0, 1).unsqueeze(0).to(SUPIR_device)[:, :3, :, :]
-        captions = ['']
-
-        samples = model.batchify_sample(LQ, captions, num_steps=edm_steps, restoration_scale=s_stage1, s_churn=s_churn,
-                                    s_noise=s_noise, cfg_scale=s_cfg, control_scale=s_stage2, seed=seed,
-                                    num_samples=num_samples, p_p=a_prompt, n_p=n_prompt, color_fix_type=color_fix_type,
-                                    use_linear_CFG=linear_CFG, use_linear_control_scale=linear_s_stage2,
-                                    cfg_scale_start=spt_linear_CFG, control_scale_start=spt_linear_s_stage2)
-
-        x_samples = (einops.rearrange(samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().round().clip(
-            0, 255).astype(np.uint8)
-        results = [x_samples[i] for i in range(num_samples)]
-    torch.cuda.empty_cache()
-
-    # All the results have the same size
-    input_height, input_width, input_channel = np.array(input_image).shape
-    result_height, result_width, result_channel = np.array(results[0]).shape
-
-    print('<<== restore')
-    end = time.time()
-    secondes = int(end - start)
-    minutes = math.floor(secondes / 60)
-    secondes = secondes - (minutes * 60)
-    hours = math.floor(minutes / 60)
-    minutes = minutes - (hours * 60)
-    information = ("Start the process again if you want a different result. " if randomize_seed else "") + \
-    "If you don't get the image you wanted, add more details in the « Image description ». " + \
-    "Wait " + str(allocation) + " min before a new run to avoid quota penalty or use another computer. " + \
-    "The image" + (" has" if len(results) == 1 else "s have") + " been generated in " + \
-    ((str(hours) + " h, ") if hours != 0 else "") + \
-    ((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \
-    str(secondes) + " sec. " + \
-    "The new image resolution is " + str(result_width) + \
-    " pixels large and " + str(result_height) + \
-    " pixels high, so a resolution of " + f'{result_width * result_height:,}' + " pixels."
-    print(information)
-    try:
-        print("Initial resolution: " + f'{input_width * input_height:,}')
-        print("Final resolution: " + f'{result_width * result_height:,}')
-        print("edm_steps: " + str(edm_steps))
-        print("num_samples: " + str(num_samples))
-        print("downscale: " + str(downscale))
-        print("Estimated minutes: " + f'{(((result_width * result_height**(1/1.75)) * input_width * input_height * (edm_steps**(1/2)) * (num_samples**(1/2.5)))**(1/2.5)) / 25000:,}')
-    except Exception as e:
-        print('Exception of Estimation')
-
-    # Only one image can be shown in the slider
-    return [noisy_image] + [results[0]], gr.update(label="Downloadable results in *." + output_format + " format", format = output_format, value = results), gr.update(value = information, visible = True), gr.update(visible=True)
-
-def load_and_reset(param_setting):
-    print('load_and_reset ==>>')
-    if torch.cuda.device_count() == 0:
-        gr.Warning('Set this space to GPU config to make it work.')
-        return None, None, None, None, None, None, None, None, None, None, None, None, None, None
-    edm_steps = default_setting.edm_steps
-    s_stage2 = 1.0
-    s_stage1 = -1.0
-    s_churn = 5
-    s_noise = 1.003
-    a_prompt = 'Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - ' \
-               'realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore ' \
-               'detailing, hyper sharpness, perfect without deformations.'
-    n_prompt = 'painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, ' \
-               '3D render, unreal engine, blurring, dirty, messy, worst quality, low quality, frames, watermark, ' \
-               'signature, jpeg artifacts, deformed, lowres, over-smooth'
-    color_fix_type = 'Wavelet'
-    spt_linear_s_stage2 = 0.0
-    linear_s_stage2 = False
-    linear_CFG = True
-    if param_setting == "Quality":
-        s_cfg = default_setting.s_cfg_Quality
-        spt_linear_CFG = default_setting.spt_linear_CFG_Quality
-        model_select = "v0-Q"
-    elif param_setting == "Fidelity":
-        s_cfg = default_setting.s_cfg_Fidelity
-        spt_linear_CFG = default_setting.spt_linear_CFG_Fidelity
-        model_select = "v0-F"
-    else:
-        raise NotImplementedError
-    gr.Info('The parameters are reset.')
-    print('<<== load_and_reset')
-    return edm_steps, s_cfg, s_stage2, s_stage1, s_churn, s_noise, a_prompt, n_prompt, color_fix_type, linear_CFG, \
-        linear_s_stage2, spt_linear_CFG, spt_linear_s_stage2, model_select
-
-def log_information(result_gallery):
-    print('log_information')
-    if result_gallery is not None:
-        for i, result in enumerate(result_gallery):
-            print(result[0])
-
-def on_select_result(result_slider, result_gallery, evt: gr.SelectData):
-    print('on_select_result')
-    if result_gallery is not None:
-        for i, result in enumerate(result_gallery):
-            print(result[0])
-    return [result_slider[0], result_gallery[evt.index][0]]
-
-title_html = """
-    <h1><center>SUPIR</center></h1>
-    <big><center>Upscale your images up to x10 freely, without account, without watermark and download it</center></big>
-    <center><big><big>🤸<big><big><big><big><big><big>🤸</big></big></big></big></big></big></big></big></center>
-    
-    <p>This is an online demo of SUPIR, a practicing model scaling for photo-realistic image restoration.
-    The content added by SUPIR is <b><u>imagination, not real-world information</u></b>.
-    SUPIR is for beauty and illustration only.
-    Most of the processes last few minutes.
-    If you want to upscale AI-generated images, be noticed that <i>PixArt Sigma</i> space can directly generate 5984x5984 images.
-    Due to Gradio issues, the generated image is slightly less satured than the original.
-    Please leave a <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR/discussions/new">message in discussion</a> if you encounter issues.
-    You can also use <a href="https://huggingface.co/spaces/gokaygokay/AuraSR">AuraSR</a> to upscale x4.
-    
-    <p><center><a href="https://arxiv.org/abs/2401.13627">Paper</a> &emsp; <a href="http://supir.xpixel.group/">Project Page</a> &emsp; <a href="https://huggingface.co/blog/MonsterMMORPG/supir-sota-image-upscale-better-than-magnific-ai">Local Install Guide</a></center></p>
-    <p><center><a style="display:inline-block" href='https://github.com/Fanghua-Yu/SUPIR'><img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/Fanghua-Yu/SUPIR?style=social"></a></center></p>
-    """
-
-
-claim_md = """
-## **Piracy**
-The images are not stored but the logs are saved during a month.
-## **How to get SUPIR**
-You can get SUPIR on HuggingFace by [duplicating this space](https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR?duplicate=true) and set GPU.
-You can also install SUPIR on your computer following [this tutorial](https://huggingface.co/blog/MonsterMMORPG/supir-sota-image-upscale-better-than-magnific-ai).
-You can install _Pinokio_ on your computer and then install _SUPIR_ into it. It should be quite easy if you have an Nvidia GPU.
-## **Terms of use**
-By using this service, users are required to agree to the following terms: The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research. Please submit a feedback to us if you get any inappropriate answer! We will collect those to keep improving our models. For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
-## **License**
-The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/Fanghua-Yu/SUPIR) of SUPIR.
-"""
-
-# Gradio interface
-with gr.Blocks() as interface:
-    if torch.cuda.device_count() == 0:
-        with gr.Row():
-            gr.HTML("""
-    <p style="background-color: red;"><big><big><big><b>⚠️To use SUPIR, <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR?duplicate=true">duplicate this space</a> and set a GPU with 30 GB VRAM.</b>
-    
-    You can't use SUPIR directly here because this space runs on a CPU, which is not enough for SUPIR. Please provide <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR/discussions/new">feedback</a> if you have issues.
-    </big></big></big></p>
-    """)
-    gr.HTML(title_html)
-
-    input_image = gr.Image(label="Input (*.png, *.webp, *.jpeg, *.jpg, *.gif, *.bmp, *.heic)", show_label=True, type="filepath", height=600, elem_id="image-input")
-    rotation = gr.Radio([["No rotation", 0], ["⤵ Rotate +90°", 90], ["↩ Return 180°", 180], ["⤴ Rotate -90°", -90]], label="Orientation correction", info="Will apply the following rotation before restoring the image; the AI needs a good orientation to understand the content", value=0, interactive=True, visible=False)
-    with gr.Group():
-        prompt = gr.Textbox(label="Image description", info="Help the AI understand what the image represents; describe as much as possible, especially the details we can't see on the original image; you can write in any language", value="", placeholder="A 33 years old man, walking, in the street, Santiago, morning, Summer, photorealistic", lines=3)
-        prompt_hint = gr.HTML("You can use a <a href='"'https://huggingface.co/spaces/badayvedat/LLaVA'"'>LlaVa space</a> to auto-generate the description of your image.")
-        upscale = gr.Radio([["x1", 1], ["x2", 2], ["x3", 3], ["x4", 4], ["x5", 5], ["x6", 6], ["x7", 7], ["x8", 8], ["x9", 9], ["x10", 10]], label="Upscale factor", info="Resolution x1 to x10", value=2, interactive=True)
-        output_format = gr.Radio([["As input", "input"], ["*.png", "png"], ["*.webp", "webp"], ["*.jpeg", "jpeg"], ["*.gif", "gif"], ["*.bmp", "bmp"]], label="Image format for result", info="File extention", value="input", interactive=True)
-        allocation = gr.Slider(label="GPU allocation time (in seconds)", info="lower=May abort run, higher=Quota penalty for next runs", value=179, minimum=59, maximum=320, step=1)
-
-    with gr.Accordion("Pre-denoising (optional)", open=False):
-        gamma_correction = gr.Slider(label="Gamma Correction", info = "lower=lighter, higher=darker", minimum=0.1, maximum=2.0, value=1.0, step=0.1)
-        denoise_button = gr.Button(value="Pre-denoise")
-        denoise_image = gr.Image(label="Denoised image", show_label=True, type="filepath", sources=[], interactive = False, height=600, elem_id="image-s1")
-        denoise_information = gr.HTML(value="If present, the denoised image will be used for the restoration instead of the input image.", visible=False)
-
-    with gr.Accordion("Advanced options", open=False):
-        a_prompt = gr.Textbox(label="Additional image description",
-                              info="Completes the main image description",
-                              value='Cinematic, High Contrast, highly detailed, taken using a Canon EOS R '
-                                    'camera, hyper detailed photo - realistic maximum detail, 32k, Color '
-                                    'Grading, ultra HD, extreme meticulous detailing, skin pore detailing, clothing fabric detailing, '
-                                    'hyper sharpness, perfect without deformations.',
-                              lines=3)
-        n_prompt = gr.Textbox(label="Negative image description",
-                              info="Disambiguate by listing what the image does NOT represent",
-                              value='painting, oil painting, illustration, drawing, art, sketch, anime, '
-                                    'cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, pixel, unsharp, weird textures, ugly, dirty, messy, '
-                                    'worst quality, low quality, frames, watermark, signature, jpeg artifacts, '
-                                    'deformed, lowres, over-smooth',
-                              lines=3)
-        edm_steps = gr.Slider(label="Steps", info="lower=faster, higher=more details; too many steps create a checker effect", minimum=1, maximum=200, value=default_setting.edm_steps if torch.cuda.device_count() > 0 else 1, step=1)
-        num_samples = gr.Slider(label="Num Samples", info="Number of generated results", minimum=1, maximum=4 if not args.use_image_slider else 1
-                                , value=1, step=1)
-        min_size = gr.Slider(label="Minimum size", info="Minimum height, minimum width of the result", minimum=32, maximum=4096, value=1024, step=32)
-        downscale = gr.Radio([["/1", 1], ["/2", 2], ["/3", 3], ["/4", 4], ["/5", 5], ["/6", 6], ["/7", 7], ["/8", 8], ["/9", 9], ["/10", 10]], label="Pre-downscale factor", info="Reducing blurred image reduce the process time", value=1, interactive=True)
-        with gr.Row():
-            with gr.Column():
-                model_select = gr.Radio([["💃 Quality (v0-Q)", "v0-Q"], ["🎯 Fidelity (v0-F)", "v0-F"]], label="Model Selection", info="Pretrained model", value="v0-Q",
-                                        interactive=True)
-            with gr.Column():
-                color_fix_type = gr.Radio([["None", "None"], ["AdaIn (improve as a photo)", "AdaIn"], ["Wavelet (for JPEG artifacts)", "Wavelet"]], label="Color-Fix Type", info="AdaIn=Improve following a style, Wavelet=For JPEG artifacts", value="AdaIn",
-                                          interactive=True)
-        s_cfg = gr.Slider(label="Text Guidance Scale", info="lower=follow the image, higher=follow the prompt", minimum=1.0, maximum=15.0,
-                          value=default_setting.s_cfg_Quality if torch.cuda.device_count() > 0 else 1.0, step=0.1)
-        s_stage2 = gr.Slider(label="Restoring Guidance Strength", minimum=0., maximum=1., value=1., step=0.05)
-        s_stage1 = gr.Slider(label="Pre-denoising Guidance Strength", minimum=-1.0, maximum=6.0, value=-1.0, step=1.0)
-        s_churn = gr.Slider(label="S-Churn", minimum=0, maximum=40, value=5, step=1)
-        s_noise = gr.Slider(label="S-Noise", minimum=1.0, maximum=1.1, value=1.003, step=0.001)
-        with gr.Row():
-            with gr.Column():
-                linear_CFG = gr.Checkbox(label="Linear CFG", value=True)
-                spt_linear_CFG = gr.Slider(label="CFG Start", minimum=1.0,
-                                                maximum=9.0, value=default_setting.spt_linear_CFG_Quality if torch.cuda.device_count() > 0 else 1.0, step=0.5)
-            with gr.Column():
-                linear_s_stage2 = gr.Checkbox(label="Linear Restoring Guidance", value=False)
-                spt_linear_s_stage2 = gr.Slider(label="Guidance Start", minimum=0.,
-                                                maximum=1., value=0., step=0.05)
-            with gr.Column():
-                diff_dtype = gr.Radio([["fp32 (precision)", "fp32"], ["fp16 (medium)", "fp16"], ["bf16 (speed)", "bf16"]], label="Diffusion Data Type", value="fp32",
-                                      interactive=True)
-            with gr.Column():
-                ae_dtype = gr.Radio([["fp32 (precision)", "fp32"], ["bf16 (speed)", "bf16"]], label="Auto-Encoder Data Type", value="fp32",
-                                    interactive=True)
-        randomize_seed = gr.Checkbox(label = "\U0001F3B2 Randomize seed", value = True, info = "If checked, result is always different")
-        seed = gr.Slider(label="Seed", minimum=0, maximum=max_64_bit_int, step=1, randomize=True)
-        with gr.Group():
-            param_setting = gr.Radio(["Quality", "Fidelity"], interactive=True, label="Presetting", value = "Quality")
-            restart_button = gr.Button(value="Apply presetting")
-
-    with gr.Column():
-        diffusion_button = gr.Button(value="🚀 Upscale/Restore", variant = "primary", elem_id = "process_button")
-        reset_btn = gr.Button(value="🧹 Reinit page", variant="stop", elem_id="reset_button", visible = False)
-
-        warning = gr.HTML(value = "<center><big>Your computer must <u>not</u> enter into standby mode.</big><br/>On Chrome, you can force to keep a tab alive in <code>chrome://discards/</code></center>", visible = False)
-        restore_information = gr.HTML(value = "Restart the process to get another result.", visible = False)
-        result_slider = ImageSlider(label = 'Comparator', show_label = False, interactive = False, elem_id = "slider1", show_download_button = False)
-        result_gallery = gr.Gallery(label = 'Downloadable results', show_label = True, interactive = False, elem_id = "gallery1")
-
-    gr.Examples(
-        examples = [
-                [
-                    "./Examples/Example1.png",
-                    0,
-                    None,
-                    "Group of people, walking, happy, in the street, photorealistic, 8k, extremely detailled",
-                    "Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.",
-                    "painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, pixel, unsharp, weird textures, ugly, dirty, messy, worst quality, low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth",
-                    2,
-                    1024,
-                    1,
-                    8,
-                    100,
-                    -1,
-                    1,
-                    7.5,
-                    False,
-                    42,
-                    5,
-                    1.003,
-                    "AdaIn",
-                    "fp16",
-                    "bf16",
-                    1.0,
-                    True,
-                    4,
-                    False,
-                    0.,
-                    "v0-Q",
-                    "input",
-                    179
-                ],
-                [
-                    "./Examples/Example2.jpeg",
-                    0,
-                    None,
-                    "La cabeza de un gato atigrado, en una casa, fotorrealista, 8k, extremadamente detallada",
-                    "Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.",
-                    "painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, pixel, unsharp, weird textures, ugly, dirty, messy, worst quality, low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth",
-                    1,
-                    1024,
-                    1,
-                    1,
-                    200,
-                    -1,
-                    1,
-                    7.5,
-                    False,
-                    42,
-                    5,
-                    1.003,
-                    "Wavelet",
-                    "fp16",
-                    "bf16",
-                    1.0,
-                    True,
-                    4,
-                    False,
-                    0.,
-                    "v0-Q",
-                    "input",
-                    179
-                ],
-                [
-                    "./Examples/Example3.webp",
-                    0,
-                    None,
-                    "A red apple",
-                    "Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.",
-                    "painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, pixel, unsharp, weird textures, ugly, dirty, messy, worst quality, low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth",
-                    1,
-                    1024,
-                    1,
-                    1,
-                    200,
-                    -1,
-                    1,
-                    7.5,
-                    False,
-                    42,
-                    5,
-                    1.003,
-                    "Wavelet",
-                    "fp16",
-                    "bf16",
-                    1.0,
-                    True,
-                    4,
-                    False,
-                    0.,
-                    "v0-Q",
-                    "input",
-                    179
-                ],
-                [
-                    "./Examples/Example3.webp",
-                    0,
-                    None,
-                    "A red marble",
-                    "Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.",
-                    "painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, pixel, unsharp, weird textures, ugly, dirty, messy, worst quality, low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth",
-                    1,
-                    1024,
-                    1,
-                    1,
-                    200,
-                    -1,
-                    1,
-                    7.5,
-                    False,
-                    42,
-                    5,
-                    1.003,
-                    "Wavelet",
-                    "fp16",
-                    "bf16",
-                    1.0,
-                    True,
-                    4,
-                    False,
-                    0.,
-                    "v0-Q",
-                    "input",
-                    179
-                ],
-            ],
-        run_on_click = True,
-        fn = stage2_process,
-	    inputs = [
-            input_image,
-            rotation,
-            denoise_image,
-            prompt,
-            a_prompt,
-            n_prompt,
-            num_samples,
-            min_size,
-            downscale,
-            upscale,
-            edm_steps,
-            s_stage1,
-            s_stage2,
-            s_cfg,
-            randomize_seed,
-            seed,
-            s_churn,
-            s_noise,
-            color_fix_type,
-            diff_dtype,
-            ae_dtype,
-            gamma_correction,
-            linear_CFG,
-            linear_s_stage2,
-            spt_linear_CFG,
-            spt_linear_s_stage2,
-            model_select,
-            output_format,
-            allocation
-        ],
-	    outputs = [
-            result_slider,
-            result_gallery,
-            restore_information,
-            reset_btn
-        ],
-        cache_examples = False,
-    )
-
-    with gr.Row():
-        gr.Markdown(claim_md)
-    
-    input_image.upload(fn = check_upload, inputs = [
-        input_image
-    ], outputs = [
-        rotation
-    ], queue = False, show_progress = False)
-
-    denoise_button.click(fn = check_and_update, inputs = [
-        input_image
-    ], outputs = [warning], queue = False, show_progress = False).success(fn = stage1_process, inputs = [
-        input_image,
-        gamma_correction,
-        diff_dtype,
-        ae_dtype
-    ], outputs=[
-        denoise_image,
-        denoise_information
-    ])
-
-    diffusion_button.click(fn = update_seed, inputs = [
-        randomize_seed,
-        seed
-    ], outputs = [
-        seed
-    ], queue = False, show_progress = False).then(fn = check_and_update, inputs = [
-        input_image
-    ], outputs = [warning], queue = False, show_progress = False).success(fn=stage2_process, inputs = [
-        input_image,
-        rotation,
-        denoise_image,
-        prompt,
-        a_prompt,
-        n_prompt,
-        num_samples,
-        min_size,
-        downscale,
-        upscale,
-        edm_steps,
-        s_stage1,
-        s_stage2,
-        s_cfg,
-        randomize_seed,
-        seed,
-        s_churn,
-        s_noise,
-        color_fix_type,
-        diff_dtype,
-        ae_dtype,
-        gamma_correction,
-        linear_CFG,
-        linear_s_stage2,
-        spt_linear_CFG,
-        spt_linear_s_stage2,
-        model_select,
-        output_format,
-        allocation
-    ], outputs = [
-        result_slider,
-        result_gallery,
-        restore_information,
-        reset_btn
-    ]).success(fn = log_information, inputs = [
-        result_gallery
-    ], outputs = [], queue = False, show_progress = False)
-
-    result_gallery.change(on_select_result, [result_slider, result_gallery], result_slider)
-    result_gallery.select(on_select_result, [result_slider, result_gallery], result_slider)
-
-    restart_button.click(fn = load_and_reset, inputs = [
-        param_setting
-    ], outputs = [
-        edm_steps,
-        s_cfg,
-        s_stage2,
-        s_stage1,
-        s_churn,
-        s_noise,
-        a_prompt,
-        n_prompt,
-        color_fix_type,
-        linear_CFG,
-        linear_s_stage2,
-        spt_linear_CFG,
-        spt_linear_s_stage2,
-        model_select
-    ])
-
-    reset_btn.click(fn = reset, inputs = [], outputs = [
-            input_image,
-            rotation,
-            denoise_image,
-            prompt,
-            a_prompt,
-            n_prompt,
-            num_samples,
-            min_size,
-            downscale,
-            upscale,
-            edm_steps,
-            s_stage1,
-            s_stage2,
-            s_cfg,
-            randomize_seed,
-            seed,
-            s_churn,
-            s_noise,
-            color_fix_type,
-            diff_dtype,
-            ae_dtype,
-            gamma_correction,
-            linear_CFG,
-            linear_s_stage2,
-            spt_linear_CFG,
-            spt_linear_s_stage2,
-            model_select,
-            output_format,
-            allocation
-        ], queue = False, show_progress = False)
-        
-    interface.queue(10).launch()
+import gradio as gr
+import torch
+import spaces
+import numpy as np
+import random
+import os
+import yaml
+from pathlib import Path
+import imageio
+import tempfile
+from PIL import Image
+from huggingface_hub import hf_hub_download
+import shutil
+
+from inference import (
+    create_ltx_video_pipeline,
+    create_latent_upsampler,
+    load_image_to_tensor_with_resize_and_crop,
+    seed_everething,
+    get_device,
+    calculate_padding,
+    load_media_file
+)
+from ltx_video.pipelines.pipeline_ltx_video import ConditioningItem, LTXMultiScalePipeline, LTXVideoPipeline
+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
+
+config_file_path = "configs/ltxv-13b-0.9.7-distilled.yaml"
+with open(config_file_path, "r") as file:
+    PIPELINE_CONFIG_YAML = yaml.safe_load(file)
+
+LTX_REPO = "Lightricks/LTX-Video"
+MAX_IMAGE_SIZE = PIPELINE_CONFIG_YAML.get("max_resolution", 1280)
+MAX_NUM_FRAMES = 257
+
+FPS = 30.0 
+
+# --- Global variables for loaded models ---
+pipeline_instance = None
+latent_upsampler_instance = None
+models_dir = "downloaded_models_gradio_cpu_init"
+Path(models_dir).mkdir(parents=True, exist_ok=True)
+
+print("Downloading models (if not present)...")
+distilled_model_actual_path = hf_hub_download(
+    repo_id=LTX_REPO,
+    filename=PIPELINE_CONFIG_YAML["checkpoint_path"],
+    local_dir=models_dir,
+    local_dir_use_symlinks=False
+)
+PIPELINE_CONFIG_YAML["checkpoint_path"] = distilled_model_actual_path
+print(f"Distilled model path: {distilled_model_actual_path}")
+
+SPATIAL_UPSCALER_FILENAME = PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"]
+spatial_upscaler_actual_path = hf_hub_download(
+    repo_id=LTX_REPO,
+    filename=SPATIAL_UPSCALER_FILENAME,
+    local_dir=models_dir,
+    local_dir_use_symlinks=False
+)
+PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"] = spatial_upscaler_actual_path
+print(f"Spatial upscaler model path: {spatial_upscaler_actual_path}")
+
+print("Creating LTX Video pipeline on CPU...")
+pipeline_instance = create_ltx_video_pipeline(
+    ckpt_path=PIPELINE_CONFIG_YAML["checkpoint_path"],
+    precision=PIPELINE_CONFIG_YAML["precision"],
+    text_encoder_model_name_or_path=PIPELINE_CONFIG_YAML["text_encoder_model_name_or_path"],
+    sampler=PIPELINE_CONFIG_YAML["sampler"],
+    device="cpu",
+    enhance_prompt=False,
+    prompt_enhancer_image_caption_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_image_caption_model_name_or_path"],
+    prompt_enhancer_llm_model_name_or_path=PIPELINE_CONFIG_YAML["prompt_enhancer_llm_model_name_or_path"],
+)
+print("LTX Video pipeline created on CPU.")
+
+if PIPELINE_CONFIG_YAML.get("spatial_upscaler_model_path"):
+    print("Creating latent upsampler on CPU...")
+    latent_upsampler_instance = create_latent_upsampler(
+        PIPELINE_CONFIG_YAML["spatial_upscaler_model_path"],
+        device="cpu"
+    )
+    print("Latent upsampler created on CPU.")
+
+target_inference_device = "cuda"
+print(f"Target inference device: {target_inference_device}")
+pipeline_instance.to(target_inference_device)
+if latent_upsampler_instance: 
+    latent_upsampler_instance.to(target_inference_device)
+
+
+# --- Helper function for dimension calculation ---
+MIN_DIM_SLIDER = 256  # As defined in the sliders minimum attribute
+TARGET_FIXED_SIDE = 768 # Desired fixed side length as per requirement
+
+def calculate_new_dimensions(orig_w, orig_h):
+    """
+    Calculates new dimensions for height and width sliders based on original media dimensions.
+    Ensures one side is TARGET_FIXED_SIDE, the other is scaled proportionally,
+    both are multiples of 32, and within [MIN_DIM_SLIDER, MAX_IMAGE_SIZE].
+    """
+    if orig_w == 0 or orig_h == 0:
+        # Default to TARGET_FIXED_SIDE square if original dimensions are invalid
+        return int(TARGET_FIXED_SIDE), int(TARGET_FIXED_SIDE)
+
+    if orig_w >= orig_h:  # Landscape or square
+        new_h = TARGET_FIXED_SIDE
+        aspect_ratio = orig_w / orig_h
+        new_w_ideal = new_h * aspect_ratio
+        
+        # Round to nearest multiple of 32
+        new_w = round(new_w_ideal / 32) * 32
+        
+        # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
+        new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
+        # Ensure new_h is also clamped (TARGET_FIXED_SIDE should be within these bounds if configured correctly)
+        new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE)) 
+    else:  # Portrait
+        new_w = TARGET_FIXED_SIDE
+        aspect_ratio = orig_h / orig_w # Use H/W ratio for portrait scaling
+        new_h_ideal = new_w * aspect_ratio
+        
+        # Round to nearest multiple of 32
+        new_h = round(new_h_ideal / 32) * 32
+        
+        # Clamp to [MIN_DIM_SLIDER, MAX_IMAGE_SIZE]
+        new_h = max(MIN_DIM_SLIDER, min(new_h, MAX_IMAGE_SIZE))
+        # Ensure new_w is also clamped
+        new_w = max(MIN_DIM_SLIDER, min(new_w, MAX_IMAGE_SIZE))
+
+    return int(new_h), int(new_w)
+
+def get_duration(prompt, negative_prompt, input_image_filepath, input_video_filepath,
+             height_ui, width_ui, mode,
+             duration_ui, # Removed ui_steps
+             ui_frames_to_use,
+             seed_ui, randomize_seed, ui_guidance_scale, improve_texture_flag,
+             progress):
+    if duration_ui > 7:
+        return 75
+    else:
+        return 60
+
+@spaces.GPU(duration=get_duration)
+def generate(prompt, negative_prompt, input_image_filepath, input_video_filepath,
+             height_ui, width_ui, mode,
+             duration_ui, 
+             ui_frames_to_use,
+             seed_ui, randomize_seed, ui_guidance_scale, improve_texture_flag,
+             progress=gr.Progress(track_tqdm=True)):
+
+    if randomize_seed:
+        seed_ui = random.randint(0, 2**32 - 1)
+    seed_everething(int(seed_ui))
+    
+    target_frames_ideal = duration_ui * FPS
+    target_frames_rounded = round(target_frames_ideal)
+    if target_frames_rounded < 1: 
+        target_frames_rounded = 1
+    
+    n_val = round((float(target_frames_rounded) - 1.0) / 8.0)
+    actual_num_frames = int(n_val * 8 + 1)
+
+    actual_num_frames = max(9, actual_num_frames)
+    actual_num_frames = min(MAX_NUM_FRAMES, actual_num_frames)
+    
+    actual_height = int(height_ui)
+    actual_width = int(width_ui)
+
+    height_padded = ((actual_height - 1) // 32 + 1) * 32
+    width_padded = ((actual_width - 1) // 32 + 1) * 32
+    num_frames_padded = ((actual_num_frames - 2) // 8 + 1) * 8 + 1 
+    if num_frames_padded != actual_num_frames:
+        print(f"Warning: actual_num_frames ({actual_num_frames}) and num_frames_padded ({num_frames_padded}) differ. Using num_frames_padded for pipeline.")
+    
+    padding_values = calculate_padding(actual_height, actual_width, height_padded, width_padded)
+
+    call_kwargs = {
+        "prompt": prompt,
+        "negative_prompt": negative_prompt,
+        "height": height_padded,
+        "width": width_padded,
+        "num_frames": num_frames_padded, 
+        "frame_rate": int(FPS), 
+        "generator": torch.Generator(device=target_inference_device).manual_seed(int(seed_ui)),
+        "output_type": "pt", 
+        "conditioning_items": None,
+        "media_items": None,
+        "decode_timestep": PIPELINE_CONFIG_YAML["decode_timestep"],
+        "decode_noise_scale": PIPELINE_CONFIG_YAML["decode_noise_scale"],
+        "stochastic_sampling": PIPELINE_CONFIG_YAML["stochastic_sampling"],
+        "image_cond_noise_scale": 0.15,
+        "is_video": True,
+        "vae_per_channel_normalize": True,
+        "mixed_precision": (PIPELINE_CONFIG_YAML["precision"] == "mixed_precision"),
+        "offload_to_cpu": False,
+        "enhance_prompt": False,
+    }
+
+    stg_mode_str = PIPELINE_CONFIG_YAML.get("stg_mode", "attention_values")
+    if stg_mode_str.lower() in ["stg_av", "attention_values"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionValues
+    elif stg_mode_str.lower() in ["stg_as", "attention_skip"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.AttentionSkip
+    elif stg_mode_str.lower() in ["stg_r", "residual"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.Residual
+    elif stg_mode_str.lower() in ["stg_t", "transformer_block"]:
+        call_kwargs["skip_layer_strategy"] = SkipLayerStrategy.TransformerBlock
+    else:
+        raise ValueError(f"Invalid stg_mode: {stg_mode_str}")
+
+    if mode == "image-to-video" and input_image_filepath:
+        try:
+            media_tensor = load_image_to_tensor_with_resize_and_crop(
+                input_image_filepath, actual_height, actual_width
+            )
+            media_tensor = torch.nn.functional.pad(media_tensor, padding_values)
+            call_kwargs["conditioning_items"] = [ConditioningItem(media_tensor.to(target_inference_device), 0, 1.0)]
+        except Exception as e:
+            print(f"Error loading image {input_image_filepath}: {e}")
+            raise gr.Error(f"Could not load image: {e}")
+    elif mode == "video-to-video" and input_video_filepath:
+        try:
+            call_kwargs["media_items"] = load_media_file(
+                media_path=input_video_filepath,
+                height=actual_height, 
+                width=actual_width,
+                max_frames=int(ui_frames_to_use), 
+                padding=padding_values
+            ).to(target_inference_device)
+        except Exception as e:
+            print(f"Error loading video {input_video_filepath}: {e}")
+            raise gr.Error(f"Could not load video: {e}")
+
+    print(f"Moving models to {target_inference_device} for inference (if not already there)...")
+    
+    active_latent_upsampler = None
+    if improve_texture_flag and latent_upsampler_instance:
+        active_latent_upsampler = latent_upsampler_instance
+
+    result_images_tensor = None
+    if improve_texture_flag:
+        if not active_latent_upsampler:
+            raise gr.Error("Spatial upscaler model not loaded or improve_texture not selected, cannot use multi-scale.")
+        
+        multi_scale_pipeline_obj = LTXMultiScalePipeline(pipeline_instance, active_latent_upsampler)
+        
+        first_pass_args = PIPELINE_CONFIG_YAML.get("first_pass", {}).copy()
+        first_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
+        # num_inference_steps will be derived from len(timesteps) in the pipeline
+        first_pass_args.pop("num_inference_steps", None)
+
+
+        second_pass_args = PIPELINE_CONFIG_YAML.get("second_pass", {}).copy()
+        second_pass_args["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
+        # num_inference_steps will be derived from len(timesteps) in the pipeline
+        second_pass_args.pop("num_inference_steps", None)
+        
+        multi_scale_call_kwargs = call_kwargs.copy()
+        multi_scale_call_kwargs.update({
+            "downscale_factor": PIPELINE_CONFIG_YAML["downscale_factor"],
+            "first_pass": first_pass_args,
+            "second_pass": second_pass_args,
+        })
+        
+        print(f"Calling multi-scale pipeline (eff. HxW: {actual_height}x{actual_width}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
+        result_images_tensor = multi_scale_pipeline_obj(**multi_scale_call_kwargs).images
+    else:
+        single_pass_call_kwargs = call_kwargs.copy()
+        first_pass_config_from_yaml = PIPELINE_CONFIG_YAML.get("first_pass", {})
+
+        single_pass_call_kwargs["timesteps"] = first_pass_config_from_yaml.get("timesteps")
+        single_pass_call_kwargs["guidance_scale"] = float(ui_guidance_scale) # UI overrides YAML
+        single_pass_call_kwargs["stg_scale"] = first_pass_config_from_yaml.get("stg_scale")
+        single_pass_call_kwargs["rescaling_scale"] = first_pass_config_from_yaml.get("rescaling_scale")
+        single_pass_call_kwargs["skip_block_list"] = first_pass_config_from_yaml.get("skip_block_list")
+        
+        # Remove keys that might conflict or are not used in single pass / handled by above
+        single_pass_call_kwargs.pop("num_inference_steps", None) 
+        single_pass_call_kwargs.pop("first_pass", None) 
+        single_pass_call_kwargs.pop("second_pass", None)
+        single_pass_call_kwargs.pop("downscale_factor", None)
+        
+        print(f"Calling base pipeline (padded HxW: {height_padded}x{width_padded}, Frames: {actual_num_frames} -> Padded: {num_frames_padded}) on {target_inference_device}")
+        result_images_tensor = pipeline_instance(**single_pass_call_kwargs).images
+
+    if result_images_tensor is None:
+        raise gr.Error("Generation failed.")
+
+    pad_left, pad_right, pad_top, pad_bottom = padding_values
+    slice_h_end = -pad_bottom if pad_bottom > 0 else None
+    slice_w_end = -pad_right if pad_right > 0 else None
+    
+    result_images_tensor = result_images_tensor[
+        :, :, :actual_num_frames, pad_top:slice_h_end, pad_left:slice_w_end
+    ]
+
+    video_np = result_images_tensor[0].permute(1, 2, 3, 0).cpu().float().numpy()
+    
+    video_np = np.clip(video_np, 0, 1) 
+    video_np = (video_np * 255).astype(np.uint8)
+
+    temp_dir = tempfile.mkdtemp()
+    timestamp = random.randint(10000,99999)
+    output_video_path = os.path.join(temp_dir, f"output_{timestamp}.mp4")
+    
+    try:
+        with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], macro_block_size=1) as video_writer:
+            for frame_idx in range(video_np.shape[0]):
+                progress(frame_idx / video_np.shape[0], desc="Saving video")
+                video_writer.append_data(video_np[frame_idx])
+    except Exception as e:
+        print(f"Error saving video with macro_block_size=1: {e}")
+        try:
+            with imageio.get_writer(output_video_path, fps=call_kwargs["frame_rate"], format='FFMPEG', codec='libx264', quality=8) as video_writer:
+                 for frame_idx in range(video_np.shape[0]):
+                    progress(frame_idx / video_np.shape[0], desc="Saving video (fallback ffmpeg)")
+                    video_writer.append_data(video_np[frame_idx])
+        except Exception as e2:
+            print(f"Fallback video saving error: {e2}")
+            raise gr.Error(f"Failed to save video: {e2}")
+            
+    return output_video_path, seed_ui
+
+def update_task_image():
+    return "image-to-video"
+
+def update_task_text():
+    return "text-to-video"
+
+def update_task_video():
+    return "video-to-video"
+
+# --- Gradio UI Definition ---
+css="""
+#col-container {
+    margin: 0 auto;
+    max-width: 900px;
+}
+"""
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown("# LTX Video 0.9.7 Distilled")
+    gr.Markdown("Fast high quality video generation. [Model](https://huggingface.co/Lightricks/LTX-Video/blob/main/ltxv-13b-0.9.7-distilled.safetensors) [GitHub](https://github.com/Lightricks/LTX-Video) [Diffusers](#)")
+    
+    with gr.Row():
+        with gr.Column():
+            with gr.Tab("image-to-video") as image_tab:
+                video_i_hidden = gr.Textbox(label="video_i", visible=False, value=None)
+                image_i2v = gr.Image(label="Input Image", type="filepath", sources=["upload", "webcam", "clipboard"])
+                i2v_prompt = gr.Textbox(label="Prompt", value="The creature from the image starts to move", lines=3)
+                i2v_button = gr.Button("Generate Image-to-Video", variant="primary")
+            with gr.Tab("text-to-video") as text_tab:
+                image_n_hidden = gr.Textbox(label="image_n", visible=False, value=None)
+                video_n_hidden = gr.Textbox(label="video_n", visible=False, value=None)
+                t2v_prompt = gr.Textbox(label="Prompt", value="A majestic dragon flying over a medieval castle", lines=3)
+                t2v_button = gr.Button("Generate Text-to-Video", variant="primary")
+            with gr.Tab("video-to-video", visible=False) as video_tab:
+                image_v_hidden = gr.Textbox(label="image_v", visible=False, value=None)
+                video_v2v = gr.Video(label="Input Video", sources=["upload", "webcam"]) # type defaults to filepath
+                frames_to_use = gr.Slider(label="Frames to use from input video", minimum=9, maximum=MAX_NUM_FRAMES, value=9, step=8, info="Number of initial frames to use for conditioning/transformation. Must be N*8+1.")
+                v2v_prompt = gr.Textbox(label="Prompt", value="Change the style to cinematic anime", lines=3)
+                v2v_button = gr.Button("Generate Video-to-Video", variant="primary")
+
+            duration_input = gr.Slider(
+                label="Video Duration (seconds)", 
+                minimum=0.3, 
+                maximum=8.5, 
+                value=2,  
+                step=0.1, 
+                info=f"Target video duration (0.3s to 8.5s)"
+            )
+            improve_texture = gr.Checkbox(label="Improve Texture (multi-scale)", value=True, info="Uses a two-pass generation for better quality, but is slower. Recommended for final output.")
+
+        with gr.Column():
+            output_video = gr.Video(label="Generated Video", interactive=False)
+            # gr.DeepLinkButton()
+
+    with gr.Accordion("Advanced settings", open=False):
+        mode = gr.Dropdown(["text-to-video", "image-to-video", "video-to-video"], label="task", value="image-to-video", visible=False)
+        negative_prompt_input = gr.Textbox(label="Negative Prompt", value="worst quality, inconsistent motion, blurry, jittery, distorted", lines=2)
+        with gr.Row():
+            seed_input = gr.Number(label="Seed", value=42, precision=0, minimum=0, maximum=2**32-1)
+            randomize_seed_input = gr.Checkbox(label="Randomize Seed", value=True)
+        with gr.Row():
+            guidance_scale_input = gr.Slider(label="Guidance Scale (CFG)", minimum=1.0, maximum=10.0, value=PIPELINE_CONFIG_YAML.get("first_pass", {}).get("guidance_scale", 1.0), step=0.1, info="Controls how much the prompt influences the output. Higher values = stronger influence.")
+        with gr.Row():
+            height_input = gr.Slider(label="Height", value=512, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
+            width_input = gr.Slider(label="Width", value=704, step=32, minimum=MIN_DIM_SLIDER, maximum=MAX_IMAGE_SIZE, info="Must be divisible by 32.")
+
+
+    # --- Event handlers for updating dimensions on upload ---
+    def handle_image_upload_for_dims(image_filepath, current_h, current_w):
+        if not image_filepath:  # Image cleared or no image initially
+            # Keep current slider values if image is cleared or no input
+            return gr.update(value=current_h), gr.update(value=current_w)
+        try:
+            img = Image.open(image_filepath)
+            orig_w, orig_h = img.size
+            new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
+            return gr.update(value=new_h), gr.update(value=new_w)
+        except Exception as e:
+            print(f"Error processing image for dimension update: {e}")
+            # Keep current slider values on error
+            return gr.update(value=current_h), gr.update(value=current_w)
+
+    def handle_video_upload_for_dims(video_filepath, current_h, current_w):
+        if not video_filepath:  # Video cleared or no video initially
+            return gr.update(value=current_h), gr.update(value=current_w)
+        try:
+            # Ensure video_filepath is a string for os.path.exists and imageio
+            video_filepath_str = str(video_filepath) 
+            if not os.path.exists(video_filepath_str):
+                print(f"Video file path does not exist for dimension update: {video_filepath_str}")
+                return gr.update(value=current_h), gr.update(value=current_w)
+
+            orig_w, orig_h = -1, -1
+            with imageio.get_reader(video_filepath_str) as reader:
+                meta = reader.get_meta_data()
+                if 'size' in meta:
+                    orig_w, orig_h = meta['size']
+                else:
+                    # Fallback: read first frame if 'size' not in metadata
+                    try:
+                        first_frame = reader.get_data(0)
+                        # Shape is (h, w, c) for frames
+                        orig_h, orig_w = first_frame.shape[0], first_frame.shape[1]
+                    except Exception as e_frame:
+                        print(f"Could not get video size from metadata or first frame: {e_frame}")
+                        return gr.update(value=current_h), gr.update(value=current_w)
+            
+            if orig_w == -1 or orig_h == -1: # If dimensions couldn't be determined
+                 print(f"Could not determine dimensions for video: {video_filepath_str}")
+                 return gr.update(value=current_h), gr.update(value=current_w)
+
+            new_h, new_w = calculate_new_dimensions(orig_w, orig_h)
+            return gr.update(value=new_h), gr.update(value=new_w)
+        except Exception as e:
+            # Log type of video_filepath for debugging if it's not a path-like string
+            print(f"Error processing video for dimension update: {e} (Path: {video_filepath}, Type: {type(video_filepath)})")
+            return gr.update(value=current_h), gr.update(value=current_w)
+
+    
+    image_i2v.upload(
+        fn=handle_image_upload_for_dims,
+        inputs=[image_i2v, height_input, width_input],
+        outputs=[height_input, width_input]
+    )
+    video_v2v.upload(
+        fn=handle_video_upload_for_dims,
+        inputs=[video_v2v, height_input, width_input],
+        outputs=[height_input, width_input]
+    )
+
+    image_tab.select(
+        fn=update_task_image,
+        outputs=[mode]
+    )
+    text_tab.select(
+        fn=update_task_text,
+        outputs=[mode]
+    )
+    
+    t2v_inputs = [t2v_prompt, negative_prompt_input, image_n_hidden, video_n_hidden,
+                  height_input, width_input, mode,
+                  duration_input, frames_to_use, 
+                  seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
+    
+    i2v_inputs = [i2v_prompt, negative_prompt_input, image_i2v, video_i_hidden,
+                  height_input, width_input, mode,
+                  duration_input, frames_to_use, 
+                  seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
+
+    v2v_inputs = [v2v_prompt, negative_prompt_input, image_v_hidden, video_v2v,
+                  height_input, width_input, mode,
+                  duration_input, frames_to_use, 
+                  seed_input, randomize_seed_input, guidance_scale_input, improve_texture]
+
+    t2v_button.click(fn=generate, inputs=t2v_inputs, outputs=[output_video, seed_input], api_name="text_to_video")
+    i2v_button.click(fn=generate, inputs=i2v_inputs, outputs=[output_video, seed_input], api_name="image_to_video")
+    v2v_button.click(fn=generate, inputs=v2v_inputs, outputs=[output_video, seed_input], api_name="video_to_video")
+
+if __name__ == "__main__":
+    if os.path.exists(models_dir) and os.path.isdir(models_dir):
+        print(f"Model directory: {Path(models_dir).resolve()}")
+    
+    demo.queue().launch(debug=True, share=False, mcp_server=True)

inference.py

@@ -0,0 +1,774 @@
+import argparse
+import os
+import random
+from datetime import datetime
+from pathlib import Path
+from diffusers.utils import logging
+from typing import Optional, List, Union
+import yaml
+
+import imageio
+import json
+import numpy as np
+import torch
+import cv2
+from safetensors import safe_open
+from PIL import Image
+from transformers import (
+    T5EncoderModel,
+    T5Tokenizer,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    AutoTokenizer,
+)
+from huggingface_hub import hf_hub_download
+
+from ltx_video.models.autoencoders.causal_video_autoencoder import (
+    CausalVideoAutoencoder,
+)
+from ltx_video.models.transformers.symmetric_patchifier import SymmetricPatchifier
+from ltx_video.models.transformers.transformer3d import Transformer3DModel
+from ltx_video.pipelines.pipeline_ltx_video import (
+    ConditioningItem,
+    LTXVideoPipeline,
+    LTXMultiScalePipeline,
+)
+from ltx_video.schedulers.rf import RectifiedFlowScheduler
+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
+from ltx_video.models.autoencoders.latent_upsampler import LatentUpsampler
+import ltx_video.pipelines.crf_compressor as crf_compressor
+
+MAX_HEIGHT = 720
+MAX_WIDTH = 1280
+MAX_NUM_FRAMES = 257
+
+logger = logging.get_logger("LTX-Video")
+
+
+def get_total_gpu_memory():
+    if torch.cuda.is_available():
+        total_memory = torch.cuda.get_device_properties(0).total_memory / (1024**3)
+        return total_memory
+    return 0
+
+
+def get_device():
+    if torch.cuda.is_available():
+        return "cuda"
+    elif torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+def load_image_to_tensor_with_resize_and_crop(
+    image_input: Union[str, Image.Image],
+    target_height: int = 512,
+    target_width: int = 768,
+    just_crop: bool = False,
+) -> torch.Tensor:
+    """Load and process an image into a tensor.
+
+    Args:
+        image_input: Either a file path (str) or a PIL Image object
+        target_height: Desired height of output tensor
+        target_width: Desired width of output tensor
+        just_crop: If True, only crop the image to the target size without resizing
+    """
+    if isinstance(image_input, str):
+        image = Image.open(image_input).convert("RGB")
+    elif isinstance(image_input, Image.Image):
+        image = image_input
+    else:
+        raise ValueError("image_input must be either a file path or a PIL Image object")
+
+    input_width, input_height = image.size
+    aspect_ratio_target = target_width / target_height
+    aspect_ratio_frame = input_width / input_height
+    if aspect_ratio_frame > aspect_ratio_target:
+        new_width = int(input_height * aspect_ratio_target)
+        new_height = input_height
+        x_start = (input_width - new_width) // 2
+        y_start = 0
+    else:
+        new_width = input_width
+        new_height = int(input_width / aspect_ratio_target)
+        x_start = 0
+        y_start = (input_height - new_height) // 2
+
+    image = image.crop((x_start, y_start, x_start + new_width, y_start + new_height))
+    if not just_crop:
+        image = image.resize((target_width, target_height))
+
+    image = np.array(image)
+    image = cv2.GaussianBlur(image, (3, 3), 0)
+    frame_tensor = torch.from_numpy(image).float()
+    frame_tensor = crf_compressor.compress(frame_tensor / 255.0) * 255.0
+    frame_tensor = frame_tensor.permute(2, 0, 1)
+    frame_tensor = (frame_tensor / 127.5) - 1.0
+    # Create 5D tensor: (batch_size=1, channels=3, num_frames=1, height, width)
+    return frame_tensor.unsqueeze(0).unsqueeze(2)
+
+
+def calculate_padding(
+    source_height: int, source_width: int, target_height: int, target_width: int
+) -> tuple[int, int, int, int]:
+
+    # Calculate total padding needed
+    pad_height = target_height - source_height
+    pad_width = target_width - source_width
+
+    # Calculate padding for each side
+    pad_top = pad_height // 2
+    pad_bottom = pad_height - pad_top  # Handles odd padding
+    pad_left = pad_width // 2
+    pad_right = pad_width - pad_left  # Handles odd padding
+
+    # Return padded tensor
+    # Padding format is (left, right, top, bottom)
+    padding = (pad_left, pad_right, pad_top, pad_bottom)
+    return padding
+
+
+def convert_prompt_to_filename(text: str, max_len: int = 20) -> str:
+    # Remove non-letters and convert to lowercase
+    clean_text = "".join(
+        char.lower() for char in text if char.isalpha() or char.isspace()
+    )
+
+    # Split into words
+    words = clean_text.split()
+
+    # Build result string keeping track of length
+    result = []
+    current_length = 0
+
+    for word in words:
+        # Add word length plus 1 for underscore (except for first word)
+        new_length = current_length + len(word)
+
+        if new_length <= max_len:
+            result.append(word)
+            current_length += len(word)
+        else:
+            break
+
+    return "-".join(result)
+
+
+# Generate output video name
+def get_unique_filename(
+    base: str,
+    ext: str,
+    prompt: str,
+    seed: int,
+    resolution: tuple[int, int, int],
+    dir: Path,
+    endswith=None,
+    index_range=1000,
+) -> Path:
+    base_filename = f"{base}_{convert_prompt_to_filename(prompt, max_len=30)}_{seed}_{resolution[0]}x{resolution[1]}x{resolution[2]}"
+    for i in range(index_range):
+        filename = dir / f"{base_filename}_{i}{endswith if endswith else ''}{ext}"
+        if not os.path.exists(filename):
+            return filename
+    raise FileExistsError(
+        f"Could not find a unique filename after {index_range} attempts."
+    )
+
+
+def seed_everething(seed: int):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed(seed)
+    if torch.backends.mps.is_available():
+        torch.mps.manual_seed(seed)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Load models from separate directories and run the pipeline."
+    )
+
+    # Directories
+    parser.add_argument(
+        "--output_path",
+        type=str,
+        default=None,
+        help="Path to the folder to save output video, if None will save in outputs/ directory.",
+    )
+    parser.add_argument("--seed", type=int, default="171198")
+
+    # Pipeline parameters
+    parser.add_argument(
+        "--num_images_per_prompt",
+        type=int,
+        default=1,
+        help="Number of images per prompt",
+    )
+    parser.add_argument(
+        "--image_cond_noise_scale",
+        type=float,
+        default=0.15,
+        help="Amount of noise to add to the conditioned image",
+    )
+    parser.add_argument(
+        "--height",
+        type=int,
+        default=704,
+        help="Height of the output video frames. Optional if an input image provided.",
+    )
+    parser.add_argument(
+        "--width",
+        type=int,
+        default=1216,
+        help="Width of the output video frames. If None will infer from input image.",
+    )
+    parser.add_argument(
+        "--num_frames",
+        type=int,
+        default=121,
+        help="Number of frames to generate in the output video",
+    )
+    parser.add_argument(
+        "--frame_rate", type=int, default=30, help="Frame rate for the output video"
+    )
+    parser.add_argument(
+        "--device",
+        default=None,
+        help="Device to run inference on. If not specified, will automatically detect and use CUDA or MPS if available, else CPU.",
+    )
+    parser.add_argument(
+        "--pipeline_config",
+        type=str,
+        default="configs/ltxv-13b-0.9.7-dev.yaml",
+        help="The path to the config file for the pipeline, which contains the parameters for the pipeline",
+    )
+
+    # Prompts
+    parser.add_argument(
+        "--prompt",
+        type=str,
+        help="Text prompt to guide generation",
+    )
+    parser.add_argument(
+        "--negative_prompt",
+        type=str,
+        default="worst quality, inconsistent motion, blurry, jittery, distorted",
+        help="Negative prompt for undesired features",
+    )
+
+    parser.add_argument(
+        "--offload_to_cpu",
+        action="store_true",
+        help="Offloading unnecessary computations to CPU.",
+    )
+
+    # video-to-video arguments:
+    parser.add_argument(
+        "--input_media_path",
+        type=str,
+        default=None,
+        help="Path to the input video (or imaage) to be modified using the video-to-video pipeline",
+    )
+
+    # Conditioning arguments
+    parser.add_argument(
+        "--conditioning_media_paths",
+        type=str,
+        nargs="*",
+        help="List of paths to conditioning media (images or videos). Each path will be used as a conditioning item.",
+    )
+    parser.add_argument(
+        "--conditioning_strengths",
+        type=float,
+        nargs="*",
+        help="List of conditioning strengths (between 0 and 1) for each conditioning item. Must match the number of conditioning items.",
+    )
+    parser.add_argument(
+        "--conditioning_start_frames",
+        type=int,
+        nargs="*",
+        help="List of frame indices where each conditioning item should be applied. Must match the number of conditioning items.",
+    )
+
+    args = parser.parse_args()
+    logger.warning(f"Running generation with arguments: {args}")
+    infer(**vars(args))
+
+
+def create_ltx_video_pipeline(
+    ckpt_path: str,
+    precision: str,
+    text_encoder_model_name_or_path: str,
+    sampler: Optional[str] = None,
+    device: Optional[str] = None,
+    enhance_prompt: bool = False,
+    prompt_enhancer_image_caption_model_name_or_path: Optional[str] = None,
+    prompt_enhancer_llm_model_name_or_path: Optional[str] = None,
+) -> LTXVideoPipeline:
+    ckpt_path = Path(ckpt_path)
+    assert os.path.exists(
+        ckpt_path
+    ), f"Ckpt path provided (--ckpt_path) {ckpt_path} does not exist"
+
+    with safe_open(ckpt_path, framework="pt") as f:
+        metadata = f.metadata()
+        config_str = metadata.get("config")
+        configs = json.loads(config_str)
+        allowed_inference_steps = configs.get("allowed_inference_steps", None)
+
+    vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)
+    transformer = Transformer3DModel.from_pretrained(ckpt_path)
+
+    # Use constructor if sampler is specified, otherwise use from_pretrained
+    if sampler == "from_checkpoint" or not sampler:
+        scheduler = RectifiedFlowScheduler.from_pretrained(ckpt_path)
+    else:
+        scheduler = RectifiedFlowScheduler(
+            sampler=("Uniform" if sampler.lower() == "uniform" else "LinearQuadratic")
+        )
+
+    text_encoder = T5EncoderModel.from_pretrained(
+        text_encoder_model_name_or_path, subfolder="text_encoder"
+    )
+    patchifier = SymmetricPatchifier(patch_size=1)
+    tokenizer = T5Tokenizer.from_pretrained(
+        text_encoder_model_name_or_path, subfolder="tokenizer"
+    )
+
+    transformer = transformer.to(device)
+    vae = vae.to(device)
+    text_encoder = text_encoder.to(device)
+
+    if enhance_prompt:
+        prompt_enhancer_image_caption_model = AutoModelForCausalLM.from_pretrained(
+            prompt_enhancer_image_caption_model_name_or_path, trust_remote_code=True
+        )
+        prompt_enhancer_image_caption_processor = AutoProcessor.from_pretrained(
+            prompt_enhancer_image_caption_model_name_or_path, trust_remote_code=True
+        )
+        prompt_enhancer_llm_model = AutoModelForCausalLM.from_pretrained(
+            prompt_enhancer_llm_model_name_or_path,
+            torch_dtype="bfloat16",
+        )
+        prompt_enhancer_llm_tokenizer = AutoTokenizer.from_pretrained(
+            prompt_enhancer_llm_model_name_or_path,
+        )
+    else:
+        prompt_enhancer_image_caption_model = None
+        prompt_enhancer_image_caption_processor = None
+        prompt_enhancer_llm_model = None
+        prompt_enhancer_llm_tokenizer = None
+
+    vae = vae.to(torch.bfloat16)
+    if precision == "bfloat16" and transformer.dtype != torch.bfloat16:
+        transformer = transformer.to(torch.bfloat16)
+    text_encoder = text_encoder.to(torch.bfloat16)
+
+    # Use submodels for the pipeline
+    submodel_dict = {
+        "transformer": transformer,
+        "patchifier": patchifier,
+        "text_encoder": text_encoder,
+        "tokenizer": tokenizer,
+        "scheduler": scheduler,
+        "vae": vae,
+        "prompt_enhancer_image_caption_model": prompt_enhancer_image_caption_model,
+        "prompt_enhancer_image_caption_processor": prompt_enhancer_image_caption_processor,
+        "prompt_enhancer_llm_model": prompt_enhancer_llm_model,
+        "prompt_enhancer_llm_tokenizer": prompt_enhancer_llm_tokenizer,
+        "allowed_inference_steps": allowed_inference_steps,
+    }
+
+    pipeline = LTXVideoPipeline(**submodel_dict)
+    pipeline = pipeline.to(device)
+    return pipeline
+
+
+def create_latent_upsampler(latent_upsampler_model_path: str, device: str):
+    latent_upsampler = LatentUpsampler.from_pretrained(latent_upsampler_model_path)
+    latent_upsampler.to(device)
+    latent_upsampler.eval()
+    return latent_upsampler
+
+
+def infer(
+    output_path: Optional[str],
+    seed: int,
+    pipeline_config: str,
+    image_cond_noise_scale: float,
+    height: Optional[int],
+    width: Optional[int],
+    num_frames: int,
+    frame_rate: int,
+    prompt: str,
+    negative_prompt: str,
+    offload_to_cpu: bool,
+    input_media_path: Optional[str] = None,
+    conditioning_media_paths: Optional[List[str]] = None,
+    conditioning_strengths: Optional[List[float]] = None,
+    conditioning_start_frames: Optional[List[int]] = None,
+    device: Optional[str] = None,
+    **kwargs,
+):
+    # check if pipeline_config is a file
+    if not os.path.isfile(pipeline_config):
+        raise ValueError(f"Pipeline config file {pipeline_config} does not exist")
+    with open(pipeline_config, "r") as f:
+        pipeline_config = yaml.safe_load(f)
+
+    models_dir = "MODEL_DIR"
+
+    ltxv_model_name_or_path = pipeline_config["checkpoint_path"]
+    if not os.path.isfile(ltxv_model_name_or_path):
+        ltxv_model_path = hf_hub_download(
+            repo_id="Lightricks/LTX-Video",
+            filename=ltxv_model_name_or_path,
+            local_dir=models_dir,
+            repo_type="model",
+        )
+    else:
+        ltxv_model_path = ltxv_model_name_or_path
+
+    spatial_upscaler_model_name_or_path = pipeline_config.get(
+        "spatial_upscaler_model_path"
+    )
+    if spatial_upscaler_model_name_or_path and not os.path.isfile(
+        spatial_upscaler_model_name_or_path
+    ):
+        spatial_upscaler_model_path = hf_hub_download(
+            repo_id="Lightricks/LTX-Video",
+            filename=spatial_upscaler_model_name_or_path,
+            local_dir=models_dir,
+            repo_type="model",
+        )
+    else:
+        spatial_upscaler_model_path = spatial_upscaler_model_name_or_path
+
+    if kwargs.get("input_image_path", None):
+        logger.warning(
+            "Please use conditioning_media_paths instead of input_image_path."
+        )
+        assert not conditioning_media_paths and not conditioning_start_frames
+        conditioning_media_paths = [kwargs["input_image_path"]]
+        conditioning_start_frames = [0]
+
+    # Validate conditioning arguments
+    if conditioning_media_paths:
+        # Use default strengths of 1.0
+        if not conditioning_strengths:
+            conditioning_strengths = [1.0] * len(conditioning_media_paths)
+        if not conditioning_start_frames:
+            raise ValueError(
+                "If `conditioning_media_paths` is provided, "
+                "`conditioning_start_frames` must also be provided"
+            )
+        if len(conditioning_media_paths) != len(conditioning_strengths) or len(
+            conditioning_media_paths
+        ) != len(conditioning_start_frames):
+            raise ValueError(
+                "`conditioning_media_paths`, `conditioning_strengths`, "
+                "and `conditioning_start_frames` must have the same length"
+            )
+        if any(s < 0 or s > 1 for s in conditioning_strengths):
+            raise ValueError("All conditioning strengths must be between 0 and 1")
+        if any(f < 0 or f >= num_frames for f in conditioning_start_frames):
+            raise ValueError(
+                f"All conditioning start frames must be between 0 and {num_frames-1}"
+            )
+
+    seed_everething(seed)
+    if offload_to_cpu and not torch.cuda.is_available():
+        logger.warning(
+            "offload_to_cpu is set to True, but offloading will not occur since the model is already running on CPU."
+        )
+        offload_to_cpu = False
+    else:
+        offload_to_cpu = offload_to_cpu and get_total_gpu_memory() < 30
+
+    output_dir = (
+        Path(output_path)
+        if output_path
+        else Path(f"outputs/{datetime.today().strftime('%Y-%m-%d')}")
+    )
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Adjust dimensions to be divisible by 32 and num_frames to be (N * 8 + 1)
+    height_padded = ((height - 1) // 32 + 1) * 32
+    width_padded = ((width - 1) // 32 + 1) * 32
+    num_frames_padded = ((num_frames - 2) // 8 + 1) * 8 + 1
+
+    padding = calculate_padding(height, width, height_padded, width_padded)
+
+    logger.warning(
+        f"Padded dimensions: {height_padded}x{width_padded}x{num_frames_padded}"
+    )
+
+    prompt_enhancement_words_threshold = pipeline_config[
+        "prompt_enhancement_words_threshold"
+    ]
+
+    prompt_word_count = len(prompt.split())
+    enhance_prompt = (
+        prompt_enhancement_words_threshold > 0
+        and prompt_word_count < prompt_enhancement_words_threshold
+    )
+
+    if prompt_enhancement_words_threshold > 0 and not enhance_prompt:
+        logger.info(
+            f"Prompt has {prompt_word_count} words, which exceeds the threshold of {prompt_enhancement_words_threshold}. Prompt enhancement disabled."
+        )
+
+    precision = pipeline_config["precision"]
+    text_encoder_model_name_or_path = pipeline_config["text_encoder_model_name_or_path"]
+    sampler = pipeline_config["sampler"]
+    prompt_enhancer_image_caption_model_name_or_path = pipeline_config[
+        "prompt_enhancer_image_caption_model_name_or_path"
+    ]
+    prompt_enhancer_llm_model_name_or_path = pipeline_config[
+        "prompt_enhancer_llm_model_name_or_path"
+    ]
+
+    pipeline = create_ltx_video_pipeline(
+        ckpt_path=ltxv_model_path,
+        precision=precision,
+        text_encoder_model_name_or_path=text_encoder_model_name_or_path,
+        sampler=sampler,
+        device=kwargs.get("device", get_device()),
+        enhance_prompt=enhance_prompt,
+        prompt_enhancer_image_caption_model_name_or_path=prompt_enhancer_image_caption_model_name_or_path,
+        prompt_enhancer_llm_model_name_or_path=prompt_enhancer_llm_model_name_or_path,
+    )
+
+    if pipeline_config.get("pipeline_type", None) == "multi-scale":
+        if not spatial_upscaler_model_path:
+            raise ValueError(
+                "spatial upscaler model path is missing from pipeline config file and is required for multi-scale rendering"
+            )
+        latent_upsampler = create_latent_upsampler(
+            spatial_upscaler_model_path, pipeline.device
+        )
+        pipeline = LTXMultiScalePipeline(pipeline, latent_upsampler=latent_upsampler)
+
+    media_item = None
+    if input_media_path:
+        media_item = load_media_file(
+            media_path=input_media_path,
+            height=height,
+            width=width,
+            max_frames=num_frames_padded,
+            padding=padding,
+        )
+
+    conditioning_items = (
+        prepare_conditioning(
+            conditioning_media_paths=conditioning_media_paths,
+            conditioning_strengths=conditioning_strengths,
+            conditioning_start_frames=conditioning_start_frames,
+            height=height,
+            width=width,
+            num_frames=num_frames,
+            padding=padding,
+            pipeline=pipeline,
+        )
+        if conditioning_media_paths
+        else None
+    )
+
+    stg_mode = pipeline_config.get("stg_mode", "attention_values")
+    del pipeline_config["stg_mode"]
+    if stg_mode.lower() == "stg_av" or stg_mode.lower() == "attention_values":
+        skip_layer_strategy = SkipLayerStrategy.AttentionValues
+    elif stg_mode.lower() == "stg_as" or stg_mode.lower() == "attention_skip":
+        skip_layer_strategy = SkipLayerStrategy.AttentionSkip
+    elif stg_mode.lower() == "stg_r" or stg_mode.lower() == "residual":
+        skip_layer_strategy = SkipLayerStrategy.Residual
+    elif stg_mode.lower() == "stg_t" or stg_mode.lower() == "transformer_block":
+        skip_layer_strategy = SkipLayerStrategy.TransformerBlock
+    else:
+        raise ValueError(f"Invalid spatiotemporal guidance mode: {stg_mode}")
+
+    # Prepare input for the pipeline
+    sample = {
+        "prompt": prompt,
+        "prompt_attention_mask": None,
+        "negative_prompt": negative_prompt,
+        "negative_prompt_attention_mask": None,
+    }
+
+    device = device or get_device()
+    generator = torch.Generator(device=device).manual_seed(seed)
+
+    images = pipeline(
+        **pipeline_config,
+        skip_layer_strategy=skip_layer_strategy,
+        generator=generator,
+        output_type="pt",
+        callback_on_step_end=None,
+        height=height_padded,
+        width=width_padded,
+        num_frames=num_frames_padded,
+        frame_rate=frame_rate,
+        **sample,
+        media_items=media_item,
+        conditioning_items=conditioning_items,
+        is_video=True,
+        vae_per_channel_normalize=True,
+        image_cond_noise_scale=image_cond_noise_scale,
+        mixed_precision=(precision == "mixed_precision"),
+        offload_to_cpu=offload_to_cpu,
+        device=device,
+        enhance_prompt=enhance_prompt,
+    ).images
+
+    # Crop the padded images to the desired resolution and number of frames
+    (pad_left, pad_right, pad_top, pad_bottom) = padding
+    pad_bottom = -pad_bottom
+    pad_right = -pad_right
+    if pad_bottom == 0:
+        pad_bottom = images.shape[3]
+    if pad_right == 0:
+        pad_right = images.shape[4]
+    images = images[:, :, :num_frames, pad_top:pad_bottom, pad_left:pad_right]
+
+    for i in range(images.shape[0]):
+        # Gathering from B, C, F, H, W to C, F, H, W and then permuting to F, H, W, C
+        video_np = images[i].permute(1, 2, 3, 0).cpu().float().numpy()
+        # Unnormalizing images to [0, 255] range
+        video_np = (video_np * 255).astype(np.uint8)
+        fps = frame_rate
+        height, width = video_np.shape[1:3]
+        # In case a single image is generated
+        if video_np.shape[0] == 1:
+            output_filename = get_unique_filename(
+                f"image_output_{i}",
+                ".png",
+                prompt=prompt,
+                seed=seed,
+                resolution=(height, width, num_frames),
+                dir=output_dir,
+            )
+            imageio.imwrite(output_filename, video_np[0])
+        else:
+            output_filename = get_unique_filename(
+                f"video_output_{i}",
+                ".mp4",
+                prompt=prompt,
+                seed=seed,
+                resolution=(height, width, num_frames),
+                dir=output_dir,
+            )
+
+            # Write video
+            with imageio.get_writer(output_filename, fps=fps) as video:
+                for frame in video_np:
+                    video.append_data(frame)
+
+        logger.warning(f"Output saved to {output_filename}")
+
+
+def prepare_conditioning(
+    conditioning_media_paths: List[str],
+    conditioning_strengths: List[float],
+    conditioning_start_frames: List[int],
+    height: int,
+    width: int,
+    num_frames: int,
+    padding: tuple[int, int, int, int],
+    pipeline: LTXVideoPipeline,
+) -> Optional[List[ConditioningItem]]:
+    """Prepare conditioning items based on input media paths and their parameters.
+
+    Args:
+        conditioning_media_paths: List of paths to conditioning media (images or videos)
+        conditioning_strengths: List of conditioning strengths for each media item
+        conditioning_start_frames: List of frame indices where each item should be applied
+        height: Height of the output frames
+        width: Width of the output frames
+        num_frames: Number of frames in the output video
+        padding: Padding to apply to the frames
+        pipeline: LTXVideoPipeline object used for condition video trimming
+
+    Returns:
+        A list of ConditioningItem objects.
+    """
+    conditioning_items = []
+    for path, strength, start_frame in zip(
+        conditioning_media_paths, conditioning_strengths, conditioning_start_frames
+    ):
+        num_input_frames = orig_num_input_frames = get_media_num_frames(path)
+        if hasattr(pipeline, "trim_conditioning_sequence") and callable(
+            getattr(pipeline, "trim_conditioning_sequence")
+        ):
+            num_input_frames = pipeline.trim_conditioning_sequence(
+                start_frame, orig_num_input_frames, num_frames
+            )
+        if num_input_frames < orig_num_input_frames:
+            logger.warning(
+                f"Trimming conditioning video {path} from {orig_num_input_frames} to {num_input_frames} frames."
+            )
+
+        media_tensor = load_media_file(
+            media_path=path,
+            height=height,
+            width=width,
+            max_frames=num_input_frames,
+            padding=padding,
+            just_crop=True,
+        )
+        conditioning_items.append(ConditioningItem(media_tensor, start_frame, strength))
+    return conditioning_items
+
+
+def get_media_num_frames(media_path: str) -> int:
+    is_video = any(
+        media_path.lower().endswith(ext) for ext in [".mp4", ".avi", ".mov", ".mkv"]
+    )
+    num_frames = 1
+    if is_video:
+        reader = imageio.get_reader(media_path)
+        num_frames = reader.count_frames()
+        reader.close()
+    return num_frames
+
+
+def load_media_file(
+    media_path: str,
+    height: int,
+    width: int,
+    max_frames: int,
+    padding: tuple[int, int, int, int],
+    just_crop: bool = False,
+) -> torch.Tensor:
+    is_video = any(
+        media_path.lower().endswith(ext) for ext in [".mp4", ".avi", ".mov", ".mkv"]
+    )
+    if is_video:
+        reader = imageio.get_reader(media_path)
+        num_input_frames = min(reader.count_frames(), max_frames)
+
+        # Read and preprocess the relevant frames from the video file.
+        frames = []
+        for i in range(num_input_frames):
+            frame = Image.fromarray(reader.get_data(i))
+            frame_tensor = load_image_to_tensor_with_resize_and_crop(
+                frame, height, width, just_crop=just_crop
+            )
+            frame_tensor = torch.nn.functional.pad(frame_tensor, padding)
+            frames.append(frame_tensor)
+        reader.close()
+
+        # Stack frames along the temporal dimension
+        media_tensor = torch.cat(frames, dim=2)
+    else:  # Input image
+        media_tensor = load_image_to_tensor_with_resize_and_crop(
+            media_path, height, width, just_crop=just_crop
+        )
+        media_tensor = torch.nn.functional.pad(media_tensor, padding)
+    return media_tensor
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,48 +1,15 @@
-pydantic==2.10.6
-fastapi==0.115.8
-gradio_imageslider==0.0.20
-gradio_client==1.7.0
-numpy==1.26.4
-requests==2.32.3
-sentencepiece==0.2.0
-tokenizers==0.19.1
-torchvision==0.18.1
-uvicorn==0.30.1
-wandb==0.17.4
-httpx==0.27.0
-transformers==4.42.4
-accelerate==0.32.1
-scikit-learn==1.5.1
-einops==0.8.0
-einops-exts==0.0.4
-timm==1.0.7
-openai-clip==1.0.1
-fsspec==2024.6.1
-kornia==0.7.3
-matplotlib==3.9.1
-ninja==1.11.1.1
-omegaconf==2.3.0
-opencv-python==4.10.0.84
-pandas==2.2.2
-pillow==10.4.0
-pytorch-lightning==2.3.3
-PyYAML==6.0.1
-scipy==1.14.0
-tqdm==4.66.4
-triton==2.3.1
-urllib3==2.2.2
-webdataset==0.2.86
-xformers==0.0.27
-facexlib==0.3.0
-k-diffusion==0.1.1.post1
-diffusers==0.30.0
-pillow-heif==0.18.0
-
-open-clip-torch==2.24.0
-
-torchaudio
-easydict==1.13
-fairscale==0.4.13
-torchsde==0.2.6
-huggingface_hub==0.23.3
-gradio
+accelerate
+transformers
+sentencepiece
+pillow
+numpy
+torchvision
+huggingface_hub
+spaces
+opencv-python
+imageio
+imageio-ffmpeg
+einops
+timm
+av
+git+https://github.com/huggingface/diffusers.git@main