Commit ·
d365baa
1
Parent(s): 2661777
Add optional Turbo validation toggle
Browse files- __pycache__/app.cpython-312.pyc +0 -0
- __pycache__/jobs.cpython-312.pyc +0 -0
- app.py +11 -4
- jobs.py +8 -3
__pycache__/app.cpython-312.pyc
CHANGED
|
Binary files a/__pycache__/app.cpython-312.pyc and b/__pycache__/app.cpython-312.pyc differ
|
|
|
__pycache__/jobs.cpython-312.pyc
CHANGED
|
Binary files a/__pycache__/jobs.cpython-312.pyc and b/__pycache__/jobs.cpython-312.pyc differ
|
|
|
app.py
CHANGED
|
@@ -91,7 +91,7 @@ def start_training(
|
|
| 91 |
dataset_rows, lora_name, instance_prompt, validation_prompt, rank, lora_alpha, max_train_steps,
|
| 92 |
learning_rate, lr_scheduler, resolution, repeats, train_batch_size, gradient_accumulation_steps,
|
| 93 |
seed, optimizer, use_8bit_adam, cache_latents, gradient_checkpointing, offload, quantization,
|
| 94 |
-
lora_layers, validation_epochs, hub_model_id, flavor, timeout,
|
| 95 |
profile: gr.OAuthProfile | None = None, oauth_token: gr.OAuthToken | None = None,
|
| 96 |
):
|
| 97 |
if oauth_token is None or profile is None:
|
|
@@ -113,7 +113,8 @@ def start_training(
|
|
| 113 |
"optimizer": optimizer, "use_8bit_adam": bool(use_8bit_adam),
|
| 114 |
"cache_latents": bool(cache_latents), "gradient_checkpointing": bool(gradient_checkpointing),
|
| 115 |
"offload": bool(offload), "quantization": quantization, "lora_layers": lora_layers,
|
| 116 |
-
"
|
|
|
|
| 117 |
}
|
| 118 |
try:
|
| 119 |
res = jobs.submit(params, image_paths, captions, flavor=flavor, timeout=timeout)
|
|
@@ -213,6 +214,12 @@ with gr.Blocks(title="Krea 2 LoRA Trainer") as demo:
|
|
| 213 |
train_batch_size = gr.Number(label="Batch size", value=1, precision=0)
|
| 214 |
gradient_accumulation_steps = gr.Number(label="Grad accumulation", value=1, precision=0)
|
| 215 |
with gr.Accordion("Validation", open=False):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
validation_prompt = gr.Textbox(
|
| 217 |
label="Validation prompt", placeholder="(defaults to your trigger)",
|
| 218 |
info="Generated on Turbo every N epochs to preview progress.",
|
|
@@ -259,8 +266,8 @@ with gr.Blocks(title="Krea 2 LoRA Trainer") as demo:
|
|
| 259 |
inputs=[dataset_state, lora_name, instance_prompt, validation_prompt, rank, lora_alpha,
|
| 260 |
max_train_steps, learning_rate, lr_scheduler, resolution, repeats, train_batch_size,
|
| 261 |
gradient_accumulation_steps, seed, optimizer, use_8bit_adam, cache_latents,
|
| 262 |
-
gradient_checkpointing, offload, quantization, lora_layers,
|
| 263 |
-
hub_model_id, flavor, timeout],
|
| 264 |
outputs=[status, joblink, job_id],
|
| 265 |
)
|
| 266 |
refresh_btn.click(refresh, inputs=[job_id], outputs=[mon_status, mon_logs])
|
|
|
|
| 91 |
dataset_rows, lora_name, instance_prompt, validation_prompt, rank, lora_alpha, max_train_steps,
|
| 92 |
learning_rate, lr_scheduler, resolution, repeats, train_batch_size, gradient_accumulation_steps,
|
| 93 |
seed, optimizer, use_8bit_adam, cache_latents, gradient_checkpointing, offload, quantization,
|
| 94 |
+
lora_layers, run_validation, validation_epochs, hub_model_id, flavor, timeout,
|
| 95 |
profile: gr.OAuthProfile | None = None, oauth_token: gr.OAuthToken | None = None,
|
| 96 |
):
|
| 97 |
if oauth_token is None or profile is None:
|
|
|
|
| 113 |
"optimizer": optimizer, "use_8bit_adam": bool(use_8bit_adam),
|
| 114 |
"cache_latents": bool(cache_latents), "gradient_checkpointing": bool(gradient_checkpointing),
|
| 115 |
"offload": bool(offload), "quantization": quantization, "lora_layers": lora_layers,
|
| 116 |
+
"run_validation": bool(run_validation), "validation_epochs": validation_epochs,
|
| 117 |
+
"hf_token": oauth_token.token,
|
| 118 |
}
|
| 119 |
try:
|
| 120 |
res = jobs.submit(params, image_paths, captions, flavor=flavor, timeout=timeout)
|
|
|
|
| 214 |
train_batch_size = gr.Number(label="Batch size", value=1, precision=0)
|
| 215 |
gradient_accumulation_steps = gr.Number(label="Grad accumulation", value=1, precision=0)
|
| 216 |
with gr.Accordion("Validation", open=False):
|
| 217 |
+
run_validation = gr.Checkbox(
|
| 218 |
+
label="Run validation on Turbo", value=True,
|
| 219 |
+
info="Generate preview images on Krea 2 Turbo during/after training. "
|
| 220 |
+
"Turn off to train faster (and to skip it while the diffusers PR's "
|
| 221 |
+
"validation path is still being finalized upstream).",
|
| 222 |
+
)
|
| 223 |
validation_prompt = gr.Textbox(
|
| 224 |
label="Validation prompt", placeholder="(defaults to your trigger)",
|
| 225 |
info="Generated on Turbo every N epochs to preview progress.",
|
|
|
|
| 266 |
inputs=[dataset_state, lora_name, instance_prompt, validation_prompt, rank, lora_alpha,
|
| 267 |
max_train_steps, learning_rate, lr_scheduler, resolution, repeats, train_batch_size,
|
| 268 |
gradient_accumulation_steps, seed, optimizer, use_8bit_adam, cache_latents,
|
| 269 |
+
gradient_checkpointing, offload, quantization, lora_layers, run_validation,
|
| 270 |
+
validation_epochs, hub_model_id, flavor, timeout],
|
| 271 |
outputs=[status, joblink, job_id],
|
| 272 |
)
|
| 273 |
refresh_btn.click(refresh, inputs=[job_id], outputs=[mon_status, mon_logs])
|
jobs.py
CHANGED
|
@@ -94,12 +94,17 @@ def build_train_args(params: dict, hub_model_id: str) -> list[str]:
|
|
| 94 |
"--max_train_steps", str(int(params["max_train_steps"])),
|
| 95 |
"--optimizer", str(params["optimizer"]),
|
| 96 |
"--seed", str(int(params["seed"])),
|
| 97 |
-
"--validation_prompt", val_prompt,
|
| 98 |
-
"--validation_epochs", str(int(params["validation_epochs"])),
|
| 99 |
-
"--num_validation_images", "2",
|
| 100 |
"--push_to_hub",
|
| 101 |
"--hub_model_id", hub_model_id,
|
| 102 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
if params.get("lora_layers"):
|
| 104 |
args += ["--lora_layers", str(params["lora_layers"]).strip()]
|
| 105 |
if params.get("gradient_checkpointing", True):
|
|
|
|
| 94 |
"--max_train_steps", str(int(params["max_train_steps"])),
|
| 95 |
"--optimizer", str(params["optimizer"]),
|
| 96 |
"--seed", str(int(params["seed"])),
|
|
|
|
|
|
|
|
|
|
| 97 |
"--push_to_hub",
|
| 98 |
"--hub_model_id", hub_model_id,
|
| 99 |
]
|
| 100 |
+
if params.get("run_validation", True):
|
| 101 |
+
args += [
|
| 102 |
+
"--validation_prompt", val_prompt,
|
| 103 |
+
"--validation_epochs", str(int(params["validation_epochs"])),
|
| 104 |
+
"--num_validation_images", "2",
|
| 105 |
+
]
|
| 106 |
+
else:
|
| 107 |
+
args += ["--skip_final_inference"]
|
| 108 |
if params.get("lora_layers"):
|
| 109 |
args += ["--lora_layers", str(params["lora_layers"]).strip()]
|
| 110 |
if params.get("gradient_checkpointing", True):
|