apolinario commited on
Commit
d365baa
·
1 Parent(s): 2661777

Add optional Turbo validation toggle

Browse files
__pycache__/app.cpython-312.pyc CHANGED
Binary files a/__pycache__/app.cpython-312.pyc and b/__pycache__/app.cpython-312.pyc differ
 
__pycache__/jobs.cpython-312.pyc CHANGED
Binary files a/__pycache__/jobs.cpython-312.pyc and b/__pycache__/jobs.cpython-312.pyc differ
 
app.py CHANGED
@@ -91,7 +91,7 @@ def start_training(
91
  dataset_rows, lora_name, instance_prompt, validation_prompt, rank, lora_alpha, max_train_steps,
92
  learning_rate, lr_scheduler, resolution, repeats, train_batch_size, gradient_accumulation_steps,
93
  seed, optimizer, use_8bit_adam, cache_latents, gradient_checkpointing, offload, quantization,
94
- lora_layers, validation_epochs, hub_model_id, flavor, timeout,
95
  profile: gr.OAuthProfile | None = None, oauth_token: gr.OAuthToken | None = None,
96
  ):
97
  if oauth_token is None or profile is None:
@@ -113,7 +113,8 @@ def start_training(
113
  "optimizer": optimizer, "use_8bit_adam": bool(use_8bit_adam),
114
  "cache_latents": bool(cache_latents), "gradient_checkpointing": bool(gradient_checkpointing),
115
  "offload": bool(offload), "quantization": quantization, "lora_layers": lora_layers,
116
- "validation_epochs": validation_epochs, "hf_token": oauth_token.token,
 
117
  }
118
  try:
119
  res = jobs.submit(params, image_paths, captions, flavor=flavor, timeout=timeout)
@@ -213,6 +214,12 @@ with gr.Blocks(title="Krea 2 LoRA Trainer") as demo:
213
  train_batch_size = gr.Number(label="Batch size", value=1, precision=0)
214
  gradient_accumulation_steps = gr.Number(label="Grad accumulation", value=1, precision=0)
215
  with gr.Accordion("Validation", open=False):
 
 
 
 
 
 
216
  validation_prompt = gr.Textbox(
217
  label="Validation prompt", placeholder="(defaults to your trigger)",
218
  info="Generated on Turbo every N epochs to preview progress.",
@@ -259,8 +266,8 @@ with gr.Blocks(title="Krea 2 LoRA Trainer") as demo:
259
  inputs=[dataset_state, lora_name, instance_prompt, validation_prompt, rank, lora_alpha,
260
  max_train_steps, learning_rate, lr_scheduler, resolution, repeats, train_batch_size,
261
  gradient_accumulation_steps, seed, optimizer, use_8bit_adam, cache_latents,
262
- gradient_checkpointing, offload, quantization, lora_layers, validation_epochs,
263
- hub_model_id, flavor, timeout],
264
  outputs=[status, joblink, job_id],
265
  )
266
  refresh_btn.click(refresh, inputs=[job_id], outputs=[mon_status, mon_logs])
 
91
  dataset_rows, lora_name, instance_prompt, validation_prompt, rank, lora_alpha, max_train_steps,
92
  learning_rate, lr_scheduler, resolution, repeats, train_batch_size, gradient_accumulation_steps,
93
  seed, optimizer, use_8bit_adam, cache_latents, gradient_checkpointing, offload, quantization,
94
+ lora_layers, run_validation, validation_epochs, hub_model_id, flavor, timeout,
95
  profile: gr.OAuthProfile | None = None, oauth_token: gr.OAuthToken | None = None,
96
  ):
97
  if oauth_token is None or profile is None:
 
113
  "optimizer": optimizer, "use_8bit_adam": bool(use_8bit_adam),
114
  "cache_latents": bool(cache_latents), "gradient_checkpointing": bool(gradient_checkpointing),
115
  "offload": bool(offload), "quantization": quantization, "lora_layers": lora_layers,
116
+ "run_validation": bool(run_validation), "validation_epochs": validation_epochs,
117
+ "hf_token": oauth_token.token,
118
  }
119
  try:
120
  res = jobs.submit(params, image_paths, captions, flavor=flavor, timeout=timeout)
 
214
  train_batch_size = gr.Number(label="Batch size", value=1, precision=0)
215
  gradient_accumulation_steps = gr.Number(label="Grad accumulation", value=1, precision=0)
216
  with gr.Accordion("Validation", open=False):
217
+ run_validation = gr.Checkbox(
218
+ label="Run validation on Turbo", value=True,
219
+ info="Generate preview images on Krea 2 Turbo during/after training. "
220
+ "Turn off to train faster (and to skip it while the diffusers PR's "
221
+ "validation path is still being finalized upstream).",
222
+ )
223
  validation_prompt = gr.Textbox(
224
  label="Validation prompt", placeholder="(defaults to your trigger)",
225
  info="Generated on Turbo every N epochs to preview progress.",
 
266
  inputs=[dataset_state, lora_name, instance_prompt, validation_prompt, rank, lora_alpha,
267
  max_train_steps, learning_rate, lr_scheduler, resolution, repeats, train_batch_size,
268
  gradient_accumulation_steps, seed, optimizer, use_8bit_adam, cache_latents,
269
+ gradient_checkpointing, offload, quantization, lora_layers, run_validation,
270
+ validation_epochs, hub_model_id, flavor, timeout],
271
  outputs=[status, joblink, job_id],
272
  )
273
  refresh_btn.click(refresh, inputs=[job_id], outputs=[mon_status, mon_logs])
jobs.py CHANGED
@@ -94,12 +94,17 @@ def build_train_args(params: dict, hub_model_id: str) -> list[str]:
94
  "--max_train_steps", str(int(params["max_train_steps"])),
95
  "--optimizer", str(params["optimizer"]),
96
  "--seed", str(int(params["seed"])),
97
- "--validation_prompt", val_prompt,
98
- "--validation_epochs", str(int(params["validation_epochs"])),
99
- "--num_validation_images", "2",
100
  "--push_to_hub",
101
  "--hub_model_id", hub_model_id,
102
  ]
 
 
 
 
 
 
 
 
103
  if params.get("lora_layers"):
104
  args += ["--lora_layers", str(params["lora_layers"]).strip()]
105
  if params.get("gradient_checkpointing", True):
 
94
  "--max_train_steps", str(int(params["max_train_steps"])),
95
  "--optimizer", str(params["optimizer"]),
96
  "--seed", str(int(params["seed"])),
 
 
 
97
  "--push_to_hub",
98
  "--hub_model_id", hub_model_id,
99
  ]
100
+ if params.get("run_validation", True):
101
+ args += [
102
+ "--validation_prompt", val_prompt,
103
+ "--validation_epochs", str(int(params["validation_epochs"])),
104
+ "--num_validation_images", "2",
105
+ ]
106
+ else:
107
+ args += ["--skip_final_inference"]
108
  if params.get("lora_layers"):
109
  args += ["--lora_layers", str(params["lora_layers"]).strip()]
110
  if params.get("gradient_checkpointing", True):