add 8bit and fix some value

src/f5_tts/train/finetune_cli.py

@@ -13,6 +13,9 @@ from importlib.resources import files
 target_sample_rate = 24000
 n_mel_channels = 100
 hop_length = 256
+win_length = 1024
+n_fft = 1024
+mel_spec_type = "vocos"  # 'vocos' or 'bigvgan'
 
 
 # -------------------------- Argument Parsing --------------------------- #
@@ -40,7 +43,7 @@ def parse_args():
     parser.add_argument("--max_samples", type=int, default=64, help="Max sequences per batch")
     parser.add_argument("--grad_accumulation_steps", type=int, default=1, help="Gradient accumulation steps")
     parser.add_argument("--max_grad_norm", type=float, default=1.0, help="Max gradient norm for clipping")
-    parser.add_argument("--epochs", type=int, default=10, help="Number of training epochs")
+    parser.add_argument("--epochs", type=int, default=100, help="Number of training epochs")
     parser.add_argument("--num_warmup_updates", type=int, default=300, help="Warmup steps")
     parser.add_argument("--save_per_updates", type=int, default=10000, help="Save checkpoint every X steps")
     parser.add_argument("--last_per_steps", type=int, default=50000, help="Save last checkpoint every X steps")
@@ -121,11 +124,15 @@ def main():
     vocab_char_map, vocab_size = get_tokenizer(tokenizer_path, tokenizer)
 
     print("\nvocab : ", vocab_size)
+    print("\nvocoder : ", mel_spec_type)
 
     mel_spec_kwargs = dict(
-        target_sample_rate=target_sample_rate,
-        n_mel_channels=n_mel_channels,
+        n_fft=n_fft,
         hop_length=hop_length,
+        win_length=win_length,
+        n_mel_channels=n_mel_channels,
+        target_sample_rate=target_sample_rate,
+        mel_spec_type=mel_spec_type,
     )
 
     model = CFM(

src/f5_tts/train/finetune_gradio.py

@@ -43,6 +43,13 @@ last_ema = None
 
 path_data = str(files("f5_tts").joinpath("../../data"))
 path_project_ckpts = str(files("f5_tts").joinpath("../../ckpts"))
+
+from pathlib import Path
+
+base_path = Path(__file__).resolve().parent.parent.parent.parent
+path_data = str(base_path / "data")
+path_project_ckpts = str(base_path / "ckpts")
+
 file_train = "src/f5_tts/train/finetune_cli.py"
 
 device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
@@ -70,6 +77,7 @@ def save_settings(
     tokenizer_file,
     mixed_precision,
     logger,
+    ch_8bit_adam,
 ):
     path_project = os.path.join(path_project_ckpts, project_name)
     os.makedirs(path_project, exist_ok=True)
@@ -93,6 +101,7 @@ def save_settings(
         "tokenizer_file": tokenizer_file,
         "mixed_precision": mixed_precision,
         "logger": logger,
+        "bnb_optimizer": ch_8bit_adam,
     }
     with open(file_setting, "w") as f:
         json.dump(settings, f, indent=4)
@@ -124,6 +133,7 @@ def load_settings(project_name):
             "tokenizer_file": "",
             "mixed_precision": "none",
             "logger": "wandb",
+            "bnb_optimizer": False,
         }
         return (
             settings["exp_name"],
@@ -143,12 +153,15 @@ def load_settings(project_name):
             settings["tokenizer_file"],
             settings["mixed_precision"],
             settings["logger"],
+            settings["bnb_optimizer"],
         )
 
     with open(file_setting, "r") as f:
         settings = json.load(f)
         if "logger" not in settings:
             settings["logger"] = "wandb"
+        if "bnb_optimizer" not in settings:
+            settings["bnb_optimizer"] = False
     return (
         settings["exp_name"],
         settings["learning_rate"],
@@ -167,6 +180,7 @@ def load_settings(project_name):
         settings["tokenizer_file"],
         settings["mixed_precision"],
         settings["logger"],
+        settings["bnb_optimizer"],
     )
 
 
@@ -381,6 +395,7 @@ def start_training(
     mixed_precision="fp16",
     stream=False,
     logger="wandb",
+    ch_8bit_adam=False,
 ):
     global training_process, tts_api, stop_signal, pipe
 
@@ -447,11 +462,10 @@ def start_training(
         f"--dataset_name {dataset_name}"
     )
 
-    if finetune:
-        cmd += f" --finetune {finetune}"
+    cmd += f" --finetune {finetune}"
 
     if file_checkpoint_train != "":
-        cmd += f" --file_checkpoint_train {file_checkpoint_train}"
+        cmd += f" --pretrain {file_checkpoint_train}"
 
     if tokenizer_file != "":
         cmd += f" --tokenizer_path {tokenizer_file}"
@@ -460,7 +474,10 @@ def start_training(
 
     cmd += f" --log_samples True --logger {logger} "
 
-    print(cmd)
+    if ch_8bit_adam:
+        cmd += " --bnb_optimizer True "
+
+    print("run command : \n" + cmd + "\n")
 
     save_settings(
         dataset_name,
@@ -481,6 +498,7 @@ def start_training(
         tokenizer_file,
         mixed_precision,
         logger,
+        ch_8bit_adam,
     )
 
     try:
@@ -758,11 +776,9 @@ def get_correct_audio_path(
     # Case 2: If it has a supported extension but is not a full path
     elif has_supported_extension(audio_input) and not os.path.isabs(audio_input):
         file_audio = os.path.join(base_path, audio_input)
-        print("2")
 
     # Case 3: If only the name is given (no extension and not a full path)
     elif not has_supported_extension(audio_input) and not os.path.isabs(audio_input):
-        print("3")
         for ext in supported_formats:
             potential_file = os.path.join(base_path, f"{audio_input}.{ext}")
             if os.path.exists(potential_file):
@@ -773,6 +789,18 @@ def get_correct_audio_path(
     return file_audio
 
 
+def get_nested_value(data, format):
+    keys = format.split("/")
+
+    item = data
+    for key in keys:
+        item = item.get(key)
+        if item is None:
+            return None
+
+    return item
+
+
 def create_metadata(name_project, ch_tokenizer, progress=gr.Progress()):
     path_project = os.path.join(path_data, name_project)
     path_project_wavs = os.path.join(path_project, "wavs")
@@ -816,9 +844,12 @@ def create_metadata(name_project, ch_tokenizer, progress=gr.Progress()):
             continue
 
         if duration < 1 or duration > 25:
-            error_files.append([file_audio, "duration < 1 or > 25 "])
+            if duration > 25:
+                error_files.append([file_audio, "duration > 25 sec"])
+            if duration < 1:
+                error_files.append([file_audio, "duration < 1 sec "])
             continue
-        if len(text) < 4:
+        if len(text) < 3:
             error_files.append([file_audio, "very small text len 3"])
             continue
 
@@ -1208,7 +1239,9 @@ def get_random_sample_infer(project_name):
     )
 
 
-def infer(project, file_checkpoint, exp_name, ref_text, ref_audio, gen_text, nfe_step, use_ema):
+def infer(
+    project, file_checkpoint, exp_name, ref_text, ref_audio, gen_text, nfe_step, use_ema, speed, seed, remove_silence
+):
     global last_checkpoint, last_device, tts_api, last_ema
 
     if not os.path.isfile(file_checkpoint):
@@ -1238,8 +1271,17 @@ def infer(project, file_checkpoint, exp_name, ref_text, ref_audio, gen_text, nfe
         print("update >> ", device_test, file_checkpoint, use_ema)
 
     with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f:
-        tts_api.infer(gen_text=gen_text, ref_text=ref_text, ref_file=ref_audio, nfe_step=nfe_step, file_wave=f.name)
-        return f.name, tts_api.device
+        tts_api.infer(
+            gen_text=gen_text.lower().strip(),
+            ref_text=ref_text.lower().strip(),
+            ref_file=ref_audio,
+            nfe_step=nfe_step,
+            file_wave=f.name,
+            speed=speed,
+            seed=seed,
+            remove_silence=remove_silence,
+        )
+        return f.name, tts_api.device, str(tts_api.seed)
 
 
 def check_finetune(finetune):
@@ -1506,6 +1548,7 @@ Skip this step if you have your dataset, raw.arrow, duration.json, and vocab.txt
      ```"""
             )
             ch_tokenizern = gr.Checkbox(label="Create Vocabulary", value=False, visible=False)
+
             bt_prepare = bt_create = gr.Button("Prepare")
             txt_info_prepare = gr.Text(label="Info", value="")
             txt_vocab_prepare = gr.Text(label="Vocab", value="")
@@ -1560,6 +1603,7 @@ If you encounter a memory error, try reducing the batch size per GPU to a smalle
                 last_per_steps = gr.Number(label="Last per Steps", value=100)
 
             with gr.Row():
+                ch_8bit_adam = gr.Checkbox(label="Use 8-bit Adam optimizer")
                 mixed_precision = gr.Radio(label="mixed_precision", choices=["none", "fp16", "bf16"], value="none")
                 cd_logger = gr.Radio(label="logger", choices=["wandb", "tensorboard"], value="wandb")
                 start_button = gr.Button("Start Training")
@@ -1584,6 +1628,7 @@ If you encounter a memory error, try reducing the batch size per GPU to a smalle
                     tokenizer_filev,
                     mixed_precisionv,
                     cd_loggerv,
+                    ch_8bit_adamv,
                 ) = load_settings(projects_selelect)
                 exp_name.value = exp_namev
                 learning_rate.value = learning_ratev
@@ -1602,6 +1647,7 @@ If you encounter a memory error, try reducing the batch size per GPU to a smalle
                 tokenizer_file.value = tokenizer_filev
                 mixed_precision.value = mixed_precisionv
                 cd_logger.value = cd_loggerv
+                ch_8bit_adam.value = ch_8bit_adamv
 
             ch_stream = gr.Checkbox(label="Stream Output Experiment", value=True)
             txt_info_train = gr.Text(label="Info", value="")
@@ -1660,6 +1706,7 @@ If you encounter a memory error, try reducing the batch size per GPU to a smalle
                     mixed_precision,
                     ch_stream,
                     cd_logger,
+                    ch_8bit_adam,
                 ],
                 outputs=[txt_info_train, start_button, stop_button],
             )
@@ -1732,12 +1779,17 @@ If you encounter a memory error, try reducing the batch size per GPU to a smalle
 
         with gr.TabItem("Test Model"):
             gr.Markdown("""```plaintext 
-SOS: Check the use_ema setting (True or False) for your model to see what works best for you. 
+SOS: Check the use_ema setting (True or False) for your model to see what works best for you. use seed -1 from random
 ```""")
             exp_name = gr.Radio(label="Model", choices=["F5-TTS", "E2-TTS"], value="F5-TTS")
             list_checkpoints, checkpoint_select = get_checkpoints_project(projects_selelect, False)
 
-            nfe_step = gr.Number(label="NFE Step", value=32)
+            with gr.Row():
+                nfe_step = gr.Number(label="NFE Step", value=32)
+                speed = gr.Slider(label="Speed", value=1.0, minimum=0.3, maximum=2.0, step=0.1)
+                seed = gr.Number(label="Seed", value=-1, minimum=-1)
+                remove_silence = gr.Checkbox(label="Remove Silence")
+
             ch_use_ema = gr.Checkbox(label="Use EMA", value=True)
             with gr.Row():
                 cm_checkpoint = gr.Dropdown(
@@ -1757,14 +1809,27 @@ SOS: Check the use_ema setting (True or False) for your model to see what works
 
             with gr.Row():
                 txt_info_gpu = gr.Textbox("", label="Device")
+                seed_info = gr.Text(label="Seed :")
                 check_button_infer = gr.Button("Infer")
 
             gen_audio = gr.Audio(label="Audio Gen", type="filepath")
 
             check_button_infer.click(
                 fn=infer,
-                inputs=[cm_project, cm_checkpoint, exp_name, ref_text, ref_audio, gen_text, nfe_step, ch_use_ema],
-                outputs=[gen_audio, txt_info_gpu],
+                inputs=[
+                    cm_project,
+                    cm_checkpoint,
+                    exp_name,
+                    ref_text,
+                    ref_audio,
+                    gen_text,
+                    nfe_step,
+                    ch_use_ema,
+                    speed,
+                    seed,
+                    remove_silence,
+                ],
+                outputs=[gen_audio, txt_info_gpu, seed_info],
             )
 
             bt_checkpoint_refresh.click(fn=get_checkpoints_project, inputs=[cm_project], outputs=[cm_checkpoint])