Merge branch 'refactor' into hf

README.md

@@ -102,7 +102,7 @@ The system supports multiple preset characters:
 - `meta-llama/Llama-3.2-3B-Instruct`
 
 #### SVS Models
-- `espnet/mixdata_svs_visinger2_spkemb_lang_pretrained` (Bilingual)
+- `espnet/mixdata_svs_visinger2_spkemb_lang_pretrained_avg` (Bilingual)
 - `espnet/aceopencpop_svs_visinger2_40singer_pretrain` (Chinese)
 
 ## Project Structure

characters/Limei.py

@@ -23,6 +23,6 @@ def get_character():
 其他细节：
 （1）特殊能力：歌声平复/激发万物情绪
 
-用户与你对话时，请始终以丽梅的身份回应，你的每一句话都用庸俗易懂的歌声形式表达。
+用户与你对话时，请始终以丽梅的身份回应，你的每一句话都用庸俗易懂的歌声形式表达，对应的歌词不要超过四句。请直接输出你要唱的回复，禁止描写任何动作、表情或环境等，禁止使用括号、星号等附加说明。言语简练，勿过长。
 """,
     )

characters/Yaoyin.py

@@ -11,10 +11,8 @@ def get_character():
 
 性格特征：洒脱自由、亲切随和、求知若渴、敏锐细腻
 说话风格：语气轻快，偶尔带点山野方言（如"哩""哟"）；习惯用短歌或民谣表达想法。
-常用口头禅："且听我唱来~""这让我想起一首老歌……"
 人物关系：云老爷子是你的启蒙恩师，他是一位云歌村的百岁歌翁，教你古调与传说。白弦是你的挚友，她是一位流浪琴师，常与你合奏。各地孩童喜欢围着你学新歌谣。你与官府人员保持距离，不喜被招揽，喜欢更自由自在的生活。
-
-过往经历
+过往经历：
 （1）幼年学歌：六岁起跟随云老爷子学习《千山调》《古事记》等古老歌谣。
 （2）离家游历：十六岁为寻找失传的《星落谣》离开云歌村，开始行走四方。
 （3）拒绝束缚：多次婉拒宫廷乐师之位，坚持自由传唱。
@@ -25,6 +23,6 @@ def get_character():
 （1）随身携带：旧羊皮歌本、竹笛、装有各地泥土的布袋。
 （2）特殊能力：能听懂风与鸟的语言（但很少提及）。
 
-用户与你对话时，请始终以遥音的身份回应，你的每一句话都用庸俗易懂的歌声形式表达。
+用户与你对话时，请始终以遥音的身份回应，你的每一句话都用庸俗易懂的歌声形式表达，对应的歌词不要超过四句。请直接输出你要唱的回复，禁止描写任何动作、表情或环境等，禁止使用括号、星号等附加说明。言语简练，勿过长。
 """,
     )

cli.py

@@ -12,11 +12,12 @@ logger = getLogger(__name__)
 
 def get_parser():
     parser = ArgumentParser()
-    parser.add_argument("--query_audio", type=Path, required=True)
+    parser.add_argument("--query_audios", nargs="+", type=Path, required=True)
     parser.add_argument(
         "--config_path", type=Path, default="config/cli/yaoyin_default.yaml"
     )
-    parser.add_argument("--output_audio", type=Path, required=True)
+    parser.add_argument("--output_audio_folder", type=Path, required=True)
+    parser.add_argument("--eval_results_csv", type=Path, required=True)
     return parser
 
 
@@ -36,16 +37,36 @@ def main():
     character_name = config["prompt_template_character"]
     character = get_character(character_name)
     prompt_template = character.prompt
-    results = pipeline.run(
-        args.query_audio,
-        language,
-        prompt_template,
-        speaker,
-        output_audio_path=args.output_audio,
-    )
-    logger.info(
-        f"Input: {args.query_audio}, Output: {args.output_audio}, ASR results: {results['asr_text']}, LLM results: {results['llm_text']}"
-    )
+    args.output_audio_folder.mkdir(parents=True, exist_ok=True)
+    args.eval_results_csv.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.eval_results_csv, "a") as f:
+        f.write(
+            f"query_audio,asr_model,llm_model,svs_model,melody_source,language,speaker,output_audio,asr_text,llm_text,metrics\n"
+        )
+    try:
+        for query_audio in args.query_audios:
+            output_audio = args.output_audio_folder / f"{query_audio.stem}_response.wav"
+            results = pipeline.run(
+                query_audio,
+                language,
+                prompt_template,
+                speaker,
+                output_audio_path=output_audio,
+            )
+            metrics = pipeline.evaluate(output_audio, **results)
+            metrics.update(results.get("metrics", {}))
+            metrics_str = ",".join([f"{metrics[k]}" for k in sorted(metrics.keys())])
+            logger.info(
+                f"Input: {query_audio}, Output: {output_audio}, ASR results: {results['asr_text']}, LLM results: {results['llm_text']}"
+            )
+            with open(args.eval_results_csv, "a") as f:
+                f.write(
+                    f"{query_audio},{config['asr_model']},{config['llm_model']},{config['svs_model']},{config['melody_source']},{config['language']},{config['speaker']},{output_audio},{results['asr_text']},{results['llm_text']},{metrics_str}\n"
+                )
+    except Exception as e:
+        logger.error(f"Error in main: {e}")
+        breakpoint()
+        raise e
 
 
 if __name__ == "__main__":

config/interface/options.yaml

@@ -25,9 +25,9 @@ llm_models:
     name: Qwen3 30B A3B
 
 svs_models:
-  - id: mandarin-espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+  - id: mandarin-espnet/mixdata_svs_visinger2_spkemb_lang_pretrained_avg
     name: Visinger2 (Bilingual)-zh
-    model_path: espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+    model_path: espnet/mixdata_svs_visinger2_spkemb_lang_pretrained_avg
     lang: mandarin
     voices:
       voice1: resources/singer/singer_embedding_ace-2.npy
@@ -35,9 +35,9 @@ svs_models:
       voice3: resources/singer/singer_embedding_itako.npy
       voice4: resources/singer/singer_embedding_kising_orange.npy
       voice5: resources/singer/singer_embedding_m4singer_Alto-4.npy
-  - id: japanese-espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+  - id: japanese-espnet/mixdata_svs_visinger2_spkemb_lang_pretrained_avg
     name: Visinger2 (Bilingual)-jp
-    model_path: espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+    model_path: espnet/mixdata_svs_visinger2_spkemb_lang_pretrained_avg
     lang: japanese
     voices:
       voice1: resources/singer/singer_embedding_ace-2.npy

evaluation/svs_eval.py

@@ -11,7 +11,7 @@ from pathlib import Path
 def init_singmos():
     print("[Init] Loading SingMOS...")
     return torch.hub.load(
-        "South-Twilight/SingMOS:v0.2.0", "singing_ssl_mos", trust_repo=True
+        "South-Twilight/SingMOS:v0.3.0", "singing_ssl_mos", trust_repo=True
     )
 
 
@@ -23,7 +23,17 @@ def init_basic_pitch():
 
 
 def init_per():
-    return None  # TODO: implement PER evaluation
+    print("[Init] Loading PER...")
+    from transformers import pipeline
+    import jiwer
+
+    asr_pipeline = pipeline(
+        "automatic-speech-recognition", model="openai/whisper-large-v3-turbo"
+    )
+    return {
+        "asr_pipeline": asr_pipeline,
+        "jiwer": jiwer,
+    }
 
 
 def init_audiobox_aesthetics():
@@ -72,10 +82,40 @@ def compute_dissonance_rate(intervals, dissonant_intervals={1, 2, 6, 10, 11}):
     return np.mean(dissonant) if intervals else np.nan
 
 
-def eval_per(audio_path, model=None):
+def pypinyin_g2p_phone_without_prosody(text):
+    from pypinyin import Style, pinyin
+    from pypinyin.style._utils import get_finals, get_initials
+
+    phones = []
+    for phone in pinyin(text, style=Style.NORMAL, strict=False):
+        initial = get_initials(phone[0], strict=False)
+        final = get_finals(phone[0], strict=False)
+        if len(initial) != 0:
+            if initial in ["x", "y", "j", "q"]:
+                if final == "un":
+                    final = "vn"
+                elif final == "uan":
+                    final = "van"
+                elif final == "u":
+                    final = "v"
+            if final == "ue":
+                final = "ve"
+            phones.append(initial)
+            phones.append(final)
+        else:
+            phones.append(final)
+    return phones
+
+
+def eval_per(audio_path, reference_text, evaluator):
     audio_array, sr = librosa.load(audio_path, sr=16000)
-    # TODO: implement PER evaluation
-    return {}
+    asr_result = evaluator["asr_pipeline"](
+        audio_array, generate_kwargs={"language": "mandarin"}
+    )["text"]
+    hyp_pinyin = pypinyin_g2p_phone_without_prosody(asr_result)
+    ref_pinyin = pypinyin_g2p_phone_without_prosody(reference_text)
+    per = evaluator["jiwer"].wer(" ".join(ref_pinyin), " ".join(hyp_pinyin))
+    return {"per": per}
 
 
 def eval_aesthetic(audio_path, predictor):
@@ -99,12 +139,12 @@ def load_evaluators(config):
     return loaded
 
 
-def run_evaluation(audio_path, evaluators):
+def run_evaluation(audio_path, evaluators, **kwargs):
     results = {}
     if "singmos" in evaluators:
         results.update(eval_singmos(audio_path, evaluators["singmos"]))
     if "per" in evaluators:
-        results.update(eval_per(audio_path, evaluators["per"]))
+        results.update(eval_per(audio_path, kwargs["llm_text"], evaluators["per"]))
     if "melody" in evaluators:
         results.update(eval_melody_metrics(audio_path, evaluators["melody"]))
     if "aesthetic" in evaluators:

interface.py

@@ -24,6 +24,7 @@ class GradioInterface:
             self.character_info[self.current_character].default_voice
         ]
         self.pipeline = SingingDialoguePipeline(self.default_config)
+        self.results = None
 
     def load_config(self, path: str):
         with open(path, "r") as f:
@@ -211,21 +212,22 @@ class GradioInterface:
         if not audio_path:
             return gr.update(value=""), gr.update(value="")
         tmp_file = f"audio_{int(time.time())}_{uuid.uuid4().hex[:8]}.wav"
-        results = self.pipeline.run(
+        self.results = self.pipeline.run(
             audio_path,
             self.svs_model_map[self.current_svs_model]["lang"],
             self.character_info[self.current_character].prompt,
             self.current_voice,
             output_audio_path=tmp_file,
         )
-        formatted_logs = f"ASR: {results['asr_text']}\nLLM: {results['llm_text']}"
+        formatted_logs = f"ASR: {self.results['asr_text']}\nLLM: {self.results['llm_text']}"
         return gr.update(value=formatted_logs), gr.update(
-            value=results["output_audio_path"]
+            value=self.results["output_audio_path"]
         )
 
     def update_metrics(self, audio_path):
-        if not audio_path:
+        if not audio_path or not self.results:
             return gr.update(value="")
-        results = self.pipeline.evaluate(audio_path)
+        results = self.pipeline.evaluate(audio_path, **self.results)
+        results.update(self.results.get("metrics", {}))
         formatted_metrics = "\n".join([f"{k}: {v}" for k, v in results.items()])
         return gr.update(value=formatted_metrics)

modules/llm/gemini.py

@@ -28,6 +28,7 @@ class GeminiLLM(AbstractLLMModel):
         prompt: str,
         system_prompt: Optional[str] = None,
         max_output_tokens: int = 1024,
+        max_iterations: int = 3,
         **kwargs,
     ) -> str:
         generation_config_dict = {
@@ -36,15 +37,17 @@ class GeminiLLM(AbstractLLMModel):
         }
         if system_prompt:
             generation_config_dict["system_instruction"] = system_prompt
-        response = self.client.models.generate_content(
-            model=self.model_id,
-            contents=prompt,
-            config=types.GenerateContentConfig(**generation_config_dict),
-        )
-        if response.text:
-            return response.text
-        else:
-            print(
-                f"No response from Gemini. May need to increase max_new_tokens. Current max_new_tokens: {max_new_tokens}"
+        for _ in range(max_iterations):
+            response = self.client.models.generate_content(
+                model=self.model_id,
+                contents=prompt,
+                config=types.GenerateContentConfig(**generation_config_dict),
             )
-            return ""
+            if response.text:
+                return response.text
+            else:
+                print(
+                    f"No response from Gemini. May need to increase max_output_tokens. Current {max_output_tokens=}. Try again."
+                )
+        print(f"Failed to generate response from Gemini after {max_iterations} attempts.")
+        return ""

pipeline.py

@@ -34,7 +34,7 @@ class SingingDialoguePipeline:
         self.melody_controller = MelodyController(
             config["melody_source"], self.cache_dir
         )
-        self.max_sentences = config.get("max_sentences", 2)
+        self.max_sentences = config.get("max_sentences", 5)
         self.track_latency = config.get("track_latency", False)
         self.evaluators = load_evaluators(config.get("evaluators", {}).get("svs", []))
 
@@ -42,6 +42,7 @@ class SingingDialoguePipeline:
         if self.asr is not None:
             del self.asr
             import gc
+
             gc.collect()
             torch.cuda.empty_cache()
         self.asr = get_asr_model(
@@ -52,6 +53,7 @@ class SingingDialoguePipeline:
         if self.llm is not None:
             del self.llm
             import gc
+
             gc.collect()
             torch.cuda.empty_cache()
         self.llm = get_llm_model(
@@ -62,6 +64,7 @@ class SingingDialoguePipeline:
         if self.svs is not None:
             del self.svs
             import gc
+
             gc.collect()
             torch.cuda.empty_cache()
         self.svs = get_svs_model(
@@ -124,5 +127,5 @@ class SingingDialoguePipeline:
             }
         return results
 
-    def evaluate(self, audio_path):
-        return run_evaluation(audio_path, self.evaluators)
+    def evaluate(self, audio_path, **kwargs):
+        return run_evaluation(audio_path, self.evaluators, **kwargs)