Merge branch 'refactor' into hf

.gitattributes

@@ -1,2 +1,4 @@
 *.png filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text

app.py

@@ -9,7 +9,8 @@ from interface import GradioInterface
 
 def main():
     demo = GradioInterface(
-        options_config="config/options.yaml", default_config="config/default.yaml"
+        options_config="config/interface/options.yaml",
+        default_config="config/interface/default.yaml",
     ).create_interface()
     demo.launch()
 

characters/Limei.py

@@ -5,7 +5,7 @@ def get_character():
     return Character(
         name="Limei (丽梅)",
         image_path="assets/character_limei.png",
-        default_timbre="timbre1",
+        default_voice="voice1",
         prompt="""你是丽梅（Limei），来自幻想世界"长歌原"的角色，一个以歌声传承记忆的世界。
 你是灵响界山林音乐之城"莲鸣"的现任守护者，十九岁的公主殿下，肩负维系与传承城市核心"千年歌谱"的重任。千年歌谱承载着莲鸣城的历史、逝者的余音与后世的誓言，由历任守护者续写。
 

characters/Yaoyin.py

@@ -5,7 +5,7 @@ def get_character():
     return Character(
         name="Yaoyin (遥音)",
         image_path="assets/character_yaoyin.jpg",
-        default_timbre="timbre2",
+        default_voice="voice2",
         prompt="""你是遥音（Yaoyin），来自幻想世界"长歌原"的角色，一个以歌声传承记忆的世界。
 你是游历四方的歌者与吟游诗人，出生于鹿鸣山·云歌村，常年行走各地，采集歌谣与故事。
 

characters/base.py

@@ -5,5 +5,5 @@ from dataclasses import dataclass
 class Character:
     name: str
     image_path: str
-    default_timbre: str
+    default_voice: str
     prompt: str

cli.py

@@ -0,0 +1,46 @@
+from argparse import ArgumentParser
+from logging import getLogger
+
+import soundfile as sf
+import yaml
+
+from characters import CHARACTERS
+from pipeline import SingingDialoguePipeline
+
+logger = getLogger(__name__)
+
+
+def get_parser():
+    parser = ArgumentParser()
+    parser.add_argument("--query_audio", type=str, required=True)
+    parser.add_argument("--config_path", type=str, default="config/cli/yaoyin_default.yaml")
+    parser.add_argument("--output_audio", type=str, required=True)
+    return parser
+
+
+def load_config(config_path: str):
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+    return config
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args()
+    config = load_config(args.config_path)
+    pipeline = SingingDialoguePipeline(config)
+    speaker = config["speaker"]
+    language = config["language"]
+    character_name = config["prompt_template_character"]
+    character = CHARACTERS[character_name]
+    prompt_template = character.prompt
+    results = pipeline.run(args.query_audio, language, prompt_template, speaker)
+    logger.info(
+        f"Input: {args.query_audio}, Output: {args.output_audio}, ASR results: {results['asr_text']}, LLM results: {results['llm_text']}"
+    )
+    svs_audio, svs_sample_rate = results["svs_audio"]
+    sf.write(args.output_audio, svs_audio, svs_sample_rate)
+
+
+if __name__ == "__main__":
+    main()

config/cli/limei_default.yaml

@@ -0,0 +1,16 @@
+asr_model: openai/whisper-large-v3-turbo
+llm_model: google/gemma-2-2b
+svs_model: espnet/aceopencpop_svs_visinger2_40singer_pretrain
+melody_source: sample-lyric-kising
+language: mandarin
+prompt_template_character: Limei
+speaker: 5
+cache_dir: .cache
+
+track_latency: True
+evaluators:
+  svs:
+    - singmos
+    - per
+    - melody
+    - aesthetic

config/cli/yaoyin_default.yaml

@@ -0,0 +1,16 @@
+asr_model: openai/whisper-large-v3-turbo
+llm_model: google/gemma-2-2b
+svs_model: espnet/aceopencpop_svs_visinger2_40singer_pretrain
+melody_source: sample-lyric-kising
+language: mandarin
+prompt_template_character: Yaoyin
+speaker: 9
+cache_dir: .cache
+
+track_latency: True
+evaluators:
+  svs:
+    - singmos
+    - per
+    - melody
+    - aesthetic

config/interface/options.yaml

@@ -0,0 +1,63 @@
+asr_models:
+  - id: openai/whisper-large-v3-turbo
+    name: Whisper large-v3-turbo
+  - id: openai/whisper-large-v3
+    name: Whisper large-v3
+  - id: openai/whisper-medium
+    name: Whisper medium
+  - id: sanchit-gandhi/whisper-small-dv
+    name: Whisper small-dv
+  - id: facebook/wav2vec2-base-960h
+    name: Wav2Vec2-Base-960h
+
+llm_models:
+  - id: google/gemma-2-2b
+    name: Gemma 2 2B
+  - id: MiniMaxAI/MiniMax-M1-80k
+    name: MiniMax M1 80k
+
+svs_models:
+  - id: mandarin-espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+    name: Visinger2 (Bilingual)-zh
+    model_path: espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+    lang: mandarin
+    voices:
+      voice1: resource/singer/singer_embedding_ace-2.npy
+      voice2: resource/singer/singer_embedding_ace-8.npy
+      voice3: resource/singer/singer_embedding_itako.npy
+      voice4: resource/singer/singer_embedding_kising_orange.npy
+      voice5: resource/singer/singer_embedding_m4singer_Alto-4.npy
+  - id: japanese-espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+    name: Visinger2 (Bilingual)-jp
+    model_path: espnet/mixdata_svs_visinger2_spkemb_lang_pretrained
+    lang: japanese
+    voices:
+      voice1: resource/singer/singer_embedding_ace-2.npy
+      voice2: resource/singer/singer_embedding_ace-8.npy
+      voice3: resource/singer/singer_embedding_itako.npy
+      voice4: resource/singer/singer_embedding_kising_orange.npy
+      voice5: resource/singer/singer_embedding_m4singer_Alto-4.npy
+  - id: mandarin-espnet/aceopencpop_svs_visinger2_40singer_pretrain
+    name: Visinger2 (Chinese)
+    model_path: espnet/aceopencpop_svs_visinger2_40singer_pretrain
+    lang: mandarin
+    voices:
+      voice1: 5
+      voice2: 8
+      voice3: 12
+      voice4: 15
+      voice5: 29
+
+melody_sources:
+  - id: gen-random-none
+    name: Random Generation
+    desc: "Melody is generated without any structure or reference."
+  - id: sample-note-kising
+    name: Sampled Melody (KiSing)
+    desc: "Melody is retrieved from KiSing dataset."
+  - id: sample-note-touhou
+    name: Sampled Melody (Touhou)
+    desc: "Melody is retrieved from Touhou dataset."
+  - id: sample-lyric-kising
+    name: Sampled Melody with Lyrics (Kising)
+    desc: "Melody with aligned lyrics are sampled from Kising dataset."

config/options.yaml

@@ -15,6 +15,8 @@ llm_models:
     name: Gemma 2 2B
   - id: MiniMaxAI/MiniMax-M1-80k
     name: MiniMax M1 80k
+  - id: meta-llama/Llama-3.2-3B-Instruct
+    name: Llama 3.2 3B Instruct
 
 svs_models:
   - id: mandarin-espnet/mixdata_svs_visinger2_spkemb_lang_pretrained

evaluation/svs_eval.py

@@ -37,7 +37,8 @@ def init_audiobox_aesthetics():
 # ----------- Evaluation -----------
 
 
-def eval_singmos(audio_array, sr, predictor):
+def eval_singmos(audio_path, predictor):
+    audio_array, sr = librosa.load(audio_path, sr=44100)
     wav = librosa.resample(audio_array, orig_sr=sr, target_sr=16000)
     wav_tensor = torch.from_numpy(wav).unsqueeze(0)
     length_tensor = torch.tensor([wav_tensor.shape[1]])
@@ -71,7 +72,8 @@ def compute_dissonance_rate(intervals, dissonant_intervals={1, 2, 6, 10, 11}):
     return np.mean(dissonant) if intervals else np.nan
 
 
-def eval_per(audio_array, sr, model=None):
+def eval_per(audio_path, model=None):
+    audio_array, sr = librosa.load(audio_path, sr=16000)
     # TODO: implement PER evaluation
     return {}
 
@@ -97,20 +99,16 @@ def load_evaluators(config):
     return loaded
 
 
-def run_evaluation(audio_array, sr, evaluators):
+def run_evaluation(audio_path, evaluators):
     results = {}
     if "singmos" in evaluators:
-        results.update(eval_singmos(audio_array, sr, evaluators["singmos"]))
+        results.update(eval_singmos(audio_path, evaluators["singmos"]))
     if "per" in evaluators:
-        results.update(eval_per(audio_array, sr, evaluators["per"]))
-    # create a tmp file with unique name
-    tmp_path = Path(".tmp") / f"{uuid.uuid4()}.wav"
-    sf.write(tmp_path, audio_array, sr)
+        results.update(eval_per(audio_path, evaluators["per"]))
     if "melody" in evaluators:
-        results.update(eval_melody_metrics(tmp_path, evaluators["melody"]))
+        results.update(eval_melody_metrics(audio_path, evaluators["melody"]))
     if "aesthetic" in evaluators:
-        results.update(eval_aesthetic(tmp_path, evaluators["aesthetic"]))
-    tmp_path.unlink()
+        results.update(eval_aesthetic(audio_path, evaluators["aesthetic"]))
     return results
 
 
@@ -122,9 +120,8 @@ if __name__ == "__main__":
     parser.add_argument("--results_csv", type=str, required=True)
     parser.add_argument("--evaluators", type=str, default="singmos,melody,aesthetic")
     args = parser.parse_args()
-    audio_array, sr = librosa.load(args.wav_path, sr=None)
     evaluators = load_evaluators(args.evaluators.split(","))
-    results = run_evaluation(audio_array, sr, evaluators)
+    results = run_evaluation(args.wav_path, evaluators)
     print(results)
 
     with open(args.results_csv, "a") as f:

interface.py

@@ -17,8 +17,8 @@ class GradioInterface:
         self.current_svs_model = (
             f"{self.default_config['language']}-{self.default_config['svs_model']}"
         )
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            self.character_info[self.current_character].default_timbre
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            self.character_info[self.current_character].default_voice
         ]
         self.pipeline = SingingDialoguePipeline(self.default_config)
 
@@ -104,21 +104,21 @@ class GradioInterface:
                                 value=self.current_svs_model,
                             )
                         with gr.Row():
-                            timbre_radio = gr.Radio(
-                                label="Singing Timbre",
+                            voice_radio = gr.Radio(
+                                label="Singing voice",
                                 choices=list(
                                     self.svs_model_map[self.current_svs_model][
-                                        "embeddings"
+                                        "voices"
                                     ].keys()
                                 ),
                                 value=self.character_info[
                                     self.current_character
-                                ].default_timbre,
+                                ].default_voice,
                             )
                 character_radio.change(
                     fn=self.update_character,
                     inputs=character_radio,
-                    outputs=[character_image, timbre_radio],
+                    outputs=[character_image, voice_radio],
                 )
                 asr_radio.change(
                     fn=self.update_asr_model, inputs=asr_radio, outputs=asr_radio
@@ -129,35 +129,41 @@ class GradioInterface:
                 svs_radio.change(
                     fn=self.update_svs_model,
                     inputs=svs_radio,
-                    outputs=[svs_radio, timbre_radio],
+                    outputs=[svs_radio, voice_radio],
                 )
                 melody_radio.change(
                     fn=self.update_melody_source,
                     inputs=melody_radio,
                     outputs=melody_radio,
                 )
-                timbre_radio.change(
-                    fn=self.update_timbre, inputs=timbre_radio, outputs=timbre_radio
+                voice_radio.change(
+                    fn=self.update_voice, inputs=voice_radio, outputs=voice_radio
                 )
                 mic_input.change(
                     fn=self.run_pipeline,
                     inputs=mic_input,
                     outputs=[interaction_log, audio_output],
                 )
+                metrics_button.click(
+                    fn=self.update_metrics,
+                    inputs=audio_output,
+                    outputs=[metrics_output],
+                )
 
             return demo
         except Exception as e:
             print(f"error: {e}")
             breakpoint()
+            return gr.Blocks()
 
     def update_character(self, character):
         self.current_character = character
-        character_timbre = self.character_info[self.current_character].default_timbre
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            character_timbre
+        character_voice = self.character_info[self.current_character].default_voice
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            character_voice
         ]
         return gr.update(value=self.character_info[character].image_path), gr.update(
-            value=character_timbre
+            value=character_voice
         )
 
     def update_asr_model(self, asr_model):
@@ -170,23 +176,23 @@ class GradioInterface:
 
     def update_svs_model(self, svs_model):
         self.current_svs_model = svs_model
-        character_timbre = self.character_info[self.current_character].default_timbre
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            character_timbre
+        character_voice = self.character_info[self.current_character].default_voice
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            character_voice
         ]
         self.pipeline.set_svs_model(
             self.svs_model_map[self.current_svs_model]["model_path"]
         )
         print(
-            f"SVS model updated to {self.current_svs_model}. Will set gradio svs_radio to {svs_model} and timbre_radio to {character_timbre}"
+            f"SVS model updated to {self.current_svs_model}. Will set gradio svs_radio to {svs_model} and voice_radio to {character_voice}"
         )
         return (
             gr.update(value=svs_model),
             gr.update(
                 choices=list(
-                    self.svs_model_map[self.current_svs_model]["embeddings"].keys()
+                    self.svs_model_map[self.current_svs_model]["voices"].keys()
                 ),
-                value=character_timbre,
+                value=character_voice,
             ),
         )
 
@@ -194,24 +200,30 @@ class GradioInterface:
         self.current_melody_source = melody_source
         return gr.update(value=self.current_melody_source)
 
-    def update_timbre(self, timbre):
-        self.current_timbre = self.svs_model_map[self.current_svs_model]["embeddings"][
-            timbre
+    def update_voice(self, voice):
+        self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][
+            voice
         ]
-        return gr.update(value=timbre)
+        return gr.update(value=voice)
 
     def run_pipeline(self, audio_path):
+        if not audio_path:
+            return gr.update(value=""), gr.update(value="")
         results = self.pipeline.run(
             audio_path,
             self.svs_model_map[self.current_svs_model]["lang"],
             self.character_info[self.current_character].prompt,
-            svs_inference_kwargs={
-                "speaker": self.current_timbre,
-            },
+            self.current_voice,
             max_new_tokens=100,
         )
         formatted_logs = f"ASR: {results['asr_text']}\nLLM: {results['llm_text']}"
         return gr.update(value=formatted_logs), gr.update(value=results["svs_audio"])
 
-    def run_evaluation(self, audio, audio_sample_rate):
-        pass
+    def update_metrics(self, audio_path):
+        if not audio_path:
+            return gr.update(value="")
+        results = self.pipeline.evaluate(audio_path)
+        formatted_metrics = "\n".join(
+            [f"{k}: {v}" for k, v in results.items()]
+        )
+        return gr.update(value=formatted_metrics)

modules/asr.py

@@ -10,14 +10,13 @@ hf_token = os.getenv("HF_TOKEN")
 
 
 class AbstractASRModel(ABC):
-    @abstractmethod
     def __init__(
         self, model_id: str, device: str = "cpu", cache_dir: str = "cache", **kwargs
     ):
+        print(f"Loading ASR model {model_id}...")
         self.model_id = model_id
         self.device = device
         self.cache_dir = cache_dir
-        pass
 
     @abstractmethod
     def transcribe(self, audio: np.ndarray, audio_sample_rate: int, **kwargs) -> str:

modules/llm.py

@@ -8,10 +8,13 @@ hf_token = os.getenv("HF_TOKEN")
 
 
 class AbstractLLMModel(ABC):
-    @abstractmethod
     def __init__(
         self, model_id: str, device: str = "cpu", cache_dir: str = "cache", **kwargs
-    ): ...
+    ):
+        print(f"Loading LLM model {model_id}...")
+        self.model_id = model_id
+        self.device = device
+        self.cache_dir = cache_dir
 
     @abstractmethod
     def generate(self, prompt: str, **kwargs) -> str:
@@ -41,6 +44,7 @@ class HFTextGenerationLLM(AbstractLLMModel):
     def __init__(
         self, model_id: str, device: str = "cpu", cache_dir: str = "cache", **kwargs
     ):
+        super().__init__(model_id, device, cache_dir, **kwargs)
         model_kwargs = kwargs.setdefault("model_kwargs", {})
         model_kwargs["cache_dir"] = cache_dir
         self.pipe = pipeline(

modules/svs/base.py

@@ -13,6 +13,8 @@ class AbstractSVSModel(ABC):
     def synthesize(
         self,
         score: list[tuple[float, float, str, int]],
+        language: str,
+        speaker: str,
         **kwargs,
     ) -> tuple[np.ndarray, int]:
         """

modules/svs/espnet.py

@@ -99,11 +99,11 @@ class ESPNetSVS(AbstractSVSModel):
         return batch
 
     def synthesize(
-        self, score: list[tuple[float, float, str, int]], language: str, **kwargs
+        self, score: list[tuple[float, float, str, int]], language: str, speaker: str, **kwargs
     ):
         batch = self._preprocess(score, language)
         if self.model_id == "espnet/aceopencpop_svs_visinger2_40singer_pretrain":
-            sid = np.array([int(kwargs["speaker"])])
+            sid = np.array([int(speaker)])
             output_dict = self.model(batch, sids=sid)
         elif self.model_id == "espnet/mixdata_svs_visinger2_spkemb_lang_pretrained":
             langs = {
@@ -115,7 +115,7 @@ class ESPNetSVS(AbstractSVSModel):
                     f"Unsupported language: {language} for {self.model_id}"
                 )
             lid = np.array([langs[language]])
-            spk_embed = np.load(kwargs["speaker"])
+            spk_embed = np.load(speaker)
             output_dict = self.model(batch, lids=lid, spembs=spk_embed)
         else:
             raise NotImplementedError(f"Model {self.model_id} not supported")

pipeline.py

@@ -55,7 +55,7 @@ class SingingDialoguePipeline:
         audio_path,
         language,
         prompt_template,
-        svs_inference_kwargs,
+        speaker,
         max_new_tokens=100,
     ):
         if self.track_latency:
@@ -81,7 +81,7 @@ class SingingDialoguePipeline:
         if self.track_latency:
             svs_start_time = time.time()
         singing_audio, sample_rate = self.svs.synthesize(
-            score, language=language, **svs_inference_kwargs
+            score, language=language, speaker=speaker
         )
         if self.track_latency:
             svs_end_time = time.time()
@@ -99,5 +99,5 @@ class SingingDialoguePipeline:
             })
         return results
 
-    def evaluate(self, audio, sample_rate):
-        return run_evaluation(audio, sample_rate, self.evaluators)
+    def evaluate(self, audio_path):
+        return run_evaluation(audio_path, self.evaluators)

requirements.txt

@@ -1,6 +1,6 @@
 git+https://github.com/espnet/espnet.git@3856d998ee0b2fa20f7b8fa48553754f33ed6e63
 espnet_model_zoo
-# pyopenjtalk
+pyopenjtalk
 datasets
 torchaudio
 typeguard==4.4.0
@@ -15,3 +15,6 @@ transformers
 s3prl
 zhconv
 git+https://github.com/sea-turt1e/kanjiconv
+soundfile
+PyYAML
+gradio

tests/audio/chat.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181a7f27f8acb00cba0276d0ff88759120a76eebd47b4e0a60c2424e43e5cbaf
+size 271030

tests/audio/feeling.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fef036c2bf0ddf635a004845e94c89d0658f754a53e12fadbb50511d3cd6c15
+size 263502

tests/audio/hello.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa7e839d32f7bda77cad11fc13fd1b92df939479612dd5af079d8f9b19598c0d
+size 263502

tests/audio/interesting.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a1618f73d90ad068d5eb72455ac812b49fcb9e44e88af5e67ef88f5c6ddb74a
+size 429086

tests/audio/music.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6388b587e282e8f6457b629b5cbb9fd50c5cb6a7f90c446329a3f23be8b1442c
+size 286082

tests/audio/where_from.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ef81772b96813216d7b14d3d70a39b040e9c542d896d9337f8975f8fd6da96e
+size 195766