.

AR/models/structs.py

@@ -17,6 +17,7 @@ Tensor = torch.Tensor
 @dataclass
 class T2SResult:
     result: List[Tensor] | None = None
+    infer_speed: float = 0.0
     status: Literal["Success", "Error"] = "Success"
     exception: Optional[Exception] = None
     traceback: Optional[str] = None

AR/models/t2s_model_flash_attn.py

@@ -226,7 +226,7 @@ class CUDAGraphRunner:
         self.kv_cache = decoder_model.init_cache(1)
         self.input_pos = torch.tensor([10]).int().cuda()
 
-    def _handle_request(self, request: T2SRequest) -> List[torch.Tensor]:
+    def _handle_request(self, request: T2SRequest):
         with self.device:
             for i in self.kv_cache:
                 i.empty()
@@ -236,6 +236,7 @@ class CUDAGraphRunner:
             self.input_pos.copy_(session.input_pos)
 
             t1 = 0.0
+            infer_speed = 0.0
             y = session.y
             bsz = y.size(0)
             torch_profiler = TorchProfiler(request.debug)
@@ -314,9 +315,7 @@ class CUDAGraphRunner:
                                     f"T2S Decoding EOS {session.prefill_len.tolist().__str__().strip('[]')} -> \n{[i.size(0) for i in session.y_results].__str__().strip('[]')}"
                                 )
                                 tqdm.write(f"Infer Speed: {(idx - 1) / (time.perf_counter() - t1):.2f} token/s")
-                                gr.Info(
-                                    f"Infer Speed: {(idx - 1) / (time.perf_counter() - t1):.2f} token/s", duration=0.75
-                                )
+                                infer_speed = (idx - 1) / (time.perf_counter() - t1)
                             break
 
                         if (
@@ -362,12 +361,12 @@ class CUDAGraphRunner:
                     torch.mtia.empty_cache()
 
             torch_profiler.end()
-            return session.y_results[: request.valid_length]
+            return session.y_results[: request.valid_length], infer_speed
 
     def generate(self, request: T2SRequest):
         try:
-            result = self._handle_request(request)
-            t2s_result = T2SResult(result=result, status="Success")
+            result, infer_speed = self._handle_request(request)
+            t2s_result = T2SResult(result=result, infer_speed=infer_speed, status="Success")
         except Exception as e:
             t2s_result = T2SResult(status="Error", exception=e, traceback=traceback.format_exc())
         return t2s_result

inference_webui.py

@@ -519,6 +519,8 @@ def get_tts_wav(
     if not ref_free:
         phones1, bert1, norm_text1 = get_phones_and_bert(prompt_text, prompt_language, version)
 
+    infer_speed: list[float] = []
+
     for i_text, text in enumerate(texts):
         # 解决输入目标文本的空行导致报错的问题
         if len(text.strip()) == 0:
@@ -559,11 +561,15 @@ def get_tts_wav(
                     # debug=True,
                 )
                 t2s_result = t2s_model.generate(t2s_request)
-                pred_semantic = t2s_result.result
-                if pred_semantic is None:
+
+                if t2s_result.exception is not None:
                     print(t2s_result.exception)
                     print(t2s_result.traceback)
                     raise RuntimeError("")
+
+                infer_speed.append(t2s_result.infer_speed)
+                pred_semantic = t2s_result.result
+                assert pred_semantic
                 cache[i_text] = pred_semantic
         t3 = ttime()
         refers = []
@@ -601,6 +607,7 @@ def get_tts_wav(
         t.extend([t2 - t1, t3 - t2, t4 - t3])
         t1 = ttime()
     print("%.3f\t%.3f\t%.3f\t%.3f" % (t[0], sum(t[1::3]), sum(t[2::3]), sum(t[3::3])))
+    gr.Info(f"Infer Speed: {sum(infer_speed) / len(infer_speed):.2f} Token/s")
     gr.Info("%.3f\t%.3f\t%.3f\t%.3f" % (t[0], sum(t[1::3]), sum(t[2::3]), sum(t[3::3])), duration=4)
     yield hps.data.sampling_rate, (np.concatenate(audio_opt, 0) * 32768).astype(np.int16)
 
@@ -735,7 +742,7 @@ def html_left(text, label="p"):
                 </div>"""
 
 
-with gr.Blocks(title="GPT-SoVITS WebUI", theme=themes.Monochrome(), analytics_enabled=False) as app:
+with gr.Blocks(title="GPT-SoVITS WebUI", theme=themes.Soft(), analytics_enabled=False) as app:
     gr.Markdown(
         value="""# GPT-SoVITS-ProPlus Zero-shot TTS demo
 ## https://github.com/RVC-Boss/GPT-SoVITS
@@ -837,8 +844,39 @@ This demo is open source under the MIT license. The author does not have any con
         )
 
 if __name__ == "__main__":
+    import tempfile
+    import wave
+
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as temp_file:
+        file_name = temp_file.name
+        with wave.open(temp_file, "w") as wav_file:
+            channels = 1
+            sample_width = 2
+            sample_rate = 44100
+            duration = 5
+            frequency = 440.0
+
+            t = np.linspace(0, duration, int(sample_rate * duration), endpoint=False)
+            sine_wave = np.sin(2 * np.pi * frequency * t)  # Sine Wave
+            int_wave = (sine_wave * 32767).astype(np.int16)
+
+            wav_file.setnchannels(channels)  # pylint: disable=no-member
+            wav_file.setsampwidth(sample_width)  # pylint: disable=no-member
+            wav_file.setframerate(sample_rate)  # pylint: disable=no-member
+            wav_file.writeframes(int_wave.tobytes())  # pylint: disable=no-member
+
+            gen = get_tts_wav(
+                ref_wav_path=file_name,
+                prompt_text="",
+                prompt_language=i18n("中文"),
+                text="犯大吴疆土者,盛必击而破之,犯大吴疆土者,盛必击而破之,犯大吴疆土者,盛必击而破之,犯大吴疆土者,盛必击而破之",
+                text_language=i18n("中文"),
+            )
+            next(gen)
+
     app.queue().launch(
         server_name="0.0.0.0",
         inbrowser=True,
         show_api=False,
+        allowed_paths=["/"],
     )