Merge branch 'main' of github.com:gpt-omni/mini-omni

README.md

@@ -24,7 +24,7 @@ Mini-Omni is an open-source multimodal large language model that can **hear, tal
 
 ✅ **Talking while thinking**, with the ability to generate text and audio at the same time.
 
-✅ **Streaming audio outupt** capabilities.
+✅ **Streaming audio output** capabilities.
 
 ✅ With "Audio-to-Text" and "Audio-to-Audio" **batch inference** to further boost the performance.
 

inference.py

@@ -399,7 +399,7 @@ class OmniInference:
         model = self.model
 
         with self.fabric.init_tensor():
-            model.set_kv_cache(batch_size=2)
+            model.set_kv_cache(batch_size=2,device=self.device)
 
         mel, leng = load_audio(audio_path)
         audio_feature, input_ids = get_input_ids_whisper_ATBatch(mel, leng, self.whispermodel, self.device)

server.py

@@ -46,9 +46,9 @@ def create_app():
     return server.server
 
 
-def serve(ip='0.0.0.0', port=60808):
+def serve(ip='0.0.0.0', port=60808, device='cuda:0'):
 
-    OmniChatServer(ip, port=port, run_app=True)
+    OmniChatServer(ip, port=port,run_app=True, device=device)
 
 
 if __name__ == "__main__":