refactor - move ffmpeg_converter_actor to within respond_to_prompt_actor

charles_actor.py

@@ -21,32 +21,25 @@ class CharlesActor:
     
     async def _initalize_resources(self):
         # Initialize resources
-        print("000")
+        print("000 - create StreamlitAVQueue")
         from streamlit_av_queue import StreamlitAVQueue
         self._streamlit_av_queue = StreamlitAVQueue()
         self._out_audio_queue = self._streamlit_av_queue.get_out_audio_queue()
 
-        print("001")
-        from ffmpeg_converter_actor import FFMpegConverterActor
-        self._ffmpeg_converter_actor = FFMpegConverterActor.remote(self._out_audio_queue)
-        await self._ffmpeg_converter_actor.start_process.remote()
-        self._ffmpeg_converter_actor.run.remote()
+        print("001 - create RespondToPromptActor")
+        from respond_to_prompt_actor import RespondToPromptActor
+        self._respond_to_prompt_actor = RespondToPromptActor.remote(self._out_audio_queue)
 
-        print("002")
+        print("002 - create SpeechToTextVoskActor")
         from speech_to_text_vosk_actor import SpeechToTextVoskActor
         self._speech_to_text_actor = SpeechToTextVoskActor.remote()
-
-        print("003")
-        from respond_to_prompt_actor import RespondToPromptActor
-        self._respond_to_prompt_actor = RespondToPromptActor.remote(self._ffmpeg_converter_actor)
         
         self._debug_queue = [
             # "hello, how are you today?",
             # "hmm, interesting, tell me more about that.",
         ]
 
-        print("004")
-        print("creating prototypes")
+        print("003 - create Prototypes")
         from prototypes import Prototypes
         self._prototypes = Prototypes()
         print("010")
@@ -85,7 +78,7 @@ class CharlesActor:
             if len(process_speech_to_text_future) > 0:
                 ready, _ = ray.wait([process_speech_to_text_future[0]], timeout=0)
                 if ready:
-                    prompt, speaker_finished = await process_speech_to_text_future[0]
+                    prompt, speaker_finished, raw_json = await process_speech_to_text_future[0]
                     del process_speech_to_text_future[0]
 
                     if speaker_finished and len(prompt) > 0:

respond_to_prompt_actor.py

@@ -60,22 +60,23 @@ class LLMSentanceToSpeechActor:
             await self.output_queue.get_async()
 
 
-@ray.remote
-class SpeechToSpeakerActor:
-    def __init__(self, input_queue, voice_id):
-        load_dotenv()
-        self.input_queue = input_queue
-        self.speaker_service = LocalSpeakerService()
+# legacy code for playing from local speaker
+# @ray.remote
+# class SpeechToSpeakerActor:
+#     def __init__(self, input_queue, voice_id):
+#         load_dotenv()
+#         self.input_queue = input_queue
+#         self.speaker_service = LocalSpeakerService()
 
-    async def run(self):
-        while True:
-            audio_chunk = await self.input_queue.get_async()
-            # print (f"Got audio chunk {len(audio_chunk)}")
-            self.speaker_service.add_audio_stream([audio_chunk])
+#     async def run(self):
+#         while True:
+#             audio_chunk = await self.input_queue.get_async()
+#             # print (f"Got audio chunk {len(audio_chunk)}")
+#             self.speaker_service.add_audio_stream([audio_chunk])
             
-    async def cancel(self):
-        while not self.input_queue.empty():
-            await self.input_queue.get_async()            
+#     async def cancel(self):
+#         while not self.input_queue.empty():
+#             await self.input_queue.get_async()            
 
 @ray.remote
 class SpeechToConverterActor:
@@ -85,6 +86,8 @@ class SpeechToConverterActor:
         self.ffmpeg_converter_actor = ffmpeg_converter_actor
 
     async def run(self):
+        await self.ffmpeg_converter_actor.start_process.remote()
+        self.ffmpeg_converter_actor.run.remote()
         while True:
             audio_chunk = await self.input_queue.get_async()
             # print (f"Got audio chunk {len(audio_chunk)}")
@@ -97,17 +100,18 @@ class SpeechToConverterActor:
 
 @ray.remote
 class RespondToPromptActor:
-    def __init__(self, ffmpeg_converter_actor):
+    def __init__(self, out_audio_queue):
         voice_id="2OviOUQc1JsQRQgNkVBj"
         self.prompt_queue = Queue(maxsize=100)
         self.llm_sentence_queue = Queue(maxsize=100)
         self.speech_chunk_queue = Queue(maxsize=100)
-        self.ffmepg_converter_actor = ffmpeg_converter_actor
+
+        self.ffmpeg_converter_actor = FFMpegConverterActor.remote(out_audio_queue)
         
         self.prompt_to_llm = PromptToLLMActor.remote(self.prompt_queue, self.llm_sentence_queue)
         self.llm_sentence_to_speech = LLMSentanceToSpeechActor.remote(self.llm_sentence_queue, self.speech_chunk_queue, voice_id)
         # self.speech_output = SpeechToSpeakerActor.remote(self.speech_chunk_queue, voice_id)
-        self.speech_output = SpeechToConverterActor.remote(self.speech_chunk_queue, ffmpeg_converter_actor)
+        self.speech_output = SpeechToConverterActor.remote(self.speech_chunk_queue, self.ffmpeg_converter_actor)
 
         # Start the pipeline components.
         self.prompt_to_llm.run.remote()
@@ -119,7 +123,7 @@ class RespondToPromptActor:
         prompt_to_llm_future = self.prompt_to_llm.cancel.remote()
         llm_sentence_to_speech_future = self.llm_sentence_to_speech.cancel.remote()
         speech_output_future = self.speech_output.cancel.remote()
-        ffmpeg_converter_future = self.ffmepg_converter_actor.flush_output_queue.remote()
+        ffmpeg_converter_future = self.ffmpeg_converter_actor.flush_output_queue.remote()
         await asyncio.gather(
             prompt_to_llm_future,
             llm_sentence_to_speech_future,

speech_to_text_vosk_actor.py

@@ -33,7 +33,7 @@ class SpeechToTextVoskActor:
             result = self.vosk.PartialResult()
             result_json = json.loads(result)
             text = result_json['partial']
-        return text, speaker_finished
+        return text, speaker_finished, result_json
         
 
     def add_speech_bytes(self, data: bytearray):