VAC

- performance tests pending
- TODO: timestamps after refresh are decreasing

voice_activity_controller.py

@@ -1,18 +1,5 @@
 import torch
 import numpy as np
-# import sounddevice as sd
-import torch
-import numpy as np
-import datetime
-
-
-def int2float(sound):
-    abs_max = np.abs(sound).max()
-    sound = sound.astype('float32')
-    if abs_max > 0:
-        sound *= 1/32768
-    sound = sound.squeeze()  # depends on the use case
-    return sound
 
 class VoiceActivityController:
     def __init__(
@@ -22,10 +9,10 @@ class VoiceActivityController:
             min_speech_to_final_ms = 100,
             min_silence_duration_ms = 100,
             use_vad_result = True,
-            activity_detected_callback=None,
+#            activity_detected_callback=None,
             threshold =0.3
         ):
-        self.activity_detected_callback=activity_detected_callback
+#        self.activity_detected_callback=activity_detected_callback
         self.model, self.utils = torch.hub.load(
             repo_or_dir='snakers4/silero-vad',
             model='silero_vad'
@@ -42,7 +29,6 @@ class VoiceActivityController:
         self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
 
         self.use_vad_result = use_vad_result
-        self.last_marked_chunk = None
         self.threshold = threshold
         self.reset_states()
 
@@ -55,7 +41,13 @@ class VoiceActivityController:
         self.speech_len = 0
 
     def apply_vad(self, audio):
-#        x = int2float(audio)
+        """
+        returns: triple
+            (voice_audio,
+            speech_in_wav,
+            silence_in_wav)
+
+        """
         x = audio
         if not torch.is_tensor(x):
             try:
@@ -64,16 +56,16 @@ class VoiceActivityController:
                 raise TypeError("Audio cannot be casted to tensor. Cast it manually")
 
         speech_prob = self.model(x, self.sampling_rate).item()
+        print("speech_prob",speech_prob)
         
         window_size_samples = len(x[0]) if x.dim() == 2 else len(x)
         self.current_sample += window_size_samples 
 
-
-        if (speech_prob >= self.threshold):
+        if speech_prob >= self.threshold:  # speech is detected
             self.temp_end = 0
             return audio, window_size_samples, 0
 
-        else :
+        else:  # silence detected, counting w
             if not self.temp_end:
                 self.temp_end = self.current_sample
 
@@ -84,14 +76,12 @@ class VoiceActivityController:
 
 
     def detect_speech_iter(self, data, audio_in_int16 = False):
-#        audio_block = np.frombuffer(data, dtype=np.int16) if not audio_in_int16 else data
         audio_block = data
         wav = audio_block
 
-        print(wav, len(wav), type(wav), wav.dtype)
-        
         is_final = False
         voice_audio, speech_in_wav, last_silent_in_wav = self.apply_vad(wav)
+        print("speech, last silence",speech_in_wav, last_silent_in_wav)
 
 
         if speech_in_wav > 0 :
@@ -101,27 +91,20 @@ class VoiceActivityController:
 #                self.activity_detected_callback()
 
         self.last_silence_len +=  last_silent_in_wav
+        print("self.last_silence_len",self.last_silence_len, self.final_silence_limit,self.last_silence_len>= self.final_silence_limit)
+        print("self.speech_len, final_speech_limit",self.speech_len , self.final_speech_limit,self.speech_len >= self.final_speech_limit)
         if self.last_silence_len>= self.final_silence_limit and self.speech_len >= self.final_speech_limit:
+            for i in range(10): print("TADY!!!")
 
             is_final = True
             self.last_silence_len= 0
             self.speech_len = 0                
 
-#        return voice_audio.tobytes(), is_final
         return voice_audio, is_final
 
-
-
     def detect_user_speech(self, audio_stream, audio_in_int16 = False):
         self.last_silence_len= 0
         self.speech_len = 0
 
         for data in audio_stream:  # replace with your condition of choice
             yield self.detect_speech_iter(data, audio_in_int16)
-           
-
-
-
-
-
-

whisper_online_server.py

@@ -9,7 +9,8 @@ parser = argparse.ArgumentParser()
 # server options
 parser.add_argument("--host", type=str, default='localhost')
 parser.add_argument("--port", type=int, default=43007)
-
+parser.add_argument('--vac', action="store_true", default=False, help='Use VAC = voice activity controller.')
+parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.')
 
 # options from whisper_online
 add_shared_args(parser)
@@ -57,8 +58,11 @@ if args.buffer_trimming == "sentence":
     tokenizer = create_tokenizer(tgt_language)
 else:
     tokenizer = None
-online = OnlineASRProcessor(asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
-
+if not args.vac:
+    online = OnlineASRProcessor(asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
+else:
+    from whisper_online_vac import *
+    online = VACOnlineASRProcessor(min_chunk, asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
 
 
 demo_audio_path = "cs-maji-2.16k.wav"

whisper_online_vac.py

@@ -7,52 +7,46 @@ SAMPLING_RATE = 16000
 
 class VACOnlineASRProcessor(OnlineASRProcessor):
 
-    def __init__(self, *a, **kw):
+    def __init__(self, online_chunk_size, *a, **kw):
+        self.online_chunk_size = online_chunk_size
+
         self.online = OnlineASRProcessor(*a, **kw)
-        self.vac = VoiceActivityController(use_vad_result = True)
+        self.vac = VoiceActivityController(use_vad_result = False)
 
-        self.is_currently_final = False
         self.logfile = self.online.logfile
 
-        #self.vac_buffer = io.BytesIO()
-        #self.vac_stream = self.vac.detect_user_speech(self.vac_buffer, audio_in_int16=False)
-
-        self.audio_log = open("audio_log.wav","wb")
+        self.init()
 
     def init(self):
         self.online.init()
         self.vac.reset_states()
+        self.current_online_chunk_buffer_size = 0
+        self.is_currently_final = False
+
 
     def insert_audio_chunk(self, audio):
-        print(audio, len(audio), type(audio), audio.dtype)
         r = self.vac.detect_speech_iter(audio,audio_in_int16=False)
-        raw_bytes, is_final = r
-        print("is_final",is_final)
-        print("raw_bytes", raw_bytes[:10], len(raw_bytes), type(raw_bytes))
-#        self.audio_log.write(raw_bytes)
-        #sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW")
-        #audio, _ = librosa.load(sf,sr=SAMPLING_RATE)
-        audio = raw_bytes
-        print("po překonvertování", audio, len(audio), type(audio), audio.dtype)
+        audio, is_final = r
+        print(is_final)
         self.is_currently_final = is_final
         self.online.insert_audio_chunk(audio)
-#        self.audio_log.write(audio)
-        self.audio_log.flush()
-
-        print("inserted",file=self.logfile)
+        self.current_online_chunk_buffer_size += len(audio)
 
     def process_iter(self):
         if self.is_currently_final:
             return self.finish()
-        else:
-            print(self.online.audio_buffer)
+        elif self.current_online_chunk_buffer_size > SAMPLING_RATE*self.online_chunk_size:
+            self.current_online_chunk_buffer_size = 0
             ret = self.online.process_iter()
-            print("tady",file=self.logfile)
             return ret
+        else:
+            print("no online update, only VAD", file=self.logfile)
+            return (None, None, "")
 
     def finish(self):
         ret = self.online.finish()
         self.online.init()
+        self.current_online_chunk_buffer_size = 0
         return ret
 
 
@@ -67,7 +61,7 @@ if __name__ == "__main__":
     parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.')
     parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.')
     parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.')
-    
+    parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.') 
     args = parser.parse_args()
 
     # reset to store stderr to different file stream, e.g. open(os.devnull,"w")
@@ -111,12 +105,12 @@ if __name__ == "__main__":
         asr.use_vad()
 
     
-    min_chunk = args.min_chunk_size
+    min_chunk = args.vac_chunk_size
     if args.buffer_trimming == "sentence":
         tokenizer = create_tokenizer(tgt_language)
     else:
         tokenizer = None
-    online = VACOnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
+    online = VACOnlineASRProcessor(args.min_chunk_size, asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
 
 
     # load the audio into the LRU cache before we start the timer