prepare to integrate whisper

.gitignore

@@ -1,2 +1,3 @@
 /target
 .idea/
+models

Cargo.lock

@@ -1422,6 +1422,7 @@ dependencies = [
  "poem",
  "serde",
  "serde_json",
+ "serde_yaml",
  "tokio",
  "tokio-stream",
  "tracing-subscriber",
@@ -1740,6 +1741,19 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
+dependencies = [
+ "indexmap 2.0.2",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.6"
@@ -2184,6 +2198,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
+
 [[package]]
 name = "untrusted"
 version = "0.7.1"

Cargo.toml

@@ -16,6 +16,7 @@ tracing-subscriber = "0.3.17"
 futures-util = "0.3.28"
 serde = { version = "1.0.189", features = ["derive"] }
 serde_json = { version = "1.0.107", features = [] }
+serde_yaml = "0.9.25"
 whisper-rs = {  version = "0.8.0" , features = ["coreml"] }
 
 [dependencies.poem]

config.yaml

@@ -0,0 +1,22 @@
+server:
+    port: 8080
+    host: ::1
+whisper:
+    n_threads: 4
+    step_ms: 500
+    length_ms: 5000
+    keep_ms: 200
+    capture_id: -1
+    max_tokens: 32
+    audio_ctx: 0
+    vad_thold: 0.6
+    freq_thold: 100.0
+    speed_up: false
+    translate: false
+    no_fallback: false
+    print_special: false
+    no_context: true
+    no_timestamps: false
+    tinydiarize: false
+    language: "en"
+    model: "models/ggml-base.bin"

src/config.rs

@@ -0,0 +1,76 @@
+use std::ffi::c_int;
+use std::fs;
+use std::net::IpAddr;
+use serde::{Deserialize};
+use whisper_rs::FullParams;
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct WhisperParams {
+    pub(crate) n_threads: Option<usize>,
+    pub(crate) step_ms: i32,
+    pub(crate) length_ms: i32,
+    pub(crate) keep_ms: i32,
+    pub(crate) capture_id: i32,
+    pub(crate) max_tokens: i32,
+    pub(crate) audio_ctx: i32,
+    pub(crate) vad_thold: f32,
+    pub(crate) freq_thold: f32,
+    pub(crate) speed_up: bool,
+    pub(crate) translate: bool,
+    pub(crate) no_fallback: bool,
+    pub(crate) print_special: bool,
+    pub(crate) no_context: bool,
+    pub(crate) no_timestamps: bool,
+    pub(crate) tinydiarize: bool,
+    pub(crate) language: Option<String>,
+    pub(crate) model: String,
+}
+
+const NONE: [c_int;0] = [];
+
+impl WhisperParams {
+    pub(crate) fn to_full_params<'a, 'b>(&'a self) -> FullParams<'a, 'b> {
+        let mut param = FullParams::new(Default::default());
+        param.set_print_progress(false);
+        param.set_print_special(self.print_special);
+        param.set_print_realtime(false);
+        param.set_print_timestamps(!self.no_timestamps);
+        param.set_translate(self.translate);
+        param.set_single_segment(true);
+        param.set_max_tokens(self.max_tokens);
+        let lang = self.language.as_ref().map(|s| s.as_str());
+        param.set_language(lang);
+        let num_cpus = std::thread::available_parallelism().map(|c| c.get()).unwrap_or(4);
+        param.set_n_threads(self.n_threads.unwrap_or(num_cpus) as c_int);
+        param.set_audio_ctx(self.audio_ctx);
+        param.set_speed_up(self.speed_up);
+        // param.set_tdrz_enable(self.tinydiarize);
+        if self.no_fallback {
+            param.set_temperature_inc(0.0);
+        }
+        if self.no_context {
+            param.set_tokens(&NONE);
+        }
+
+        param
+    }
+}
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct Server {
+    pub(crate) port: u16,
+    pub(crate) host: IpAddr,
+}
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct Config {
+    pub(crate) whisper: WhisperParams,
+    pub(crate) server: Server,
+}
+
+#[tokio::test]
+async fn load() {
+    let config_str = fs::read_to_string("config.yaml").expect("failed to read config file");
+    let params: Config = serde_yaml::from_str(config_str.as_str()).expect("failed to parse config file");
+    println!("{:?}", params);
+}

src/main.rs

@@ -17,13 +17,17 @@ use poem::web::websocket::{Message, WebSocket};
 use futures_util::stream::StreamExt;
 use poem::web::{Data, Query};
 
-use tokio::select;
+use tokio::{fs, select};
 use serde::{Deserialize, Serialize};
 use whisper_rs::WhisperContext;
 use lesson::{LessonsManager};
+use crate::config::Config;
 use crate::lesson::Viseme;
+use crate::whisper::run_whisper;
 
 mod lesson;
+mod config;
+mod whisper;
 
 
 #[derive(Debug, Parser)]
@@ -46,12 +50,25 @@ struct Context {
     lessons_manager: LessonsManager,
 }
 
+#[derive(Debug)]
+enum Error {
+    IoError(std::io::Error),
+    ConfigError(serde_yaml::Error),
+}
+
+async fn load_config() -> Result<Config, Error> {
+    let config_str = fs::read_to_string("config.yaml").await.map_err(|e| Error::IoError(e))?;
+    let config: Config = serde_yaml::from_str(config_str.as_str())
+        .map_err(|e| Error::ConfigError(e))?;
+    return Ok(config)
+}
+
 #[tokio::main]
 async fn main() -> Result<(), std::io::Error> {
     tracing_subscriber::fmt::init();
-    let wc = WhisperContext::new("/Users/famer.me/Downloads/ggml-base.bin");
-    let wc = wc.expect("failed to load whisper context");
-    let _ = wc.create_state().expect("failed to create state");
+
+    let config = load_config().await.expect("failed to load config");
+    run_whisper(&config).await;
 
     let Opt {
         region,

src/whisper.rs

@@ -0,0 +1,29 @@
+use whisper_rs::WhisperContext;
+use crate::config::Config;
+
+pub(crate) async fn run_whisper(config: &Config) {
+    let ctx = WhisperContext::new(&*config.whisper.model).expect("failed to load whisper context");
+    let mut _state = ctx.create_state().expect("failed to create state");
+    let params = (&config.whisper).to_full_params();
+    _state.full(params, &[]).expect("TODO: panic message");
+}
+
+async fn pcm_i16_to_f32(input: &Vec<u8>) -> Vec<f32> {
+    let pcm_i16 = input
+        .chunks_exact(2)
+        .map(|chunk| {
+            let mut buf = [0u8; 2];
+            buf.copy_from_slice(chunk);
+            i16::from_le_bytes(buf)
+        })
+        .collect::<Vec<i16>>();
+    let pcm_f32 = pcm_i16
+        .iter()
+        .map(|i| *i as f32 / i16::MAX as f32)
+        .collect::<Vec<f32>>();
+    pcm_f32
+}
+
+struct WhisperHandler {
+
+}