add hotwords (ie. speech context) demo

app/asr_worker.py

@@ -5,6 +5,8 @@ import sherpa_onnx
 import scipy.signal
 from opencc import OpenCC
 from huggingface_hub import hf_hub_download
+from typing import List
+import tempfile
 
 # Ensure Hugging Face cache is in a user-writable directory
 CACHE_DIR = Path(__file__).parent / "hf_cache"
@@ -14,6 +16,7 @@ converter = OpenCC('s2t')
 
 # Streaming Zipformer model registry: paths relative to repo root
 STREAMING_ZIPFORMER_MODELS = {
+    # bilingual zh-en with char+BPE
     "csukuangfj/k2fsa-zipformer-bilingual-zh-en-t": {
         "tokens": "data/lang_char_bpe/tokens.txt",
         "encoder_fp32": "exp/96/encoder-epoch-99-avg-1.onnx",
@@ -22,7 +25,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "exp/96/decoder-epoch-99-avg-1.int8.onnx",
         "joiner_fp32": "exp/96/joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "exp/96/joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"cjkchar+bpe",
+        "bpe_vocab":   "data/lang_char_bpe/bpe.vocab",
     },
+    # mixed Chinese+English (char+BPE)
     "pfluo/k2fsa-zipformer-chinese-english-mixed": {
         "tokens": "data/lang_char_bpe/tokens.txt",
         "encoder_fp32": "exp/encoder-epoch-99-avg-1.onnx",
@@ -31,7 +37,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": None,
         "joiner_fp32": "exp/joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "exp/joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"cjkchar+bpe",
+        "bpe_vocab":   "data/lang_char_bpe/bpe.vocab",
     },
+    # Korean-only (CJK chars)
     "k2-fsa/sherpa-onnx-streaming-zipformer-korean-2024-06-16": {
         "tokens": "tokens.txt",
         "encoder_fp32": "encoder-epoch-99-avg-1.onnx",
@@ -40,7 +49,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-99-avg-1.int8.onnx",
         "joiner_fp32": "joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"cjkchar",
+        "bpe_vocab":   None,
     },
+    # multi Chinese (Hans) (CJK chars)
     "k2-fsa/sherpa-onnx-streaming-zipformer-multi-zh-hans-2023-12-12": {
         "tokens": "tokens.txt",
         "encoder_fp32": "encoder-epoch-20-avg-1-chunk-16-left-128.onnx",
@@ -49,7 +61,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-20-avg-1-chunk-16-left-128.int8.onnx",
         "joiner_fp32": "joiner-epoch-20-avg-1-chunk-16-left-128.onnx",
         "joiner_int8": "joiner-epoch-20-avg-1-chunk-16-left-128.int8.onnx",
+        "modeling_unit":"cjkchar",
+        "bpe_vocab":   None,
     },
+    # wenetspeech streaming (CJK chars)
     "pkufool/icefall-asr-zipformer-streaming-wenetspeech-20230615": {
         "tokens": "data/lang_char/tokens.txt",
         "encoder_fp32": "exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx",
@@ -58,7 +73,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "exp/decoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx",
         "joiner_fp32": "exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx",
         "joiner_int8": "exp/joiner-epoch-12-avg-4-chunk-16-left-128.int8.onnx",
+        "modeling_unit":"cjkchar",
+        "bpe_vocab":   None,
     },
+    # English-only (BPE)
     "csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26": {
         "tokens": "tokens.txt",
         "encoder_fp32": "encoder-epoch-99-avg-1-chunk-16-left-128.onnx",
@@ -67,6 +85,8 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": None,
         "joiner_fp32": "joiner-epoch-99-avg-1-chunk-16-left-128.onnx",
         "joiner_int8": "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
+        "modeling_unit":"bpe",
+        "bpe_vocab":   None,
     },
     "csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-21": {
         "tokens": "tokens.txt",
@@ -76,6 +96,8 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-99-avg-1.int8.onnx",
         "joiner_fp32": "joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"bpe",
+        "bpe_vocab":   None,
     },
     "csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-02-21": {
         "tokens": "tokens.txt",
@@ -85,7 +107,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-99-avg-1.int8.onnx",
         "joiner_fp32": "joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"bpe",
+        "bpe_vocab":   None,
     },
+    # older bilingual zh-en (cjkchar+BPE) – no bpe.vocab shipped
     "csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20": {
         "tokens": "tokens.txt",
         "encoder_fp32": "encoder-epoch-99-avg-1.onnx",
@@ -94,7 +119,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-99-avg-1.int8.onnx",
         "joiner_fp32": "joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"cjkchar+bpe",
+        "bpe_vocab":   None,
     },
+    # French-only (BPE)
     "shaojieli/sherpa-onnx-streaming-zipformer-fr-2023-04-14": {
         "tokens": "tokens.txt",
         "encoder_fp32": "encoder-epoch-29-avg-9-with-averaged-model.onnx",
@@ -103,7 +131,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-29-avg-9-with-averaged-model.int8.onnx",
         "joiner_fp32": "joiner-epoch-29-avg-9-with-averaged-model.onnx",
         "joiner_int8": "joiner-epoch-29-avg-9-with-averaged-model.int8.onnx",
+        "modeling_unit":"bpe",
+        "bpe_vocab":   None,
     },
+    # Chinese-only small (CJK chars)
     "csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23": {
         "tokens": "tokens.txt",
         "encoder_fp32": "encoder-epoch-99-avg-1.onnx",
@@ -112,7 +143,10 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-99-avg-1.int8.onnx",
         "joiner_fp32": "joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"cjkchar",
+        "bpe_vocab":   None,
     },
+    # English-only 20M (BPE)
     "csukuangfj/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17": {
         "tokens": "tokens.txt",
         "encoder_fp32": "encoder-epoch-99-avg-1.onnx",
@@ -121,6 +155,8 @@ STREAMING_ZIPFORMER_MODELS = {
         "decoder_int8": "decoder-epoch-99-avg-1.int8.onnx",
         "joiner_fp32": "joiner-epoch-99-avg-1.onnx",
         "joiner_int8": "joiner-epoch-99-avg-1.int8.onnx",
+        "modeling_unit":"bpe",
+        "bpe_vocab":   None,
     },
 }
 
@@ -131,7 +167,12 @@ def resample_audio(audio: np.ndarray, orig_sr: int, target_sr: int) -> np.ndarra
 # Create an online recognizer for a given model and precision
 # model_id: full HF repo ID
 # precision: "int8" or "fp32"
-def create_recognizer(model_id: str, precision: str):
+def create_recognizer(
+    model_id: str,
+    precision: str,
+    hotwords: List[str] = None,
+    hotwords_score: float = 0.0,
+):
     if model_id not in STREAMING_ZIPFORMER_MODELS:
         raise ValueError(f"Model '{model_id}' is not registered.")
     entry = STREAMING_ZIPFORMER_MODELS[model_id]
@@ -146,6 +187,54 @@ def create_recognizer(model_id: str, precision: str):
     decoder_path = hf_hub_download(repo_id=model_id, filename=decoder_file, cache_dir=str(CACHE_DIR))
     joiner_path = hf_hub_download(repo_id=model_id, filename=joiner_file, cache_dir=str(CACHE_DIR))
 
+    # ——— Download BPE vocab if this model has one ———
+    modeling_unit = entry.get("modeling_unit")
+    bpe_rel_path  = entry.get("bpe_vocab")
+    bpe_vocab_path = None
+    if bpe_rel_path:
+        try:
+            bpe_vocab_path = hf_hub_download(
+                repo_id=model_id,
+                filename=bpe_rel_path,
+                cache_dir=str(CACHE_DIR),
+            )
+            print(f"[DEBUG asr_worker] Downloaded bpe_vocab: {bpe_vocab_path}")
+        except Exception as e:
+            print(f"[WARNING asr_worker] Could not download bpe_vocab '{bpe_rel_path}': {e}")
+            bpe_vocab_path = None
+
+    # ——— Decide whether to use beam search with hotword biasing ———
+    use_beam = (hotwords and hotwords_score > 0.0) and bpe_vocab_path
+    if use_beam:
+            # Write hotword list to a temp file (one entry per line)
+            tf = tempfile.NamedTemporaryFile(
+                mode="w", delete=False, suffix=".txt", dir=str(CACHE_DIR)
+            )
+            for w in hotwords:
+                tf.write(f"{w}\\n")
+            tf.flush()
+            tf.close()
+            hotwords_file_path = tf.name
+            print(f"[DEBUG asr_worker] Written {len(hotwords)} hotwords to {hotwords_file_path} with score {hotwords_score}")
+
+            # Create beam-search recognizer with biasing :contentReference[oaicite:0]{index=0}
+            return sherpa_onnx.OnlineRecognizer.from_transducer(
+                tokens=tokens_path,
+                encoder=encoder_path,
+                decoder=decoder_path,
+                joiner=joiner_path,
+                provider="cpu",
+                num_threads=1,
+                sample_rate=16000,
+                feature_dim=80,
+                decoding_method="modified_beam_search",
+                hotwords_file=hotwords_file_path,
+                hotwords_score=hotwords_score,
+                modeling_unit=modeling_unit,
+                bpe_vocab=bpe_vocab_path,
+            )
+
+    # ——— Fallback to original greedy-search (no hotword biasing) ———
     return sherpa_onnx.OnlineRecognizer.from_transducer(
         tokens=tokens_path,
         encoder=encoder_path,
@@ -155,7 +244,7 @@ def create_recognizer(model_id: str, precision: str):
         num_threads=1,
         sample_rate=16000,
         feature_dim=80,
-        decoding_method="greedy_search"
+        decoding_method="greedy_search",
     )
 
 def stream_audio(raw_pcm_bytes, stream, recognizer, orig_sr):

app/main.py

@@ -42,15 +42,27 @@ async def websocket_endpoint(websocket: WebSocket):
                     print(f"[ERROR main] JSON parse failed: {e}")
                     continue
                 if config_msg.get("type") == "config":
+                    # 1) sample rate
                     orig_sr = int(config_msg["sampleRate"])
                     print(f"[INFO main] Set original sample rate to {orig_sr}")
 
-                    # New: dynamic model & precision
-                    model_id = config_msg.get("model")
+                    # 2) model & precision
+                    model_id  = config_msg.get("model")
                     precision = config_msg.get("precision")
                     print(f"[INFO main] Selected model: {model_id}, precision: {precision}")
 
-                    recognizer = create_recognizer(model_id, precision)
+                    # 3) hotwords & boost score
+                    hotwords      = config_msg.get("hotwords", [])
+                    hotwords_score = float(config_msg.get("hotwordsScore", 0.0))
+                    print(f"[INFO main] Hotwords: {hotwords}, score: {hotwords_score}")
+
+                    # 4) create recognizer with biasing
+                    recognizer = create_recognizer(
+                        model_id,
+                        precision,
+                        hotwords=hotwords,
+                        hotwords_score=hotwords_score
+                    )
                     stream = recognizer.create_stream()
                     print("[INFO main] WebSocket connection accepted; created a streaming context.")
                 continue

app/static/index.html

@@ -21,6 +21,13 @@
       margin-bottom: 1rem;
       font-size: 2rem;
     }
+  
+    /* Added for Hotword Bias status */
+    #hotwordStatus {
+      font-size: 0.9rem;
+      color: #e1b12c;
+      font-weight: bold;
+    }
 
     #vol {
       width: 300px;
@@ -86,11 +93,19 @@
       font-weight: bold;
       color: #2f3640;
     }
-    .controls select {
+    .controls select,
+    .controls input[type="number"],
+    .controls textarea {
       padding: 0.3rem;
       border-radius: 5px;
       border: 1px solid #dcdde1;
       background: white;
+      font-size: 1rem;
+    }
+    .controls textarea {
+      flex: 1;
+      resize: vertical;
+      min-height: 4rem;
     }
 
     .model-info {
@@ -139,6 +154,22 @@
     </select>
   </div>
 
+  <div class="controls">
+    <!-- Hotwords List Input -->
+    <label for="hotwordsList">Hotwords:</label>
+    <textarea id="hotwordsList" placeholder="Enter one hotword per line"></textarea>
+    <!-- Global Boost Score Input -->  
+    <label for="boostScore">Boost Score:</label>
+    <input type="number" id="boostScore" min="0" max="10" step="0.1" value="2.0" />
+    <!-- ← NEW button to submit hotword changes -->
+    <button id="applyHotwords">Apply Hotwords</button>
+  </div>
+
+  <!-- ← NEW indicator showing whether biasing is ON or OFF -->
+  <div class="controls">
+    <span id="hotwordStatus">Hotword Bias: Off</span>
+  </div>
+
   <div class="model-info" id="modelInfo">
     Languages: <span id="modelLangs"></span> | Size: <span id="modelSize"></span> MB
   </div>
@@ -178,11 +209,24 @@
     const transcript = document.getElementById("transcript");
     const modelSelect = document.getElementById("modelSelect");
     const precisionSelect = document.getElementById("precisionSelect");
+    const hotwordsList = document.getElementById("hotwordsList");
+    const boostScore = document.getElementById("boostScore");
+    const applyBtn      = document.getElementById("applyHotwords");
+    const hotwordStatus = document.getElementById("hotwordStatus");
     const modelLangs = document.getElementById("modelLangs");
     const modelSize = document.getElementById("modelSize");
     const micNameElem = document.getElementById("micName");
     const sampleRateElem = document.getElementById("sampleRate");
 
+    // ← Helper to toggle the status text
+    function updateHotwordStatus() {
+      const enabled = hotwordsList.value.split(/\r?\n/).filter(Boolean).length > 0
+        && parseFloat(boostScore.value) > 0;
+      hotwordStatus.textContent = enabled
+        ? "Hotword Bias: On"
+        : "Hotword Bias: Off";
+    }
+
     function updateModelInfo() {
       const meta = MODEL_METADATA[modelSelect.value];
       if (Array.isArray(meta.language)) {
@@ -199,7 +243,9 @@
           type: "config",
           sampleRate: orig_sample_rate,
           model: modelSelect.value,
-          precision: precisionSelect.value
+          precision: precisionSelect.value,
+          hotwords: hotwordsList.value.split(/\r?\n/).filter(Boolean),
+          hotwordsScore: parseFloat(boostScore.value)
         }));
       } else {
         console.warn("WebSocket not open yet. Cannot send config.");
@@ -218,7 +264,7 @@
       updateModelInfo();
       
       // Now that we know the sample rate, open the WS
-      ws = new WebSocket(`wss://${location.host}/ws`);
+      ws = new WebSocket(`ws://${location.host}/ws`);
       ws.onopen = () => sendConfig();
       ws.onerror = err => console.error("WebSocket error:", err);
       ws.onclose = () => console.log("WebSocket closed");
@@ -238,8 +284,20 @@
       modelSelect.addEventListener("change", () => {
         updateModelInfo();
         sendConfig();
+        updateHotwordStatus();
+      });
+      precisionSelect.addEventListener("change", () => {
+        sendConfig();
+        updateHotwordStatus();
+      });
+      // hotwordsList.addEventListener("input", sendConfig);
+      // boostScore.addEventListener("input", sendConfig);
+
+      // ← Re-send config & update indicator when the button is clicked
+      applyBtn.addEventListener("click", () => {
+        sendConfig();
+        updateHotwordStatus();
       });
-      precisionSelect.addEventListener("change", sendConfig);
 
       const source = context.createMediaStreamSource(stream);
       const processor = context.createScriptProcessor(4096, 1, 1);