Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
@@ -4,41 +4,66 @@ import difflib
|
|
4 |
import re
|
5 |
import jiwer
|
6 |
import torch
|
7 |
-
|
8 |
-
|
9 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
from indic_transliteration import sanscript
|
11 |
from indic_transliteration.sanscript import transliterate
|
12 |
-
import
|
|
|
13 |
|
14 |
# ---------------- CONFIG ---------------- #
|
15 |
-
MODEL_NAME = "large-v2"
|
16 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
17 |
|
18 |
LANG_CODES = {
|
19 |
"English": "en",
|
20 |
-
"Tamil": "ta",
|
21 |
"Malayalam": "ml",
|
22 |
"Hindi": "hi",
|
23 |
"Sanskrit": "sa"
|
24 |
}
|
25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
26 |
LANG_PRIMERS = {
|
27 |
-
"English": ("
|
28 |
-
"Write only in English
|
29 |
-
"Tamil": ("
|
30 |
-
"เฎคเฎฎเฎฟเฎดเฏ เฎเฎดเฏเฎคเฏเฎคเฏเฎเฏเฎเฎณเฎฟเฎฒเฏ เฎฎเฎเฏเฎเฏเฎฎเฏ
|
31 |
-
"Malayalam": ("
|
32 |
-
"เดฎเดฒเดฏเดพเดณ เดฒเดฟเดชเดฟเดฏเดฟเตฝ เดฎเดพเดคเตเดฐเด
|
33 |
-
"Hindi": ("
|
34 |
-
"เคเฅเคตเคฒ เคฆเฅเคตเคจเคพเคเคฐเฅ เคฒเคฟเคชเคฟ เคฎเฅเค
|
35 |
-
"Sanskrit": ("
|
36 |
-
"
|
37 |
}
|
38 |
|
39 |
SCRIPT_PATTERNS = {
|
40 |
"Tamil": re.compile(r"[เฎ-เฏฟ]"),
|
41 |
-
"Malayalam": re.compile(r"[เด-เตฟ]"),
|
42 |
"Hindi": re.compile(r"[เค-เฅฟ]"),
|
43 |
"Sanskrit": re.compile(r"[เค-เฅฟ]"),
|
44 |
"English": re.compile(r"[A-Za-z]")
|
@@ -46,206 +71,404 @@ SCRIPT_PATTERNS = {
|
|
46 |
|
47 |
SENTENCE_BANK = {
|
48 |
"English": [
|
49 |
-
"The sun sets over the horizon.",
|
50 |
-
"Learning languages
|
51 |
-
"I
|
|
|
|
|
52 |
],
|
53 |
"Tamil": [
|
54 |
"เฎเฎฉเฏเฎฑเฏ เฎจเฎฒเฏเฎฒ เฎตเฎพเฎฉเฎฟเฎฒเฏ เฎเฎณเฏเฎณเฎคเฏ.",
|
55 |
-
"เฎจเฎพเฎฉเฏ เฎคเฎฎเฎฟเฎดเฏ เฎเฎฑเฏเฎฑเฏเฎเฏเฎเฏเฎฃเฏเฎเฏ เฎเฎฐเฏเฎเฏเฎเฎฟเฎฑเฏเฎฉเฏ.",
|
56 |
-
"เฎเฎฉเฎเฏเฎเฏ เฎชเฏเฎคเฏเฎคเฎเฎฎเฏ เฎชเฎเฎฟเฎเฏเฎ เฎตเฎฟเฎฐเฏเฎชเฏเฎชเฎฎเฏ."
|
|
|
|
|
57 |
],
|
58 |
"Malayalam": [
|
59 |
"เดเดจเดฟเดเตเดเต เดฎเดฒเดฏเดพเดณเด เดตเดณเดฐเต เดเดทเตเดเดฎเดพเดฃเต.",
|
60 |
"เดเดจเตเดจเต เดฎเดดเดชเตเดฏเตเดฏเตเดจเตเดจเต.",
|
61 |
-
"เดเดพเตป เดชเตเดธเตเดคเดเด เดตเดพเดฏเดฟเดเตเดเตเดจเตเดจเต."
|
|
|
|
|
62 |
],
|
63 |
"Hindi": [
|
64 |
-
"เคเค เคฎเฅเคธเคฎ เค
เคเฅเคเคพ เคนเฅเฅค",
|
65 |
-
"เคฎเฅเคเฅ เคนเคฟเคเคฆเฅ เคฌเฅเคฒเคจเคพ เคชเคธเคเคฆ เคนเฅเฅค",
|
66 |
-
"เคฎเฅเค เคเคฟเคคเคพเคฌ
|
|
|
|
|
67 |
],
|
68 |
"Sanskrit": [
|
69 |
"เค
เคนเค เคเฅเคฐเคจเฅเคฅเค เคชเค เคพเคฎเคฟเฅค",
|
70 |
"เค
เคฆเฅเคฏ เคธเฅเคฐเฅเคฏเค เคคเฅเคเคธเฅเคตเฅ เค
เคธเฅเคคเคฟเฅค",
|
71 |
-
"เคฎเคฎ เคจเคพเคฎ เคฐเคพเคฎเคเฅค"
|
|
|
|
|
72 |
]
|
73 |
}
|
74 |
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
"
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
print("Loading
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
91 |
|
92 |
# ---------------- HELPERS ---------------- #
|
93 |
def get_random_sentence(language_choice):
|
|
|
94 |
return random.choice(SENTENCE_BANK[language_choice])
|
95 |
|
96 |
def is_script(text, lang_name):
|
|
|
97 |
pattern = SCRIPT_PATTERNS.get(lang_name)
|
98 |
return bool(pattern.search(text)) if pattern else True
|
99 |
|
100 |
def transliterate_to_hk(text, lang_choice):
|
|
|
101 |
mapping = {
|
102 |
"Tamil": sanscript.TAMIL,
|
103 |
-
"Malayalam": sanscript.MALAYALAM,
|
104 |
"Hindi": sanscript.DEVANAGARI,
|
105 |
"Sanskrit": sanscript.DEVANAGARI,
|
106 |
"English": None
|
107 |
}
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
122 |
|
123 |
def highlight_differences(ref, hyp):
|
124 |
-
|
|
|
|
|
|
|
125 |
sm = difflib.SequenceMatcher(None, ref_words, hyp_words)
|
126 |
out_html = []
|
|
|
127 |
for tag, i1, i2, j1, j2 in sm.get_opcodes():
|
128 |
if tag == 'equal':
|
129 |
-
out_html.extend([f"<span style='color:green'>{w}</span>" for w in ref_words[i1:i2]])
|
130 |
elif tag == 'replace':
|
131 |
-
out_html.extend([f"<span style='color:red'>{w}</span>" for w in ref_words[i1:i2]])
|
132 |
-
out_html.extend([f"<span style='color:orange'>{w}</span>" for w in hyp_words[j1:j2]])
|
133 |
elif tag == 'delete':
|
134 |
-
out_html.extend([f"<span style='color:red;text-decoration:line-through'>{w}</span>" for w in ref_words[i1:i2]])
|
135 |
elif tag == 'insert':
|
136 |
-
out_html.extend([f"<span style='color:orange'
|
|
|
137 |
return " ".join(out_html)
|
138 |
|
139 |
def char_level_highlight(ref, hyp):
|
|
|
140 |
sm = difflib.SequenceMatcher(None, list(ref), list(hyp))
|
141 |
out = []
|
|
|
142 |
for tag, i1, i2, j1, j2 in sm.get_opcodes():
|
143 |
if tag == 'equal':
|
144 |
out.extend([f"<span style='color:green'>{c}</span>" for c in ref[i1:i2]])
|
145 |
elif tag in ('replace', 'delete'):
|
146 |
-
out.extend([f"<span style='color:red;text-decoration:underline'>{c}</span>" for c in ref[i1:i2]])
|
147 |
elif tag == 'insert':
|
148 |
-
out.extend([f"<span style='color:orange'>{c}</span>" for c in hyp[j1:j2]])
|
|
|
149 |
return "".join(out)
|
150 |
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
description = VOICE_STYLE.get(lang_choice, "")
|
155 |
-
description_input = parler_tts_tokenizer(description, return_tensors='pt').to(DEVICE)
|
156 |
-
prompt_input = parler_tts_tokenizer(text, return_tensors='pt').to(DEVICE)
|
157 |
-
generation = parler_tts_model.generate(
|
158 |
-
input_ids=description_input.input_ids,
|
159 |
-
attention_mask=description_input.attention_mask,
|
160 |
-
prompt_input_ids=prompt_input.input_ids,
|
161 |
-
prompt_attention_mask=prompt_input.attention_mask
|
162 |
-
)
|
163 |
-
audio_arr = generation.cpu().numpy().squeeze()
|
164 |
-
# Parler-TTS default sample rate is 24000
|
165 |
-
return 24000, audio_arr
|
166 |
-
|
167 |
-
# ---------------- MAIN ---------------- #
|
168 |
-
def compare_pronunciation(audio, language_choice, intended_sentence,
|
169 |
-
pass1_beam, pass1_temp, pass1_condition):
|
170 |
if audio is None or not intended_sentence.strip():
|
171 |
-
return ("No audio or intended sentence.", "", "", "", "", "",
|
172 |
None, None, "", "")
|
173 |
-
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
|
186 |
-
|
187 |
-
|
188 |
-
|
189 |
-
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
-
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
203 |
|
204 |
# ---------------- UI ---------------- #
|
205 |
-
|
206 |
-
gr.
|
207 |
-
|
208 |
-
|
209 |
-
|
210 |
-
|
211 |
-
|
212 |
-
|
213 |
-
|
214 |
-
|
215 |
-
|
216 |
-
|
217 |
-
|
218 |
-
|
219 |
-
|
220 |
-
|
221 |
-
|
222 |
-
|
223 |
-
|
224 |
-
|
225 |
-
|
226 |
-
|
227 |
-
|
228 |
-
|
229 |
-
|
230 |
-
|
231 |
-
|
232 |
-
|
233 |
-
|
234 |
-
|
235 |
-
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
-
|
245 |
-
|
246 |
-
|
247 |
-
|
248 |
-
|
249 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
250 |
if __name__ == "__main__":
|
251 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4 |
import re
|
5 |
import jiwer
|
6 |
import torch
|
7 |
+
import torchaudio
|
8 |
+
import numpy as np
|
9 |
+
from transformers import (
|
10 |
+
AutoProcessor,
|
11 |
+
AutoModelForSpeechSeq2Seq,
|
12 |
+
AutoTokenizer,
|
13 |
+
AutoModel
|
14 |
+
)
|
15 |
+
from TTS.api import TTS
|
16 |
+
import librosa
|
17 |
+
import soundfile as sf
|
18 |
from indic_transliteration import sanscript
|
19 |
from indic_transliteration.sanscript import transliterate
|
20 |
+
import warnings
|
21 |
+
warnings.filterwarnings("ignore")
|
22 |
|
23 |
# ---------------- CONFIG ---------------- #
|
|
|
24 |
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
25 |
|
26 |
LANG_CODES = {
|
27 |
"English": "en",
|
28 |
+
"Tamil": "ta",
|
29 |
"Malayalam": "ml",
|
30 |
"Hindi": "hi",
|
31 |
"Sanskrit": "sa"
|
32 |
}
|
33 |
|
34 |
+
# AI4Bharat model configurations
|
35 |
+
ASR_MODELS = {
|
36 |
+
"English": "openai/whisper-base.en",
|
37 |
+
"Tamil": "ai4bharat/whisper-medium-ta",
|
38 |
+
"Malayalam": "ai4bharat/whisper-medium-ml",
|
39 |
+
"Hindi": "ai4bharat/whisper-medium-hi",
|
40 |
+
"Sanskrit": "ai4bharat/whisper-medium-hi" # Fallback to Hindi for Sanskrit
|
41 |
+
}
|
42 |
+
|
43 |
+
TTS_MODELS = {
|
44 |
+
"English": "tts_models/en/ljspeech/tacotron2-DDC",
|
45 |
+
"Tamil": "tts_models/ta/mai/tacotron2-DDC",
|
46 |
+
"Malayalam": "tts_models/ml/mai/tacotron2-DDC",
|
47 |
+
"Hindi": "tts_models/hi/mai/tacotron2-DDC",
|
48 |
+
"Sanskrit": "tts_models/hi/mai/tacotron2-DDC" # Fallback to Hindi
|
49 |
+
}
|
50 |
+
|
51 |
LANG_PRIMERS = {
|
52 |
+
"English": ("Transcribe in English.",
|
53 |
+
"Write only in English. Example: This is an English sentence."),
|
54 |
+
"Tamil": ("เฎคเฎฎเฎฟเฎดเฎฟเฎฒเฏ เฎเฎดเฏเฎคเฏเฎ.",
|
55 |
+
"เฎคเฎฎเฎฟเฎดเฏ เฎเฎดเฏเฎคเฏเฎคเฏเฎเฏเฎเฎณเฎฟเฎฒเฏ เฎฎเฎเฏเฎเฏเฎฎเฏ เฎเฎดเฏเฎคเฎตเฏเฎฎเฏ. เฎเฎคเฎพเฎฐเฎฃเฎฎเฏ: เฎเฎคเฏ เฎเฎฐเฏ เฎคเฎฎเฎฟเฎดเฏ เฎตเฎพเฎเฏเฎเฎฟเฎฏเฎฎเฏ."),
|
56 |
+
"Malayalam": ("เดฎเดฒเดฏเดพเดณเดคเตเดคเดฟเตฝ เดเดดเตเดคเตเด.",
|
57 |
+
"เดฎเดฒเดฏเดพเดณ เดฒเดฟเดชเดฟเดฏเดฟเตฝ เดฎเดพเดคเตเดฐเด เดเดดเตเดคเตเด. เดเดฆเดพเดนเดฐเดฃเด: เดเดคเตเดฐเต เดฎเดฒเดฏเดพเดณ เดตเดพเดเตเดฏเดฎเดพเดฃเต."),
|
58 |
+
"Hindi": ("เคนเคฟเคเคฆเฅ เคฎเฅเค เคฒเคฟเคเฅเคเฅค",
|
59 |
+
"เคเฅเคตเคฒ เคฆเฅเคตเคจเคพเคเคฐเฅ เคฒเคฟเคชเคฟ เคฎเฅเค เคฒเคฟเคเฅเคเฅค เคเคฆเคพเคนเคฐเคฃ: เคฏเคน เคเค เคนเคฟเคเคฆเฅ เคตเคพเคเฅเคฏ เคนเฅเฅค"),
|
60 |
+
"Sanskrit": ("เคธเคเคธเฅเคเฅเคคเฅ เคฒเคฟเคเคคเฅค",
|
61 |
+
"เคฆเฅเคตเคจเคพเคเคฐเฅ เคฒเคฟเคชเคฟ เคฎเฅเค เคฒเคฟเคเฅเคเฅค เคเคฆเคพเคนเคฐเคฃ: เค
เคนเค เคธเคเคธเฅเคเฅเคคเค เคเคพเคจเคพเคฎเคฟเฅค")
|
62 |
}
|
63 |
|
64 |
SCRIPT_PATTERNS = {
|
65 |
"Tamil": re.compile(r"[เฎ-เฏฟ]"),
|
66 |
+
"Malayalam": re.compile(r"[เด-เตฟ]"),
|
67 |
"Hindi": re.compile(r"[เค-เฅฟ]"),
|
68 |
"Sanskrit": re.compile(r"[เค-เฅฟ]"),
|
69 |
"English": re.compile(r"[A-Za-z]")
|
|
|
71 |
|
72 |
SENTENCE_BANK = {
|
73 |
"English": [
|
74 |
+
"The sun sets over the beautiful horizon.",
|
75 |
+
"Learning new languages opens many doors.",
|
76 |
+
"I enjoy reading books in the evening.",
|
77 |
+
"Technology has changed our daily lives.",
|
78 |
+
"Music brings people together across cultures."
|
79 |
],
|
80 |
"Tamil": [
|
81 |
"เฎเฎฉเฏเฎฑเฏ เฎจเฎฒเฏเฎฒ เฎตเฎพเฎฉเฎฟเฎฒเฏ เฎเฎณเฏเฎณเฎคเฏ.",
|
82 |
+
"เฎจเฎพเฎฉเฏ เฎคเฎฎเฎฟเฎดเฏ เฎเฎฑเฏเฎฑเฏเฎเฏเฎเฏเฎฃเฏเฎเฏ เฎเฎฐเฏเฎเฏเฎเฎฟเฎฑเฏเฎฉเฏ.",
|
83 |
+
"เฎเฎฉเฎเฏเฎเฏ เฎชเฏเฎคเฏเฎคเฎเฎฎเฏ เฎชเฎเฎฟเฎเฏเฎ เฎตเฎฟเฎฐเฏเฎชเฏเฎชเฎฎเฏ.",
|
84 |
+
"เฎคเฎฎเฎฟเฎดเฏ เฎฎเฏเฎดเฎฟ เฎฎเฎฟเฎเฎตเฏเฎฎเฏ เฎ
เฎดเฎเฎพเฎฉเฎคเฏ.",
|
85 |
+
"เฎเฏเฎเฏเฎฎเฏเฎชเฎคเฏเฎคเฏเฎเฎฉเฏ เฎจเฏเฎฐเฎฎเฏ เฎเฏเฎฒเฎตเฎฟเฎเฏเฎตเฎคเฏ เฎฎเฏเฎเฏเฎเฎฟเฎฏเฎฎเฏ."
|
86 |
],
|
87 |
"Malayalam": [
|
88 |
"เดเดจเดฟเดเตเดเต เดฎเดฒเดฏเดพเดณเด เดตเดณเดฐเต เดเดทเตเดเดฎเดพเดฃเต.",
|
89 |
"เดเดจเตเดจเต เดฎเดดเดชเตเดฏเตเดฏเตเดจเตเดจเต.",
|
90 |
+
"เดเดพเตป เดชเตเดธเตเดคเดเด เดตเดพเดฏเดฟเดเตเดเตเดจเตเดจเต.",
|
91 |
+
"เดเตเดฐเดณเดคเตเดคเดฟเดจเตเดฑเต เดชเตเดฐเดเตเดคเดฟ เดธเตเดจเตเดฆเดฐเดฎเดพเดฃเต.",
|
92 |
+
"เดตเดฟเดฆเตเดฏเดพเดญเตเดฏเดพเดธเด เดเตเดตเดฟเดคเดคเตเดคเดฟเตฝ เดชเตเดฐเดงเดพเดจเดฎเดพเดฃเต."
|
93 |
],
|
94 |
"Hindi": [
|
95 |
+
"เคเค เคฎเฅเคธเคฎ เคฌเคนเฅเคค เค
เคเฅเคเคพ เคนเฅเฅค",
|
96 |
+
"เคฎเฅเคเฅ เคนเคฟเคเคฆเฅ เคฌเฅเคฒเคจเคพ เคชเคธเคเคฆ เคนเฅเฅค",
|
97 |
+
"เคฎเฅเค เคฐเฅเค เคเคฟเคคเคพเคฌ เคชเคขเคผเคคเคพ เคนเฅเคเฅค",
|
98 |
+
"เคญเคพเคฐเคค เคเฅ เคธเคเคธเฅเคเฅเคคเคฟ เคตเคฟเคตเคฟเคงเคคเคพเคชเฅเคฐเฅเคฃ เคนเฅเฅค",
|
99 |
+
"เคถเคฟเคเฅเคทเคพ เคนเคฎเคพเคฐเฅ เคญเคตเคฟเคทเฅเคฏ เคเฅ เคเฅเคเคเฅ เคนเฅเฅค"
|
100 |
],
|
101 |
"Sanskrit": [
|
102 |
"เค
เคนเค เคเฅเคฐเคจเฅเคฅเค เคชเค เคพเคฎเคฟเฅค",
|
103 |
"เค
เคฆเฅเคฏ เคธเฅเคฐเฅเคฏเค เคคเฅเคเคธเฅเคตเฅ เค
เคธเฅเคคเคฟเฅค",
|
104 |
+
"เคฎเคฎ เคจเคพเคฎ เคฐเคพเคฎเคเฅค",
|
105 |
+
"เคตเคฟเคฆเฅเคฏเคพ เคธเคฐเฅเคตเคคเฅเคฐ เคชเฅเคเฅเคฏเคคเฅเฅค",
|
106 |
+
"เคธเคคเฅเคฏเคฎเฅเคต เคเคฏเคคเฅเฅค"
|
107 |
]
|
108 |
}
|
109 |
|
110 |
+
# ---------------- MODEL CACHE ---------------- #
|
111 |
+
asr_models = {}
|
112 |
+
tts_models = {}
|
113 |
+
|
114 |
+
def load_asr_model(language):
|
115 |
+
"""Load ASR model for specific language"""
|
116 |
+
if language not in asr_models:
|
117 |
+
try:
|
118 |
+
model_name = ASR_MODELS[language]
|
119 |
+
print(f"Loading ASR model for {language}: {model_name}")
|
120 |
+
|
121 |
+
processor = AutoProcessor.from_pretrained(model_name)
|
122 |
+
model = AutoModelForSpeechSeq2Seq.from_pretrained(model_name).to(DEVICE)
|
123 |
+
|
124 |
+
asr_models[language] = {"processor": processor, "model": model}
|
125 |
+
print(f"โ
ASR model loaded for {language}")
|
126 |
+
except Exception as e:
|
127 |
+
print(f"โ Failed to load ASR for {language}: {e}")
|
128 |
+
# Fallback to English model
|
129 |
+
if language != "English":
|
130 |
+
print(f"๐ Falling back to English ASR for {language}")
|
131 |
+
load_asr_model("English")
|
132 |
+
asr_models[language] = asr_models["English"]
|
133 |
+
|
134 |
+
return asr_models[language]
|
135 |
+
|
136 |
+
def load_tts_model(language):
|
137 |
+
"""Load TTS model for specific language"""
|
138 |
+
if language not in tts_models:
|
139 |
+
try:
|
140 |
+
model_name = TTS_MODELS[language]
|
141 |
+
print(f"Loading TTS model for {language}: {model_name}")
|
142 |
+
|
143 |
+
tts = TTS(model_name=model_name).to(DEVICE)
|
144 |
+
tts_models[language] = tts
|
145 |
+
print(f"โ
TTS model loaded for {language}")
|
146 |
+
except Exception as e:
|
147 |
+
print(f"โ Failed to load TTS for {language}: {e}")
|
148 |
+
# Fallback to English
|
149 |
+
if language != "English":
|
150 |
+
print(f"๐ Falling back to English TTS for {language}")
|
151 |
+
load_tts_model("English")
|
152 |
+
tts_models[language] = tts_models["English"]
|
153 |
+
|
154 |
+
return tts_models[language]
|
155 |
|
156 |
# ---------------- HELPERS ---------------- #
|
157 |
def get_random_sentence(language_choice):
|
158 |
+
"""Get random sentence for practice"""
|
159 |
return random.choice(SENTENCE_BANK[language_choice])
|
160 |
|
161 |
def is_script(text, lang_name):
|
162 |
+
"""Check if text is in expected script"""
|
163 |
pattern = SCRIPT_PATTERNS.get(lang_name)
|
164 |
return bool(pattern.search(text)) if pattern else True
|
165 |
|
166 |
def transliterate_to_hk(text, lang_choice):
|
167 |
+
"""Transliterate Indic text to Harvard-Kyoto"""
|
168 |
mapping = {
|
169 |
"Tamil": sanscript.TAMIL,
|
170 |
+
"Malayalam": sanscript.MALAYALAM,
|
171 |
"Hindi": sanscript.DEVANAGARI,
|
172 |
"Sanskrit": sanscript.DEVANAGARI,
|
173 |
"English": None
|
174 |
}
|
175 |
+
|
176 |
+
script = mapping.get(lang_choice)
|
177 |
+
if script and is_script(text, lang_choice):
|
178 |
+
try:
|
179 |
+
return transliterate(text, script, sanscript.HK)
|
180 |
+
except:
|
181 |
+
return text
|
182 |
+
return text
|
183 |
+
|
184 |
+
def preprocess_audio(audio_path, target_sr=16000):
|
185 |
+
"""Preprocess audio for ASR"""
|
186 |
+
try:
|
187 |
+
# Load audio
|
188 |
+
audio, sr = librosa.load(audio_path, sr=target_sr)
|
189 |
+
|
190 |
+
# Normalize audio
|
191 |
+
audio = audio / np.max(np.abs(audio))
|
192 |
+
|
193 |
+
# Remove silence
|
194 |
+
audio, _ = librosa.effects.trim(audio, top_db=20)
|
195 |
+
|
196 |
+
return audio, target_sr
|
197 |
+
except Exception as e:
|
198 |
+
print(f"Audio preprocessing error: {e}")
|
199 |
+
return None, None
|
200 |
+
|
201 |
+
def transcribe_with_ai4bharat(audio_path, language, initial_prompt=""):
|
202 |
+
"""Transcribe audio using AI4Bharat models"""
|
203 |
+
try:
|
204 |
+
# Load model
|
205 |
+
asr_components = load_asr_model(language)
|
206 |
+
processor = asr_components["processor"]
|
207 |
+
model = asr_components["model"]
|
208 |
+
|
209 |
+
# Preprocess audio
|
210 |
+
audio, sr = preprocess_audio(audio_path)
|
211 |
+
if audio is None:
|
212 |
+
return "Error: Could not process audio"
|
213 |
+
|
214 |
+
# Prepare inputs
|
215 |
+
inputs = processor(audio, sampling_rate=sr, return_tensors="pt")
|
216 |
+
inputs = {k: v.to(DEVICE) for k, v in inputs.items()}
|
217 |
+
|
218 |
+
# Generate transcription
|
219 |
+
with torch.no_grad():
|
220 |
+
predicted_ids = model.generate(**inputs, max_length=200)
|
221 |
+
|
222 |
+
# Decode
|
223 |
+
transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)[0]
|
224 |
+
|
225 |
+
return transcription.strip()
|
226 |
+
|
227 |
+
except Exception as e:
|
228 |
+
print(f"Transcription error for {language}: {e}")
|
229 |
+
return f"Error: Transcription failed - {str(e)}"
|
230 |
+
|
231 |
+
def synthesize_with_ai4bharat(text, language):
|
232 |
+
"""Synthesize speech using AI4Bharat TTS"""
|
233 |
+
if not text.strip():
|
234 |
+
return None
|
235 |
+
|
236 |
+
try:
|
237 |
+
# Load TTS model
|
238 |
+
tts = load_tts_model(language)
|
239 |
+
|
240 |
+
# Generate audio
|
241 |
+
audio_path = f"/tmp/tts_output_{hash(text)}.wav"
|
242 |
+
tts.tts_to_file(text=text, file_path=audio_path)
|
243 |
+
|
244 |
+
# Load generated audio
|
245 |
+
audio, sr = librosa.load(audio_path, sr=22050)
|
246 |
+
|
247 |
+
return sr, audio
|
248 |
+
|
249 |
+
except Exception as e:
|
250 |
+
print(f"TTS error for {language}: {e}")
|
251 |
+
return None
|
252 |
|
253 |
def highlight_differences(ref, hyp):
|
254 |
+
"""Highlight word-level differences"""
|
255 |
+
ref_words = ref.strip().split()
|
256 |
+
hyp_words = hyp.strip().split()
|
257 |
+
|
258 |
sm = difflib.SequenceMatcher(None, ref_words, hyp_words)
|
259 |
out_html = []
|
260 |
+
|
261 |
for tag, i1, i2, j1, j2 in sm.get_opcodes():
|
262 |
if tag == 'equal':
|
263 |
+
out_html.extend([f"<span style='color:green; font-weight:bold'>{w}</span>" for w in ref_words[i1:i2]])
|
264 |
elif tag == 'replace':
|
265 |
+
out_html.extend([f"<span style='color:red; text-decoration:line-through'>{w}</span>" for w in ref_words[i1:i2]])
|
266 |
+
out_html.extend([f"<span style='color:orange; font-weight:bold'> โ {w}</span>" for w in hyp_words[j1:j2]])
|
267 |
elif tag == 'delete':
|
268 |
+
out_html.extend([f"<span style='color:red; text-decoration:line-through'>{w}</span>" for w in ref_words[i1:i2]])
|
269 |
elif tag == 'insert':
|
270 |
+
out_html.extend([f"<span style='color:orange; font-weight:bold'>+{w}</span>" for w in hyp_words[j1:j2]])
|
271 |
+
|
272 |
return " ".join(out_html)
|
273 |
|
274 |
def char_level_highlight(ref, hyp):
|
275 |
+
"""Highlight character-level differences"""
|
276 |
sm = difflib.SequenceMatcher(None, list(ref), list(hyp))
|
277 |
out = []
|
278 |
+
|
279 |
for tag, i1, i2, j1, j2 in sm.get_opcodes():
|
280 |
if tag == 'equal':
|
281 |
out.extend([f"<span style='color:green'>{c}</span>" for c in ref[i1:i2]])
|
282 |
elif tag in ('replace', 'delete'):
|
283 |
+
out.extend([f"<span style='color:red; text-decoration:underline; font-weight:bold'>{c}</span>" for c in ref[i1:i2]])
|
284 |
elif tag == 'insert':
|
285 |
+
out.extend([f"<span style='color:orange; background-color:yellow'>{c}</span>" for c in hyp[j1:j2]])
|
286 |
+
|
287 |
return "".join(out)
|
288 |
|
289 |
+
# ---------------- MAIN FUNCTION ---------------- #
|
290 |
+
def compare_pronunciation(audio, language_choice, intended_sentence):
|
291 |
+
"""Main function to compare pronunciation"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
292 |
if audio is None or not intended_sentence.strip():
|
293 |
+
return ("โ No audio or intended sentence provided.", "", "", "", "", "",
|
294 |
None, None, "", "")
|
295 |
+
|
296 |
+
try:
|
297 |
+
print(f"Processing audio for {language_choice}")
|
298 |
+
|
299 |
+
# Pass 1: Raw transcription
|
300 |
+
primer_weak, _ = LANG_PRIMERS[language_choice]
|
301 |
+
actual_text = transcribe_with_ai4bharat(audio, language_choice, primer_weak)
|
302 |
+
|
303 |
+
# Pass 2: Target-biased transcription
|
304 |
+
_, primer_strong = LANG_PRIMERS[language_choice]
|
305 |
+
strict_prompt = f"{primer_strong}\nTarget: {intended_sentence}"
|
306 |
+
corrected_text = transcribe_with_ai4bharat(audio, language_choice, strict_prompt)
|
307 |
+
|
308 |
+
# Error metrics
|
309 |
+
try:
|
310 |
+
wer_val = jiwer.wer(intended_sentence, actual_text)
|
311 |
+
cer_val = jiwer.cer(intended_sentence, actual_text)
|
312 |
+
except:
|
313 |
+
wer_val, cer_val = 1.0, 1.0
|
314 |
+
|
315 |
+
# Transliteration
|
316 |
+
hk_translit = transliterate_to_hk(actual_text, language_choice)
|
317 |
+
if not is_script(actual_text, language_choice):
|
318 |
+
hk_translit = f"โ ๏ธ Script mismatch: expected {language_choice} script"
|
319 |
+
|
320 |
+
# Visual feedback
|
321 |
+
diff_html = highlight_differences(intended_sentence, actual_text)
|
322 |
+
char_html = char_level_highlight(intended_sentence, actual_text)
|
323 |
+
|
324 |
+
# TTS synthesis
|
325 |
+
tts_intended = synthesize_with_ai4bharat(intended_sentence, language_choice)
|
326 |
+
tts_actual = synthesize_with_ai4bharat(actual_text, language_choice)
|
327 |
+
|
328 |
+
# Status message
|
329 |
+
status = f"โ
Analysis complete for {language_choice}"
|
330 |
+
if wer_val < 0.1:
|
331 |
+
status += " - Excellent pronunciation! ๐"
|
332 |
+
elif wer_val < 0.3:
|
333 |
+
status += " - Good pronunciation! ๐"
|
334 |
+
elif wer_val < 0.5:
|
335 |
+
status += " - Needs improvement ๐"
|
336 |
+
else:
|
337 |
+
status += " - Keep practicing! ๐ช"
|
338 |
+
|
339 |
+
return (
|
340 |
+
status,
|
341 |
+
actual_text,
|
342 |
+
corrected_text,
|
343 |
+
hk_translit,
|
344 |
+
f"{wer_val:.3f}",
|
345 |
+
f"{cer_val:.3f}",
|
346 |
+
diff_html,
|
347 |
+
tts_intended,
|
348 |
+
tts_actual,
|
349 |
+
char_html,
|
350 |
+
intended_sentence
|
351 |
+
)
|
352 |
+
|
353 |
+
except Exception as e:
|
354 |
+
error_msg = f"โ Error during analysis: {str(e)}"
|
355 |
+
print(error_msg)
|
356 |
+
return (error_msg, "", "", "", "", "", None, None, "", "")
|
357 |
|
358 |
# ---------------- UI ---------------- #
|
359 |
+
def create_interface():
|
360 |
+
with gr.Blocks(title="๐๏ธ AI4Bharat Pronunciation Trainer", theme=gr.themes.Soft()) as demo:
|
361 |
+
gr.Markdown("""
|
362 |
+
# ๐๏ธ AI4Bharat Pronunciation Trainer
|
363 |
+
|
364 |
+
Practice pronunciation in **Tamil, Malayalam, Hindi, Sanskrit & English** using state-of-the-art AI4Bharat models!
|
365 |
+
|
366 |
+
๐ **How to use:**
|
367 |
+
1. Select your target language
|
368 |
+
2. Generate a practice sentence
|
369 |
+
3. Record yourself reading it aloud
|
370 |
+
4. Get detailed feedback with error analysis
|
371 |
+
""")
|
372 |
+
|
373 |
+
with gr.Row():
|
374 |
+
with gr.Column(scale=2):
|
375 |
+
lang_choice = gr.Dropdown(
|
376 |
+
choices=list(LANG_CODES.keys()),
|
377 |
+
value="Tamil",
|
378 |
+
label="๐ Select Language"
|
379 |
+
)
|
380 |
+
with gr.Column(scale=1):
|
381 |
+
gen_btn = gr.Button("๐ฒ Generate Practice Sentence", variant="primary")
|
382 |
+
|
383 |
+
intended_display = gr.Textbox(
|
384 |
+
label="๐ Practice Sentence (Read this aloud)",
|
385 |
+
placeholder="Click 'Generate Practice Sentence' to get started...",
|
386 |
+
interactive=False,
|
387 |
+
lines=2
|
388 |
+
)
|
389 |
+
|
390 |
+
with gr.Row():
|
391 |
+
audio_input = gr.Audio(
|
392 |
+
sources=["microphone", "upload"],
|
393 |
+
type="filepath",
|
394 |
+
label="๐ค Record Your Pronunciation"
|
395 |
+
)
|
396 |
+
|
397 |
+
analyze_btn = gr.Button("๐ Analyze Pronunciation", variant="primary", size="lg")
|
398 |
+
|
399 |
+
status_output = gr.Textbox(label="๐ Analysis Status", interactive=False)
|
400 |
+
|
401 |
+
with gr.Row():
|
402 |
+
with gr.Column():
|
403 |
+
pass1_out = gr.Textbox(label="๐ฏ What You Actually Said", interactive=False)
|
404 |
+
wer_out = gr.Textbox(label="๐ Word Error Rate (lower = better)", interactive=False)
|
405 |
+
|
406 |
+
with gr.Column():
|
407 |
+
pass2_out = gr.Textbox(label="๐ง Target-Biased Output", interactive=False)
|
408 |
+
cer_out = gr.Textbox(label="๐ Character Error Rate (lower = better)", interactive=False)
|
409 |
+
|
410 |
+
hk_out = gr.Textbox(label="๐ค Romanization (Harvard-Kyoto)", interactive=False)
|
411 |
+
|
412 |
+
with gr.Accordion("๐ Detailed Feedback", open=True):
|
413 |
+
diff_html_box = gr.HTML(label="๐ Word-Level Differences")
|
414 |
+
char_html_box = gr.HTML(label="๐ค Character-Level Analysis")
|
415 |
+
|
416 |
+
with gr.Row():
|
417 |
+
intended_tts_audio = gr.Audio(label="๐ Reference Pronunciation", type="numpy")
|
418 |
+
actual_tts_audio = gr.Audio(label="๐ Your Pronunciation (TTS)", type="numpy")
|
419 |
+
|
420 |
+
gr.Markdown("""
|
421 |
+
### ๐จ Color Guide:
|
422 |
+
- ๐ข **Green**: Correctly pronounced
|
423 |
+
- ๐ด **Red**: Missing or incorrect words
|
424 |
+
- ๐ **Orange**: Extra or substituted words
|
425 |
+
- ๐ก **Yellow background**: Inserted characters
|
426 |
+
""")
|
427 |
+
|
428 |
+
# Event handlers
|
429 |
+
gen_btn.click(
|
430 |
+
fn=get_random_sentence,
|
431 |
+
inputs=[lang_choice],
|
432 |
+
outputs=[intended_display]
|
433 |
+
)
|
434 |
+
|
435 |
+
analyze_btn.click(
|
436 |
+
fn=compare_pronunciation,
|
437 |
+
inputs=[audio_input, lang_choice, intended_display],
|
438 |
+
outputs=[
|
439 |
+
status_output, pass1_out, pass2_out, hk_out,
|
440 |
+
wer_out, cer_out, diff_html_box,
|
441 |
+
intended_tts_audio, actual_tts_audio,
|
442 |
+
char_html_box, intended_display
|
443 |
+
]
|
444 |
+
)
|
445 |
+
|
446 |
+
# Auto-generate sentence on language change
|
447 |
+
lang_choice.change(
|
448 |
+
fn=get_random_sentence,
|
449 |
+
inputs=[lang_choice],
|
450 |
+
outputs=[intended_display]
|
451 |
+
)
|
452 |
+
|
453 |
+
return demo
|
454 |
+
|
455 |
+
# ---------------- LAUNCH ---------------- #
|
456 |
if __name__ == "__main__":
|
457 |
+
print("๐ Starting AI4Bharat Pronunciation Trainer...")
|
458 |
+
|
459 |
+
# Pre-load English models for faster startup
|
460 |
+
print("๐ฆ Pre-loading English models...")
|
461 |
+
try:
|
462 |
+
load_asr_model("English")
|
463 |
+
load_tts_model("English")
|
464 |
+
print("โ
English models loaded successfully")
|
465 |
+
except Exception as e:
|
466 |
+
print(f"โ ๏ธ Warning: Could not pre-load English models: {e}")
|
467 |
+
|
468 |
+
demo = create_interface()
|
469 |
+
demo.launch(
|
470 |
+
share=True,
|
471 |
+
show_error=True,
|
472 |
+
server_name="0.0.0.0",
|
473 |
+
server_port=7860
|
474 |
+
)
|