improve suggestion cleanning

app.py

@@ -81,25 +81,39 @@ def suggest_next(text, model_name, k, m, num_beam_groups, diversity_penalty):
     }
     if diversity_penalty and diversity_penalty > 0:
         gen_kwargs["num_beam_groups"] = num_beam_groups
-        gen_kwargs["diversity_penalty"] = float(diversity_penalty)
+        gen_kwargs["diversity_penalty"] = diversity_penalty
 
     outs = gen_pipe(text, **gen_kwargs)
 
-    # 提取純下文、過濾空字串、繁體化
-    suggestions = [
-        cc.convert(out["generated_text"][len(text):].strip())
-        for out in outs
-        if out["generated_text"][len(text):].strip()
-    ]
+    # 提取純下文、過濾空字串、繁體化、確保 strip 處理
+    raw_suggestions = []
+    for out in outs:
+        snippet = out["generated_text"][len(text):].strip()
+        if not snippet:
+            continue
+        converted = cc.convert(snippet).strip()
+        raw_suggestions.append(converted)
 
-    # 去重
+    # 去重 (基於 strip 後內容)
     unique_suggestions = []
-    for s in suggestions:
-        if s not in unique_suggestions:
-            unique_suggestions.append(s)
+    seen = set()
+    for s in raw_suggestions:
+        key = s
+        if key not in seen:
+            seen.add(key)
+            unique_suggestions.append(key)
 
     # 合併共同前綴
-    final_suggestions = merge_common_prefixes(unique_suggestions, min_len=2)
+    merged_prefixes = merge_common_prefixes(unique_suggestions, min_len=2)
+
+    # 最終去重並移除空項 (基於 strip 後內容)
+    final_suggestions = []
+    seen_final = set()
+    for s in merged_prefixes:
+        key = s.strip()
+        if key and key not in seen_final:
+            seen_final.add(key)
+            final_suggestions.append(key)
 
     return update(choices=final_suggestions, value=None)