new fuzzy method

app.py

@@ -6,19 +6,26 @@ from sentence_transformers import SentenceTransformer
 from qdrant_client import QdrantClient
 from qdrant_client.models import Filter, FieldCondition, MatchValue
 import os
+from rapidfuzz import process, fuzz
 from pythainlp.tokenize import word_tokenize
 from pyairtable import Table
 from pyairtable import Api
+import pickle
+import re
 import unicodedata
 
 
 qdrant_client = QdrantClient(
-    url=os.environ.get("Qdrant_url"),
-    api_key=os.environ.get("Qdrant_api"),
+    #url=os.environ.get("Qdrant_url"),
+    #api_key=os.environ.get("Qdrant_api"),
+    url=userdata.get("Qdrant_url"),
+    api_key=userdata.get("Qdrant_api"),
 )
 
-AIRTABLE_API_KEY = os.environ.get("airtable_api")
-BASE_ID = os.environ.get("airtable_baseid")
+#AIRTABLE_API_KEY = os.environ.get("airtable_api")
+#BASE_ID = os.environ.get("airtable_baseid")
+AIRTABLE_API_KEY = "patwFFErs6fJ8fBhP.15dd01f7be728762fbbb03fdec6284ec01ee0ab84c0abd2ec17374cdaa63500e"
+BASE_ID = "app2OU6L8LK4JLV3M"  # จาก URL ของคุณ
 TABLE_NAME = "Feedback_search"  # หรือเปลี่ยนชื่อให้ชัดเช่น 'Feedback'
 api = Api(AIRTABLE_API_KEY)
 table = api.table(BASE_ID, TABLE_NAME)
@@ -49,10 +56,45 @@ model_config = {
 # Global memory to hold feedback state
 latest_query_result = {"query": "", "result": "", "model": "", "raw_query": "", "time": ""}
 
-def fix_common_thai_typos(text: str) -> str:
-    text = unicodedata.normalize("NFC", text)  # normalize ตัวอักษรซ้อน
-    text = text.replace("เเ", "แ").replace("เแ", "แ")  # แก้เฉพาะเอกลักษณ์ผิด
-    return text
+with open("/content/drive/MyDrive/mypinmall/data/keyword_whitelist.pkl", "rb") as f:
+    keyword_whitelist = pickle.load(f)
+#with open("keyword_whitelist.pkl", "rb") as f:
+#    keyword_whitelist = pickle.load(f)
+
+def normalize(text: str) -> str:
+    text = unicodedata.normalize("NFC", text)
+    text = text.replace("เแ", "แ").replace("เเ", "แ")
+    return text.strip().lower()
+
+def smart_tokenize(text: str) -> list:
+    tokens = word_tokenize(text.strip(), engine="newmm")
+    if not tokens or len("".join(tokens)) < len(text.strip()) * 0.5:
+        return [text.strip()]
+    return tokens
+
+def correct_query_merge_phrases(query: str, whitelist, threshold=75, max_ngram=3):
+    query_norm = normalize(query)
+    tokens = smart_tokenize(query_norm)
+    corrected = []
+    i = 0
+
+    while i < len(tokens):
+        matched = False
+        for n in range(min(max_ngram, len(tokens) - i), 0, -1):
+            phrase = "".join(tokens[i:i+n])
+            match, score, _ = process.extractOne(phrase, whitelist, scorer=fuzz.ratio)
+            if score >= threshold:
+                corrected.append(match)
+                i += n
+                matched = True
+                break
+        if not matched:
+            corrected.append(tokens[i])
+            i += 1
+
+    # ✅ ตัดคำที่มีความยาว 1 ตัวอักษรและไม่ได้อยู่ใน whitelist
+    cleaned = [word for word in corrected if len(word) > 1 or word in whitelist]
+    return " ".join(cleaned)
 
 # 🌟 Main search function
 def search_product(query, model_name):
@@ -61,7 +103,7 @@ def search_product(query, model_name):
         return "<p>❌ ไม่พบโมเดล</p>"
 
     latest_query_result["raw_query"] = query
-    corrected_query = fix_common_thai_typos(query)
+    corrected_query = correct_query_merge_phrases(query,keyword_whitelist)
 
     query_embed = model_config[model_name]["func"](corrected_query)
     collection_name = model_config[model_name]["collection"]