change model

config.py

@@ -19,3 +19,5 @@ SENTENCE_EMBEDDING_FILE = os.path.join(
     DATA_DIR,
     "anchor_name_sentence_sentence_embeddings(cl-nagoya-sup-simcse-ja-for-standard-name-v0_9_10).pkl",
 )
+MODEL_TYPE = "openvino"
+DEVICE_TYPE = "cpu"

routes/predict.py

@@ -8,6 +8,7 @@ from auth import get_current_user
 from services.sentence_transformer_service import SentenceTransformerService, sentence_transformer_service
 from data_lib.input_name_data import InputNameData
 from data_lib.base_name_data import COL_NAME_SENTENCE
+from mapping_lib.subject_mapper import SubjectMapper
 from mapping_lib.name_mapper import NameMapper
 from config import UPLOAD_DIR, OUTPUT_DIR
 from models import (
@@ -48,9 +49,16 @@ async def predict(
         # Process input data
         start_time = time.time()
         try:
-            inputData = InputNameData(sentence_service.dic_standard_subject)
+            inputData = InputNameData()
             inputData.load_data_from_csv(input_file_path)
-            inputData.process_data(sentence_service.sentenceTransformerHelper)
+        except Exception as e:
+            print(f"Error processing input data: {e}")
+            raise HTTPException(status_code=500, detail=str(e))
+        try:
+            subject_mapper = SubjectMapper(sentence_service.sentenceTransformerHelper, sentence_service.dic_standard_subject)
+            dic_subject_map = subject_mapper.map_standard_subjects(inputData.dataframe)
+            inputData.dic_standard_subject = dic_subject_map
+            inputData.process_data()
         except Exception as e:
             print(f"Error processing input data: {e}")
             raise HTTPException(status_code=500, detail=str(e))
@@ -107,9 +115,13 @@ async def create_embeddings(
     Create embeddings for a list of input sentences (requires authentication)
     """
     try:
+        start_time = time.time()
         embeddings = sentence_service.sentenceTransformerHelper.create_embeddings(
             request.sentences
         )
+        end_time = time.time()
+        execution_time = end_time - start_time
+        print(f"Execution time: {execution_time} seconds")
         # Convert numpy array to list for JSON serialization
         embeddings_list = embeddings.tolist()
         return {"embeddings": embeddings_list}

services/sentence_transformer_service.py

@@ -1,6 +1,22 @@
 import pickle
+import warnings
+
+# Filter NumPy array implementation warnings specifically
+warnings.filterwarnings(
+    "ignore", 
+    message=".*array.*implementation doesn't accept a copy keyword.*", 
+    category=DeprecationWarning
+)
+
+# Or alternatively, target the exact warning message:
+warnings.filterwarnings(
+    "ignore", 
+    message=".*NumPy will pass `copy` to the `__array__` special method.*", 
+    category=DeprecationWarning
+)
+
 from config import (
-    MODEL_NAME, 
+    MODEL_NAME, MODEL_TYPE, DEVICE_TYPE,
     SENTENCE_EMBEDDING_FILE,  
     STANDARD_NAME_MAP_DATA_FILE, SUBJECT_DATA_FILE
 )
@@ -24,10 +40,8 @@ class SentenceTransformerService:
 
         print("Loading models and data...")
         # Load sentence transformer model
-        self.sentenceTransformerHelper = SentenceTransformerHelper(
-            convert_to_zenkaku_flag=True, replace_words=None, keywords=None
-        )
-        self.sentenceTransformerHelper.load_model_by_name(MODEL_NAME)
+        self.sentenceTransformerHelper = SentenceTransformerHelper(model_name=MODEL_NAME, model_type=MODEL_TYPE)
+        print(f"Loading model {MODEL_NAME} with type {MODEL_TYPE}") 
 
         # Load standard subject dictionary
         self.dic_standard_subject = SubjectData.create_standard_subject_dic_from_file(SUBJECT_DATA_FILE)
@@ -37,10 +51,10 @@ class SentenceTransformerService:
             self.anchor_name_sentence_embeddings = pickle.load(f)
 
         # Load and process sample data
-        self.standardNameMapData = StandardNameMapData()
+        self.standardNameMapData = StandardNameMapData(self.anchor_name_sentence_embeddings)
         self.standardNameMapData.load_data_from_csv(STANDARD_NAME_MAP_DATA_FILE)
-        self.standardNameMapData.process_data(self.anchor_name_sentence_embeddings)
-        
+        self.standardNameMapData.process_data()
+
         print("Models and data loaded successfully")
 
 # Global instance (singleton)