change service

config.py

@@ -12,6 +12,7 @@ UPLOAD_DIR = os.path.join(BASE_DIR, "uploads")
 OUTPUT_DIR = os.path.join(BASE_DIR, "outputs")
 SUBJECT_DATA_FILE = os.path.join(DATA_DIR, "subjectData.csv")
 SAMPLE_DATA_FILE = os.path.join(DATA_DIR, "sampleData.csv")
+STANDARD_NAME_MAP_DATA_FILE = os.path.join(DATA_DIR, "standardNameMapData.csv")
 # Model Names
 MODEL_NAME = "Detomo/cl-nagoya-sup-simcse-ja-for-standard-name-v0_9_10"
 SENTENCE_EMBEDDING_FILE = os.path.join(

data/anchor_name_sentence_sentence_embeddings(cl-nagoya-sup-simcse-ja-for-standard-name-v0_9_10).pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:062effa64e2fdd7b5b7253b596d777dea75a892b0b011592657ea51c3861ce62
-size 21565604
+oid sha256:e25d41cf2b9ab5b90f2c0e7e0f5d0ec31499f7dcb252de64d7af20ab63e91750
+size 12073124

routes/predict.py

@@ -8,7 +8,7 @@ from auth import get_current_user
 from services.sentence_transformer_service import SentenceTransformerService, sentence_transformer_service
 from data_lib.input_name_data import InputNameData
 from data_lib.base_name_data import COL_NAME_SENTENCE
-from mapping_lib.name_mapping_helper import NameMappingHelper
+from mapping_lib.name_mapper import NameMapper
 from config import UPLOAD_DIR, OUTPUT_DIR
 
 router = APIRouter()
@@ -39,36 +39,32 @@ async def predict(
     try:
         # Process input data
         start_time = time.time()
-        inputData = InputNameData(sentence_service.dic_standard_subject)
-        inputData.load_data_from_csv(input_file_path)
-        inputData.process_data(sentence_service.sentenceTransformerHelper)
-        input_name_sentences = inputData.dataframe[COL_NAME_SENTENCE]
-        input_name_sentence_embeddings = sentence_service.sentenceTransformerHelper.create_embeddings(input_name_sentences)
-        
-        # Create similarity matrix
-        similarity_matrix = sentence_service.sentenceTransformerHelper.create_similarity_matrix_from_embeddings(
-            sentence_service.sample_name_sentence_embeddings,
-            input_name_sentence_embeddings
-        )
-
+        try:
+            inputData = InputNameData(sentence_service.dic_standard_subject)
+            inputData.load_data_from_csv(input_file_path)
+            inputData.process_data(sentence_service.sentenceTransformerHelper)
+        except Exception as e:
+            print(f"Error processing input data: {e}")
+            raise HTTPException(status_code=500, detail=str(e))
         # Map standard names
-        nameMappingHelper = NameMappingHelper(
-            sentence_service.sentenceTransformerHelper,
-            inputData,
-            sentence_service.sampleData,
-            input_name_sentence_embeddings,
-            sentence_service.sample_name_sentence_embeddings,
-            similarity_matrix,
-        )
-        df_predicted = nameMappingHelper.map_standard_names()
+        try:
+            nameMapper = NameMapper(
+                sentence_service.sentenceTransformerHelper,
+                sentence_service.standardNameMapData,
+                top_count=3
+            )
+            df_predicted = nameMapper.predict(inputData)
+        except Exception as e:
+            print(f"Error mapping standard names: {e}")
+            raise HTTPException(status_code=500, detail=str(e))
 
         # Create output dataframe and save to CSV
         column_to_keep = ['ファイル名', 'シート名', '行', '科目', '中科目', '分類', '名称', '摘要', '備考']
         output_df = inputData.dataframe[column_to_keep].copy()
         output_df.reset_index(drop=False, inplace=True)
-        output_df.loc[:, "出力_科目"] = df_predicted["出力_科目"]
-        output_df.loc[:, "出力_項目名"] = df_predicted["出力_項目名"]
-        output_df.loc[:, "出力_確率度"] = df_predicted["出力_確率度"]
+        output_df.loc[:, "出力_科目"] = df_predicted["標準科目"]
+        output_df.loc[:, "出力_項目名"] = df_predicted["標準項目名"]
+        output_df.loc[:, "出力_確率度"] = df_predicted["基準名称類似度"]
 
         # Save with utf_8_sig encoding for Japanese Excel compatibility
         output_df.to_csv(output_file_path, index=False, encoding="utf_8_sig")

services/sentence_transformer_service.py

@@ -2,18 +2,18 @@ import pickle
 from config import (
     MODEL_NAME, 
     SENTENCE_EMBEDDING_FILE,  
-    SAMPLE_DATA_FILE, SUBJECT_DATA_FILE
+    STANDARD_NAME_MAP_DATA_FILE, SUBJECT_DATA_FILE
 )
 from sentence_transformer_lib.sentence_transformer_helper import SentenceTransformerHelper
 from data_lib.subject_data import SubjectData
-from data_lib.sample_name_data import SampleNameData
+from data_lib.standard_name_map_data import StandardNameMapData
 
 
 class SentenceTransformerService:
     def __init__(self):
         self.sentenceTransformerHelper = None
         self.dic_standard_subject = None
-        self.sample_name_sentence_embeddings = None
+        self.anchor_name_sentence_embeddings = None
         self.sampleData = None
 
     def load_model_data(self):
@@ -34,14 +34,13 @@ class SentenceTransformerService:
 
         # Load pre-computed embeddings and similarities
         with open(SENTENCE_EMBEDDING_FILE, "rb") as f:
-            self.sample_name_sentence_embeddings = pickle.load(f)
+            self.anchor_name_sentence_embeddings = pickle.load(f)
 
         # Load and process sample data
-        self.sampleData = SampleNameData()
-        self.sampleData.load_data_from_csv(SAMPLE_DATA_FILE)
-        self.sampleData.process_data()
-        self.sampleData.build_search_tree()
-
+        self.standardNameMapData = StandardNameMapData()
+        self.standardNameMapData.load_data_from_csv(STANDARD_NAME_MAP_DATA_FILE)
+        self.standardNameMapData.process_data(self.anchor_name_sentence_embeddings)
+        
         print("Models and data loaded successfully")
 
 # Global instance (singleton)