update

huggingface/dataset_utils.py

@@ -24,5 +24,5 @@ def get_global_data_path():
     if not hf_token or not hf_dataset:
         return default_path
 
-    folder = huggingface_hub.snapshot_download(repo_id=hf_dataset, repo_type="dataset", token=hf_token)
-    return folder+"/legal_info_search_data/"
+    folder = huggingface_hub.snapshot_download(repo_id=hf_dataset, repo_type="dataset", token=hf_token, cache_dir="/data")
+    return folder

semantic_search.py

@@ -29,7 +29,7 @@ hf_dataset = os.environ.get("HF_DATASET", None)
 hf_model_name = os.environ.get("HF_MODEL_NAME", "")
 
 if hf_token is not None and hf_dataset is not None:
-    global_data_path = dataset_utils.get_global_data_path()
+    global_data_path = dataset_utils.get_global_data_path()+global_data_path
     print(f"Global data path: {global_data_path}")
     
 

transaction_maps_search.py

@@ -5,9 +5,7 @@ from business_transaction_map.components.embedding_extraction import EmbeddingEx
 import os
 from prompts import BUSINESS_TRANSACTION_PROMPT
 from llm.common import LlmApi
-
-
-db_files_path = os.environ.get("GLOBAL_TRANSACTION_MAPS_DATA_PATH", "transaction_maps_search_data/csv/карта_проводок_new.pkl")
+from huggingface import dataset_utils
 
 model_path = os.environ.get("GLOBAL_TRANSACTION_MAPS_MODEL_PATH", "")
 
@@ -17,6 +15,14 @@ class TransactionMapsSearch:
                  model_name_or_path: str = model_path,
                  device: str = DEVICE):
         
+        hf_token = os.environ.get("HF_TOKEN", None)
+        hf_dataset = os.environ.get("HF_DATASET", None)
+        db_files_path = os.environ.get("GLOBAL_TRANSACTION_MAPS_DATA_PATH", "transaction_maps_search_data/csv/карта_проводок_new.pkl")
+        
+        if hf_token is not None and hf_dataset is not None:
+            db_files_path = dataset_utils.get_global_data_path()+db_files_path
+            print(f"Transaction maps data path: {db_files_path}")
+    
         self.device = device
         self.model = self.load_model(
             model_name_or_path=model_name_or_path,