checking rag if it works

app.py

@@ -25,30 +25,56 @@ def initialize_system():
     global query_engine
     query_engine = None
     
-    try:
-        query_engine = load_rag_system()  
-        if query_engine is not None:
-            chunk_count = 0
-            if os.path.exists(PROCESSED_DATA_FILE):
-                processed_chunks = load_processed_chunks(PROCESSED_DATA_FILE)
-                chunk_count = len(processed_chunks)
-            else:
-                try:
-                    import pickle
-                    with open(os.path.join("processed_data", 'documents.pkl'), 'rb') as f:
-                        documents = pickle.load(f)
-                        chunk_count = len(documents)
-                except:
-                    chunk_count = "неизвестно"
-            
-            return f"AIEXP система инициализирована с {chunk_count} фрагментами нормативных документов (загружена из сохраненного индекса)"
-    except Exception as e:
-        print(f"Не удалось загрузить сохраненную систему: {str(e)}")
+    # Fix: Check if existing RAG system files exist first
+    if os.path.exists(os.path.join(RAG_FILES_DIR, 'faiss_index.index')):
+        try:
+            print("Found existing RAG system files, loading...")
+            query_engine = load_rag_system()  
+            if query_engine is not None:
+                # Count chunks from existing system
+                chunk_count = 0
+                if os.path.exists(PROCESSED_DATA_FILE):
+                    processed_chunks = load_processed_chunks(PROCESSED_DATA_FILE)
+                    chunk_count = len(processed_chunks)
+                else:
+                    try:
+                        import pickle
+                        with open(os.path.join(RAG_FILES_DIR, 'documents.pkl'), 'rb') as f:
+                            documents = pickle.load(f)
+                            chunk_count = len(documents)
+                    except Exception as e:
+                        print(f"Could not count documents: {e}")
+                        chunk_count = "неизвестно"
+                
+                return f"AIEXP система инициализирована с {chunk_count} фрагментами нормативных документов (загружена из сохраненного индекса)"
+        except Exception as e:
+            print(f"Не удалось загрузить сохраненную систему: {str(e)}")
     
+    # If no existing RAG system, try to load from CSV
     if os.path.exists(PROCESSED_DATA_FILE):
         try:
-            processed_chunks = load_processed_chunks(PROCESSED_DATA_FILE).to_dict('records')
+            print("Loading from CSV file...")
+            processed_chunks_df = load_processed_chunks(PROCESSED_DATA_FILE)
+            
+            # Fix: Check for required columns with correct names from your CSV
+            required_columns = {'document_id', 'file_link', 'chunk_text', 'chunk_id'}
+            missing_columns = required_columns - set(processed_chunks_df.columns)
+            if missing_columns:
+                return f"Ошибка при инициализации из CSV: отсутствуют необходимые столбцы: {missing_columns}"
+            
+            # Fix: Fill missing optional columns
+            if 'txt_file_id' not in processed_chunks_df.columns:
+                processed_chunks_df['txt_file_id'] = processed_chunks_df['document_id']
+            if 'section' not in processed_chunks_df.columns:
+                processed_chunks_df['section'] = ''
+            if 'subsection' not in processed_chunks_df.columns:
+                processed_chunks_df['subsection'] = ''
+            if 'chunk_length' not in processed_chunks_df.columns:
+                processed_chunks_df['chunk_length'] = processed_chunks_df['chunk_text'].str.len()
+            
+            processed_chunks = processed_chunks_df.to_dict('records')
             if processed_chunks:
+                print(f"Building RAG system with {len(processed_chunks)} chunks...")
                 query_engine = build_rag_system(processed_chunks)
                 return f"AIEXP система инициализирована с {len(processed_chunks)} фрагментами нормативных документов (построена из CSV)"
         except Exception as e:

scripts/config.py

@@ -17,11 +17,10 @@ PROCESSED_DATA_FILE = "processed_data/processed_chunks.csv"
 UPLOAD_FOLDER = "UPLOADED_DOCUMENTS"
 PROCESSED_DATA_FILE = "processed_data/processed_chunks.csv"
 INDEX_STATE_FILE = "processed_data/index_store.json"
-RAG_FILES_DIR = "rag_files"
-
-GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
-LLM_MODEL = "gemini-2.5-flash"
+RAG_FILES_DIR = "processed_data"
 
+GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY', "AIzaSyDemsCp7JIdRNDRyP6DkYdMox1DLZwPcPE")
+LLM_MODEL = "gemini-2.0-flash"
 
 
 CHUNK_SIZE = 1024
@@ -34,6 +33,27 @@ SIMILARITY_THRESHOLD = 0.7
 RETRIEVER_TOP_K = 15
 RETRIEVER_SIMILARITY_CUTOFF = 0.7
 
+
+def setup_llm_settings():
+    # Set embedding model first
+    Settings.embed_model = HuggingFaceEmbedding(model_name=EMBEDDING_MODEL)
+    
+    # Only set LLM if API key is available
+    if GOOGLE_API_KEY:
+        try:
+            llm = GoogleGenAI(model=LLM_MODEL, api_key=GOOGLE_API_KEY)
+            Settings.llm = llm
+            # Set system prompt after LLM is properly initialized
+            if hasattr(llm, 'system_prompt'):
+                llm.system_prompt = CUSTOM_PROMPT
+        except Exception as e:
+            print(f"Warning: Could not initialize Google GenAI LLM: {e}")
+            Settings.llm = None
+    else:
+        print("Warning: GOOGLE_API_KEY not found. Setting LLM to None.")
+        Settings.llm = None
+
+
 CUSTOM_PROMPT = """
 You are a highly specialized Document Analysis Assistant (AIEXP). Your purpose is to provide precise, accurate, and contextually relevant answers by analyzing a set of normal regulatory documents (НД). Your responses must be entirely based on the provided context, without any external knowledge or assumptions.
 
@@ -69,11 +89,6 @@ Question:
 Answer:
 """
 
-def setup_llm_settings():
-    Settings.embed_model = HuggingFaceEmbedding(model_name=EMBEDDING_MODEL)
-    Settings.llm = GoogleGenAI(model=LLM_MODEL, api_key=GOOGLE_API_KEY)
-    Settings.llm.system_prompt = CUSTOM_PROMPT
-
 
 LLM_MODEL_PREPROCESS = "gemini-1.5-flash"
 

scripts/rag_engine.py

@@ -26,7 +26,8 @@ def create_vector_index_with_faiss(documents):
     
     index = VectorStoreIndex.from_documents(
         documents, 
-        storage_context=storage_context
+        storage_context=storage_context,
+        embed_model = EMBEDDING_MODEL
     )
     
     return index, faiss_index
@@ -177,8 +178,10 @@ def load_rag_system():
         faiss_index = faiss.read_index(os.path.join(RAG_FILES_DIR, 'faiss_index.index'))
         vector_store = FaissVectorStore(faiss_index=faiss_index)
         storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+        embed_model = HuggingFaceEmbedding(model_name=EMBEDDING_MODEL)
         
-        index = VectorStoreIndex.from_documents([], storage_context=storage_context)
+        index = VectorStoreIndex.from_documents([], storage_context=storage_context, embed_model=embed_model)
         
         with open(os.path.join(RAG_FILES_DIR, 'documents.pkl'), 'rb') as f:
             documents = pickle.load(f)
@@ -192,7 +195,6 @@ def load_rag_system():
     except Exception as e:
         print(f"Error loading RAG system: {str(e)}")
         return None
-
 def build_rag_system(processed_chunks):
     setup_llm_settings()