Update app.py

app.py

@@ -326,16 +326,24 @@ def rag_workflow(query):
 
 
 def initialize():
-    global vectorstore, chunks, llm
-
-    partial_paths = ['kadi_apy/lib/resources/']
-    file_paths = []
-    #partial_paths = ['docs/source/setup/', 'kadi_apy/lib/']
-    #file_paths = ['docs/source/usage/lib.rst']
-    all_texts, file_references = process_directory(REPOSITORY_DIRECTORY, partial_paths, file_paths)
-    chunks = split_pythoncode_into_chunks(all_texts, file_references, 512, 0)
-    #chunks = split_into_chunks(all_texts, file_references, 512, 0)
-    print(f"Total number of chunks: {len(chunks)}")
+    global docstore, vectorstore, chunks, llm
+
+    code_partial_paths = ['kadi_apy/lib/resources/']
+    code_file_path = []
+    doc_partial_paths = ['docs/source/setup/']
+    doc_file_paths = ['docs/source/usage/lib.rst']
+
+    
+    doc_files, doc_file_references = process_directory(REPOSITORY_DIRECTORY, partial_paths, file_paths)
+    
+    doc_files, doc_file_references = process_directory(REPOSITORY_DIRECTORY, partial_paths, file_paths)
+    
+    code_chunks = split_pythoncode_into_chunks(doc_files, doc_file_references, 1500, 0)
+    doc_chunks = split_into_chunks(all_texts, file_references, 1024, 128)
+
+    print(f"Total number of code_chunks: {len(code_chunks)}")
+    print(f"Total number of code_chunks: {len(doc_chunks)}")
+
     # vectorstore = setup_chroma(chunks, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY)
     # llm = setup_llm(LLM_MODEL_NAME, LLM_TEMPERATURE, GROQ_API_KEY)