Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -489,33 +489,35 @@ def rag_workflow(query):
|
|
| 489 |
|
| 490 |
def initialize():
|
| 491 |
global docstore, codestore, chunks, llm
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
|
|
|
|
|
|
| 495 |
#doc_partial_paths = []
|
| 496 |
-
doc_partial_paths = ['docs/source/setup/']
|
| 497 |
-
doc_file_paths = ['docs/source/usage/lib.rst']
|
| 498 |
|
| 499 |
-
code_files, code_file_references = process_directory(DATA_DIR,code_partial_paths, code_file_paths)
|
| 500 |
-
print("LEEEEEEEEEEEENGTH of code_files: ", len(code_files))
|
| 501 |
|
| 502 |
|
| 503 |
-
doc_files, doc_file_references = process_directory(DATA_DIR,doc_partial_paths, doc_file_paths)
|
| 504 |
-
print("LEEEEEEEEEEEENGTH of doc_files: ", len(doc_files))
|
| 505 |
#code_files, code_file_references = process_directory5(DATA_DIR, code_partial_paths, code_file_path)
|
| 506 |
|
| 507 |
#doc_files, doc_file_references = process_directory5(DATA_DIR, doc_partial_paths, doc_file_paths)
|
| 508 |
|
| 509 |
-
code_chunks = split_pythoncode_into_chunks(code_files, code_file_references, 1500, 0)
|
| 510 |
-
doc_chunks = split_into_chunks(doc_files, doc_file_references, CHUNK_SIZE, CHUNK_OVERLAP)
|
| 511 |
|
| 512 |
-
print(f"Total number of code_chunks: {len(code_chunks)}")
|
| 513 |
-
print(f"Total number of doc_chunks: {len(doc_chunks)}")
|
| 514 |
|
| 515 |
-
docstore = setup_vectorstore(doc_chunks, EMBEDDING_MODEL_NAME, PERSIST_DOC_DIRECTORY)
|
| 516 |
-
codestore = setup_vectorstore(code_chunks, EMBEDDING_MODEL_NAME, PERSIST_CODE_DIRECTORY)
|
| 517 |
|
| 518 |
-
llm = setup_llm(LLM_MODEL_NAME, LLM_TEMPERATURE, GROQ_API_KEY)
|
| 519 |
|
| 520 |
|
| 521 |
initialize()
|
|
|
|
| 489 |
|
| 490 |
def initialize():
|
| 491 |
global docstore, codestore, chunks, llm
|
| 492 |
+
|
| 493 |
+
download_gitlab_project_by_version
|
| 494 |
+
#download_gitlab_repo()
|
| 495 |
+
#code_partial_paths = ['kadi_apy/lib/']
|
| 496 |
+
#code_file_paths = []
|
| 497 |
#doc_partial_paths = []
|
| 498 |
+
#doc_partial_paths = ['docs/source/setup/']
|
| 499 |
+
#doc_file_paths = ['docs/source/usage/lib.rst']
|
| 500 |
|
| 501 |
+
#code_files, code_file_references = process_directory(DATA_DIR,code_partial_paths, code_file_paths)
|
| 502 |
+
#print("LEEEEEEEEEEEENGTH of code_files: ", len(code_files))
|
| 503 |
|
| 504 |
|
| 505 |
+
#doc_files, doc_file_references = process_directory(DATA_DIR,doc_partial_paths, doc_file_paths)
|
| 506 |
+
#print("LEEEEEEEEEEEENGTH of doc_files: ", len(doc_files))
|
| 507 |
#code_files, code_file_references = process_directory5(DATA_DIR, code_partial_paths, code_file_path)
|
| 508 |
|
| 509 |
#doc_files, doc_file_references = process_directory5(DATA_DIR, doc_partial_paths, doc_file_paths)
|
| 510 |
|
| 511 |
+
#code_chunks = split_pythoncode_into_chunks(code_files, code_file_references, 1500, 0)
|
| 512 |
+
#doc_chunks = split_into_chunks(doc_files, doc_file_references, CHUNK_SIZE, CHUNK_OVERLAP)
|
| 513 |
|
| 514 |
+
#print(f"Total number of code_chunks: {len(code_chunks)}")
|
| 515 |
+
#print(f"Total number of doc_chunks: {len(doc_chunks)}")
|
| 516 |
|
| 517 |
+
#docstore = setup_vectorstore(doc_chunks, EMBEDDING_MODEL_NAME, PERSIST_DOC_DIRECTORY)
|
| 518 |
+
#codestore = setup_vectorstore(code_chunks, EMBEDDING_MODEL_NAME, PERSIST_CODE_DIRECTORY)
|
| 519 |
|
| 520 |
+
#llm = setup_llm(LLM_MODEL_NAME, LLM_TEMPERATURE, GROQ_API_KEY)
|
| 521 |
|
| 522 |
|
| 523 |
initialize()
|