Spaces:
Running
on
T4
Running
on
T4
Update auditqa/process_chunks.py
Browse files- auditqa/process_chunks.py +15 -1
auditqa/process_chunks.py
CHANGED
|
@@ -102,9 +102,23 @@ def load_chunks():
|
|
| 102 |
qdrant_collections[file] = Qdrant.from_documents(
|
| 103 |
value,
|
| 104 |
embeddings,
|
| 105 |
-
|
| 106 |
collection_name=file,
|
| 107 |
)
|
| 108 |
print(qdrant_collections)
|
| 109 |
print("vector embeddings done")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
return qdrant_collections
|
|
|
|
| 102 |
qdrant_collections[file] = Qdrant.from_documents(
|
| 103 |
value,
|
| 104 |
embeddings,
|
| 105 |
+
path="/data/local_qdrant",
|
| 106 |
collection_name=file,
|
| 107 |
)
|
| 108 |
print(qdrant_collections)
|
| 109 |
print("vector embeddings done")
|
| 110 |
+
return qdrant_collections
|
| 111 |
+
|
| 112 |
+
def get_local_qdrant():
|
| 113 |
+
config = getconfig("./model_params.cfg")
|
| 114 |
+
qdrant_collections = {}
|
| 115 |
+
embeddings = HuggingFaceEmbeddings(
|
| 116 |
+
model_kwargs = {'device': device},
|
| 117 |
+
encode_kwargs = {'normalize_embeddings': True},
|
| 118 |
+
model_name=config.get('retriever','MODEL'))
|
| 119 |
+
#list_ = ['Consolidated','District','Ministry','allreports']
|
| 120 |
+
#for val in list_:
|
| 121 |
+
client = QdrantClient(path="/data/local_qdrant")
|
| 122 |
+
print(client.get_collections())
|
| 123 |
+
qdrant_collections['allreports'] = Qdrant(client=client, collection_name='allreports', embeddings=embeddings, )
|
| 124 |
return qdrant_collections
|