fdsa

semsearch.py

@@ -198,8 +198,111 @@ logger.debug(f"### webpageDocNames: {webpageDocNames}")
 
 ######################################################
 # Create database webpage and chunks collections.
-wpCollection = createWebpageCollection()
-wpChunkCollection = createChunksCollection()
+#wpCollection = createWebpageCollection()
+#wpChunkCollection = createChunksCollection()
+
+logger.info("#### createChunksCollection() entered.")
+if client.collections.exists("Chunks"):
+    client.collections.delete("Chunks")
+
+class_obj = {
+    "class": "Chunks",
+    "description": "Collection for document chunks.",
+    "vectorizer": "text2vec-transformers",
+    "moduleConfig": {
+        "text2vec-transformers": {
+            "vectorizeClassName": True
+        }
+    },
+    "vectorIndexType": "hnsw",  
+    "vectorIndexConfig": {
+        "distance": "cosine",
+    },
+    "properties": [
+        {
+            "name": "chunk",
+            "dataType": ["text"],
+            "description": "Single webpage chunk.",
+            "vectorizer": "text2vec-transformers",
+            "moduleConfig": {
+                "text2vec-transformers": {
+                  "vectorizePropertyName": False,
+                  "skip": False,
+                  "tokenization": "lowercase"
+                }
+            }
+        },
+        {
+            "name": "chunk_index",
+            "dataType": ["int"]
+        },
+        {
+            "name": "webpage",
+            "dataType": ["Documents"],
+            "description": "Webpage content chunks.",
+        
+            "invertedIndexConfig": {
+                "bm25": {
+                    "b": 0.75,
+                    "k1": 1.2
+                }
+            }  
+        }
+    ]
+}
+return(client.collections.create_from_dict(class_obj))
+
+logger.info("#### createWebpageCollection() entered.")
+if client.collections.exists("Documents"):
+    client.collections.delete("Documents")
+
+class_obj = {
+  "class": "Documents",
+  "description": "For first attempt at loading a Weviate database.",
+  "vectorizer": "text2vec-transformers",
+  "moduleConfig": {
+    "text2vec-transformers": {
+    "vectorizeClassName": False
+    }
+  },
+  "vectorIndexType": "hnsw",  
+  "vectorIndexConfig": {
+      "distance": "cosine",
+  },
+  "properties": [
+        {
+            "name": "title",
+            "dataType": ["text"],
+            "description": "HTML doc title.",
+            "vectorizer": "text2vec-transformers",
+            "moduleConfig": {
+                "text2vec-transformers": {
+                    "vectorizePropertyName": True,
+                    "skip": False,
+                    "tokenization": "lowercase"
+                }
+            },
+            "invertedIndexConfig": {
+                "bm25": {
+                    "b": 0.75,
+                    "k1": 1.2
+                },
+            }
+        },
+        {
+            "name": "content",
+            "dataType": ["text"],
+            "description": "HTML page content.",
+            "moduleConfig": {
+                "text2vec-transformers": {
+                    "vectorizePropertyName": True,
+                    "tokenization": "whitespace"
+                }
+            }
+        }
+    ]  
+}
+return(client.collections.create_from_dict(class_obj))
 
 ###########################################################
 # Create document and chunks objects in the database.