fdsa

semsearch.py

@@ -19,116 +19,6 @@ logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
 
-#################################################################
-# Create the chunks collection for the Weaviate database.
-def createChunksCollection():
-    logger.info("#### createChunksCollection() entered.")
-    if client.collections.exists("Chunks"):
-        client.collections.delete("Chunks")
-
-    class_obj = {
-        "class": "Chunks",
-        "description": "Collection for document chunks.",
-        "vectorizer": "text2vec-transformers",
-        "moduleConfig": {
-            "text2vec-transformers": {
-                "vectorizeClassName": True
-            }
-        },
-        "vectorIndexType": "hnsw",  
-        "vectorIndexConfig": {
-            "distance": "cosine",
-        },
-        "properties": [
-            {
-                "name": "chunk",
-                "dataType": ["text"],
-                "description": "Single webpage chunk.",
-                "vectorizer": "text2vec-transformers",
-                "moduleConfig": {
-                    "text2vec-transformers": {
-                      "vectorizePropertyName": False,
-                      "skip": False,
-                      "tokenization": "lowercase"
-                    }
-                }
-            },
-            {
-                "name": "chunk_index",
-                "dataType": ["int"]
-            },
-            {
-                "name": "webpage",
-                "dataType": ["Documents"],
-                "description": "Webpage content chunks.",
-            
-                "invertedIndexConfig": {
-                    "bm25": {
-                        "b": 0.75,
-                        "k1": 1.2
-                    }
-                }  
-            }
-        ]
-    }
-    return(client.collections.create_from_dict(class_obj))
-
-
-#####################################################################
-# Create the document collection for the Weaviate database.
-def createWebpageCollection():
-    logger.info("#### createWebpageCollection() entered.")
-    if client.collections.exists("Documents"):
-        client.collections.delete("Documents")
-
-    class_obj = {
-      "class": "Documents",
-      "description": "For first attempt at loading a Weviate database.",
-      "vectorizer": "text2vec-transformers",
-      "moduleConfig": {
-        "text2vec-transformers": {
-        "vectorizeClassName": False
-        }
-      },
-      "vectorIndexType": "hnsw",  
-      "vectorIndexConfig": {
-          "distance": "cosine",
-      },
-      "properties": [
-            {
-                "name": "title",
-                "dataType": ["text"],
-                "description": "HTML doc title.",
-                "vectorizer": "text2vec-transformers",
-                "moduleConfig": {
-                    "text2vec-transformers": {
-                        "vectorizePropertyName": True,
-                        "skip": False,
-                        "tokenization": "lowercase"
-                    }
-                },
-                "invertedIndexConfig": {
-                    "bm25": {
-                        "b": 0.75,
-                        "k1": 1.2
-                    },
-                }
-            },
-            {
-                "name": "content",
-                "dataType": ["text"],
-                "description": "HTML page content.",
-                "moduleConfig": {
-                    "text2vec-transformers": {
-                        "vectorizePropertyName": True,
-                        "tokenization": "whitespace"
-                    }
-                }
-            }
-        ]  
-    }
-    return(client.collections.create_from_dict(class_obj))
-
 
 ######################################################################
 # MAINLINE
@@ -145,19 +35,6 @@ webpageTitles = []
 webpageChunksDocNames = []
 
 
-######################################################
-# Connect to the Weaviate vector database.
-logger.info("#### Create Weaviate db client connection.")
-client = weaviate.connect_to_custom(
-    http_host="127.0.0.1",
-    http_port=8080,
-    http_secure=False,
-    grpc_host="127.0.0.1",
-    grpc_port=50051,
-    grpc_secure=False
-)
-client.connect() 
-
 #######################################################
 # Read each text input file, parse it into a document,
 # chunk it, collect chunks and document name.
@@ -196,6 +73,21 @@ for filename in os.listdir(pathString):
     
 logger.debug(f"### webpageDocNames: {webpageDocNames}")
 
+
+######################################################
+# Connect to the Weaviate vector database.
+logger.info("#### Create Weaviate db client connection.")
+client = weaviate.connect_to_custom(
+    http_host="127.0.0.1",
+    http_port=8080,
+    http_secure=False,
+    grpc_host="127.0.0.1",
+    grpc_port=50051,
+    grpc_secure=False
+)
+client.connect() 
+
+
 ######################################################
 # Create database webpage and chunks collections.
 #wpCollection = createWebpageCollection()