fdsa

semsearch.py

@@ -72,7 +72,7 @@ def readParseChunkFiles():
         page_content = re.sub(r'\n+', '\n',page_content)
         
         page_contentArray.append(page_content);
-        webpageTitles.append(title)
+        webpageTitles.append    (title)
         max_tokens = 1000
         tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
         logger.debug(f"### tokenizer: {tokenizer}")
@@ -329,8 +329,36 @@ client.connect()
 readParseChunkFiles()
 wpCollection = createWebpageCollection()
 wpChunkCollection = createChunksCollection()
-createDatabaseObjects()
 
+#createDatabaseObjects()
+logger.info("#### Create page/doc and chunk db objects.")
+    for i, className in enumerate(webpageDocNames):
+        title = webpageTitles[i]
+        logger.debug(f"## className, title: {className}, {title}")
+        # Create Webpage Object
+        page_content = page_contentArray[i]
+        # Insert the document.
+        wpCollectionObj_uuid = wpCollection.data.insert(
+          {
+            "name": className,
+            "title": title,
+            "content": page_content
+          }
+        )        
+        
+        # Insert the chunks for the document.
+        for i2, chunk in enumerate(webpageChunks[i]):
+            chunk_uuid = wpChunkCollection.data.insert(
+              {
+                "title": title,
+                "chunk": chunk,
+                "chunk_index": i2,
+                "references":
+                {
+                  "webpage": wpCollectionObj_uuid
+                }
+              }
+            )
 
 ###############################################################################
 # text contains prompt for vector DB.