logging

semsearch.py

@@ -21,7 +21,7 @@ from IPython.display import display, clear_output
 
 
 weaviate_logger = logging.getLogger("httpx")
-weaviate_logger.setLevel(logging.INFO)
+weaviate_logger.setLevel(logging.WARNING)
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
@@ -95,60 +95,9 @@ submitButton = widgets.Button(
 )
 
 
-#######################################################
-# Read each text input file, parse it into a document,
-# chunk it, collect chunks and document name.
-logger.info("#### Read and chunk input text files.")
-for filename in os.listdir(pathString):
-    logger.info(filename)
-    path = Path(pathString + "/" + filename)
-    filename = filename.rstrip(".html")
-    webpageDocNames.append(filename)
-    htmlLoader = BSHTMLLoader(path,"utf-8") 
-    htmlData = htmlLoader.load()
-
-    title   = htmlData[0].metadata['title']
-    page_content = htmlData[0].page_content
-    
-    # Clean data. Remove multiple newlines, etc.
-    page_content = re.sub(r'\n+', '\n',page_content)
-    
-    page_contentArray.append(page_content);
-    webpageTitles.append(title)
-    max_tokens = 1000
-    tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
-    logger.debug(f"### tokenizer: {tokenizer}")
-    splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True)
-    chunksOnePage = splitter.chunks(page_content, chunk_capacity=50)
-
-    chunks = []
-    for chnk in chunksOnePage:
-        logger.debug(f"#### chnk in file: {chnk}")
-        chunks.append(chnk)
-        logger.debug(f"chunks: {chunks}")
-    webpageChunks.append(chunks)
-    webpageChunksDocNames.append(filename + "Chunks")
-
-    logger.debug(f"### filename, title: {filename}, {title}")
-    
-logger.debug(f"### webpageDocNames: {webpageDocNames}")
-
-
 ######################################################
 # Connect to the Weaviate vector database.
 logger.info("#### Create Weaviate db client connection.")
-#client = weaviate.connect_to_custom(
-#    http_host="127.0.0.1",
-#    http_port=8080,
-#    http_secure=False,
-#    grpc_host="127.0.0.1",
-#    grpc_port=50051,
-#    grpc_secure=False,
-#    timeout=[600,600]
-#    #read_timeout=600,
-#    #write_timeout=90
-#)
-
 client = weaviate.WeaviateClient(
     connection_params=ConnectionParams.from_params(
         http_host="localhost",
@@ -156,20 +105,56 @@ client = weaviate.WeaviateClient(
         http_secure=False,
         grpc_host="localhost",
         grpc_port="50051",
-        grpc_secure=False,
-        log_level="WARNING"
+        grpc_secure=False
     ),
-#    auth_client_secret=weaviate.auth.AuthApiKey("secr3tk3y"),
-#    additional_headers={
-#        "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY")
-#    },
-    additional_config=AdditionalConfig(
-        timeout=Timeout(init=60, query=1800, insert=1800),  # Values in seconds
-    )
+   },
+   additional_config=AdditionalConfig(
+       timeout=Timeout(init=60, query=1800, insert=1800),  # Values in seconds
+   )
 )
 client.connect() 
 
 
+#######################################################
+# Read each text input file, parse it into a document,
+# chunk it, collect chunks and document name.
+logger.info("#### Read and chunk input text files.")
+if not client.collections.exists("Documents") || not client.collections.exists("Documents") :
+    for filename in os.listdir(pathString):
+        logger.info(filename)
+        path = Path(pathString + "/" + filename)
+        filename = filename.rstrip(".html")
+        webpageDocNames.append(filename)
+        htmlLoader = BSHTMLLoader(path,"utf-8") 
+        htmlData = htmlLoader.load()
+
+        title   = htmlData[0].metadata['title']
+        page_content = htmlData[0].page_content
+        
+        # Clean data. Remove multiple newlines, etc.
+        page_content = re.sub(r'\n+', '\n',page_content)
+        
+        page_contentArray.append(page_content);
+        webpageTitles.append(title)
+        max_tokens = 1000
+        tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
+        logger.debug(f"### tokenizer: {tokenizer}")
+        splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True)
+        chunksOnePage = splitter.chunks(page_content, chunk_capacity=50)
+
+        chunks = []
+        for chnk in chunksOnePage:
+            logger.debug(f"#### chnk in file: {chnk}")
+            chunks.append(chnk)
+            logger.debug(f"chunks: {chunks}")
+        webpageChunks.append(chunks)
+        webpageChunksDocNames.append(filename + "Chunks")
+
+        logger.debug(f"### filename, title: {filename}, {title}")
+        logger.debug(f"### webpageDocNames: {webpageDocNames}")
+
+
+
 ######################################################
 # Create database webpage and chunks collections.
 #wpCollection = createWebpageCollection()
@@ -278,20 +263,23 @@ if not client.collections.exists("Chunks"):
 
 ###########################################################
 # Create document and chunks objects in the database.
-logger.info("#### Create page/doc db objects.")
-for i, className in enumerate(webpageDocNames):
-    title = webpageTitles[i]
-    logger.debug(f"## className, title: {className}, {title}")
-    # Create Webpage Object
-    page_content = page_contentArray[i]
-    # Insert the document.
-    wpCollectionObj_uuid = wpCollection.data.insert(
-      {
-        "name": className,
-        "title": title,
-        "content": page_content
-      }
-    )        
+if not client.collections.exists("Documents") :
+    logger.info("#### Create page/doc db objects.")
+    for i, className in enumerate(webpageDocNames):
+        title = webpageTitles[i]
+        logger.debug(f"## className, title: {className}, {title}")
+        # Create Webpage Object
+        page_content = page_contentArray[i]
+        # Insert the document.
+        wpCollectionObj_uuid = wpCollection.data.insert(
+          {
+            "name": className,
+            "title": title,
+            "content": page_content
+          }
+        )
+    
+if not client.collections.exists("Chunks") :
     logger.info("#### Create chunk db objects.")
     # Insert the chunks for the document.
     for i2, chunk in enumerate(webpageChunks[i]):
@@ -436,7 +424,6 @@ submitButton.on_click(on_submitButton_clicked)
 display(output_widget)
 
 
-logger.info("#### Closing client db connection.")
-client.close()
-
-logger.info("#### Program terminating.")
+#logger.info("#### Closing client db connection.")
+#client.close()
+#logger.info("#### Program terminating.")