Create semsearch.py

semsearch.py

@@ -0,0 +1,284 @@
+import weaviate
+import weaviate.classes as wvc
+from weaviate.embedded import EmbeddedOptions
+from sentence_transformers import SentenceTransformer
+from langchain_community.document_loaders import BSHTMLLoader
+from pathlib import Path
+from lxml import html
+import logging
+from semantic_text_splitter import HuggingFaceTextSplitter
+from tokenizers import Tokenizer
+import json
+import os
+import re
+
+def createChunksCollection():
+    print("#### createChunksCollection() entered.")
+    if client.collections.exists("Chunks"):
+        client.collections.delete("Chunks")
+
+    class_obj = {
+        "class": "Chunks",
+        "description": "Collection for document chunks.",
+        "vectorizer": "text2vec-transformers",
+        "moduleConfig": {
+            "text2vec-transformers": {
+                "vectorizeClassName": True
+            }
+        },
+        "vectorIndexType": "hnsw",  
+        "vectorIndexConfig": {
+            "distance": "cosine",
+        },
+        "properties": [
+            {
+                "name": "chunk",
+                "dataType": ["text"],
+                "description": "Single webpage chunk.",
+                "vectorizer": "text2vec-transformers",
+                "moduleConfig": {
+                    "text2vec-transformers": {
+                      "vectorizePropertyName": False,
+                      "skip": False,
+                      "tokenization": "lowercase"
+                    }
+                }
+            },
+            {
+                "name": "chunk_index",
+                "dataType": ["int"]
+            },
+            {
+                "name": "webpage",
+                "dataType": ["Documents"],
+                "description": "Webpage content chunks.",
+            
+                "invertedIndexConfig": {
+                    "bm25": {
+                        "b": 0.75,
+                        "k1": 1.2
+                    }
+                }  
+            }
+        ]
+    }
+
+    return(client.collections.create_from_dict(class_obj))
+
+def createWebpageCollection():
+    print("#### createWebpageCollection() entered.")
+    if client.collections.exists("Documents"):
+        client.collections.delete("Documents")
+
+    class_obj = {
+      "class": "Documents",
+      "description": "For first attempt at loading a Weviate database.",
+      "vectorizer": "text2vec-transformers",
+      "moduleConfig": {
+        "text2vec-transformers": {
+        "vectorizeClassName": False
+        }
+      },
+      "vectorIndexType": "hnsw",  
+      "vectorIndexConfig": {
+          "distance": "cosine",
+      },
+      "properties": [
+            #{
+            #    "docname": "fdsa",
+            #    "dataType": ["text"],
+            #    "description": "Name of document"
+            #},
+            {
+                "name": "title",
+                "dataType": ["text"],
+                "description": "HTML doc title.",
+                "vectorizer": "text2vec-transformers",
+                "moduleConfig": {
+                    "text2vec-transformers": {
+                        "vectorizePropertyName": True,
+                        "skip": False,
+                        "tokenization": "lowercase"
+                    }
+                },
+                "invertedIndexConfig": {
+                    "bm25": {
+                        "b": 0.75,
+                        "k1": 1.2
+                    },
+                }
+            },
+            {
+                "name": "content",
+                "dataType": ["text"],
+                "description": "HTML page content.",
+                "moduleConfig": {
+                    "text2vec-transformers": {
+                        "vectorizePropertyName": True,
+                        "tokenization": "whitespace"
+                    }
+                }
+            }
+        ]  
+    }
+      
+    return(client.collections.create_from_dict(class_obj))
+
+
+#
+# MAINLINE
+#
+#pathString = "/Users/660565/KPSAllInOne/ProgramFilesX86/WebCopy/DownloadedWebSites/LLMPOC_HTML" 
+pathString = "inputDocs"
+chunks = []
+webpageDocNames = []
+#webpageChunksClassesNames = []
+page_contentArray = []
+webpageChunks = []
+webpageTitles = []
+webpageChunksDocNames = []
+
+#client = weaviate.WeaviateClient(
+#    embedded_options=EmbeddedOptions(
+#        additional_env_vars={
+#            "ENABLE_MODULES": "backup-filesystem,text2vec-transformers",
+#            "BACKUP_FILESYSTEM_PATH": "/tmp/backups",
+#            "PERSISTENCE_DATA_PATH": "/var/lib/weaviate",
+#            "DEFAULT_VECTORIZER_MODULE": "text2vec-transformers"
+#            #"TRANSFORMERS_INFERENCE_API": "http://huggingface.co/spaces/MVPilgrim/WeaviateDB:8080"
+#            
+#        }
+#    )
+#)
+
+client = weaviate.connect_to_custom(
+    #http_host="http://huggingface.co/spaces/MVPilgrim/WeaviateDB",
+    http_host="http://127.0.0.1",
+    http_port=8080,
+    http_secure=False,
+    #grpc_host="huggingface.co",
+    grpc_host="127.0.0.1",
+    grpc_port=50051,
+    grpc_secure=False
+    #auth_credentials=AuthApiKey(weaviate_key),   # `weaviate_key`: your Weaviate API key
+)
+print("#### client: ",client)
+
+client.connect() 
+
+for filename in os.listdir(pathString):
+    print(filename)
+    path = Path(pathString + "/" + filename)
+    filename = filename.rstrip(".html")
+    webpageDocNames.append(filename)
+    htmlLoader = BSHTMLLoader(path,"utf-8") 
+    htmlData = htmlLoader.load()
+
+    title   = htmlData[0].metadata['title']
+    page_content = htmlData[0].page_content
+    
+    #  Clean data. Remove multiple newlines, etc.
+    page_content = re.sub(r'\n+', '\n',page_content)
+    
+    page_contentArray.append(page_content);
+    webpageTitles.append(title)
+    #htmlDocument = htmlData[0]
+    max_tokens = 1000
+    tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
+    splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True)
+    chunksOnePage = splitter.chunks(page_content, chunk_capacity=50)
+
+    chunks = []
+    for chnk in chunksOnePage:
+        #print("\n\n#### chnk: ",chnk)
+        chunks.append(chnk)
+        #print("chunks: ",chunks)
+    webpageChunks.append(chunks)
+    webpageChunksDocNames.append(filename + "Chunks")
+
+    print("### filename, title: ",filename,",",title)
+print("### webpageDocNames: ",webpageDocNames)
+
+wpCollection = createWebpageCollection()
+wpChunkCollection = createChunksCollection()
+
+for i, className in enumerate(webpageDocNames):
+    title = webpageTitles[i]
+    print("## className, title: ",className,",",title)
+    # Create Webpage Object
+    page_content = page_contentArray[i]
+    #print("\n#### page_content: ",page_content)
+    
+    wpCollectionObj_uuid = wpCollection.data.insert(
+      {
+        "name": className,
+        "title": title,
+        "content": page_content
+      }
+    )        
+    
+    for i2, chunk in enumerate(webpageChunks[i]):
+        #print("#### chunk: ",chunk)
+        chunk_uuid = wpChunkCollection.data.insert(
+          {
+            "title": title,
+            "chunk": chunk,
+            "chunk_index": i2,
+            "references":
+            {
+              "webpage": wpCollectionObj_uuid
+            }
+          }
+        )
+        #print("### chunk_index,chunk: ",i2,",",chunk[0:20])
+
+#text = "List the main capabilities of artificial intelligence."
+#text = "List three of the greatest Norwegian authors."
+#text = "turkey burgers golden fried with lots of mayonaise"
+text = "human-made computer cognitive ability"
+#text = "literature authors"
+#text = "artifical intelligence"
+
+
+model = SentenceTransformer('../multi-qa-MiniLM-L6-cos-v1')
+vector = model.encode(text)
+#print("#### vector: ",vector[0])
+vectorList = []
+
+for vec in vector:
+    vectorList.append(vec)
+print("vectorList: ",vectorList[2])
+
+semChunks = wpChunkCollection.query.near_vector(
+    near_vector=vectorList,
+    distance=0.7,
+    limit=3
+)
+print("### semChunks[0]: ",semChunks)
+#print("### semChunks.objects[0]: ",semChunks.objects[0])
+
+for chunk in enumerate(semChunks.objects):
+    print("\n\n#### chunk: ",chunk)
+    #webpage_uuid = chunk.properties['references']['webpage']
+    #webpage_uuid = chunk.references.webpage
+    webpage_uuid = chunk[1].properties['references']['webpage']
+    print("\nwebpage_uuid: ",webpage_uuid)
+    wpFromChunk = wpCollection.query.fetch_object_by_id(webpage_uuid)
+    print("\n\n### wpFromChunk title: ",wpFromChunk.properties['title'])
+
+
+#print("response: ",response)
+
+if False:
+    client = weaviate.connect_to_local(
+        #cluster_url="http://localhost:8080"
+    )
+    
+    for item in wpCollection.iterator():
+        print(print("\n## webpage collection: ",item.uuid, item.properties))
+    
+    for item in wpChunkCollection.iterator():
+        print(print("\n## chunk collection: ",item.uuid, item.properties))
+    
+    client.close()
+