Update main.py

README.md

@@ -5,7 +5,7 @@ colorFrom: green
 colorTo: red
 sdk: gradio
 sdk_version: 3.33.1
-app_file: app.py
+app_file: main.py
 pinned: false
 license: mit
 ---

app.py

@@ -19,7 +19,7 @@ text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=20
 texts = text_splitter.split_documents(docs)
 
 model_name = "hkunlp/instructor-base"
-embeddings = HuggingFaceInstructEmbeddings(
+embedding = HuggingFaceInstructEmbeddings(
     model_name=model_name, model_kwargs={"device": device}
 )
 
@@ -28,11 +28,11 @@ embeddings = HuggingFaceInstructEmbeddings(
 # both                      99 chunks, Wall time: 5min 4s CPU times: total: 13min 31s
 # chunks = len / 800
 
-db = Chroma.from_documents(texts, embeddings)
+db = Chroma.from_documents(texts, embedding)
 
 db = Chroma.from_documents(
     texts,
-    embeddings,
+    embedding,
     persist_directory=PERSIST_DIRECTORY,
     client_settings=CHROMA_SETTINGS,
 )
@@ -126,7 +126,8 @@ CHROMA_SETTINGS = Settings(
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
 ns_initial = SimpleNamespace(
-    qa=None,  # in effect Chroma db
+    db=None,
+    qa=None,
     ingest_done=None,
     files_info=None,
     files_uploaded=[],
@@ -229,17 +230,17 @@ def get_vectorstore(
     persist=True,
 ):
     """Gne vectorstore."""
-    # embeddings = OpenAIEmbeddings()
+    # embedding = OpenAIEmbeddings()
     # for HuggingFaceInstructEmbeddings
     model_name = "hkunlp/instructor-xl"
     model_name = "hkunlp/instructor-large"
     model_name = "hkunlp/instructor-base"
 
-    # embeddings = HuggingFaceInstructEmbeddings(model_name=model_name)
+    # embedding = HuggingFaceInstructEmbeddings(model_name=model_name)
 
     model_name = MODEL_NAME
     logger.info(f"Loading {model_name}")
-    embeddings = SentenceTransformerEmbeddings(model_name=model_name)
+    embedding = SentenceTransformerEmbeddings(model_name=model_name)
     logger.info(f"Done loading {model_name}")
 
     if vectorstore is None:
@@ -247,20 +248,20 @@ def get_vectorstore(
 
     if vectorstore.lower() in ["chroma"]:
         logger.info(
-            "Doing vectorstore Chroma.from_texts(texts=text_chunks, embedding=embeddings)"
+            "Doing vectorstore Chroma.from_texts(texts=text_chunks, embedding=embedding)"
         )
         if persist:
             vectorstore = Chroma.from_texts(
                 texts=text_chunks,
-                embedding=embeddings,
+                embedding=embedding,
                 persist_directory=PERSIST_DIRECTORY,
                 client_settings=CHROMA_SETTINGS,
             )
         else:
-            vectorstore = Chroma.from_texts(texts=text_chunks, embedding=embeddings)
+            vectorstore = Chroma.from_texts(texts=text_chunks, embedding=embedding)
 
         logger.info(
-            "Done vectorstore FAISS.from_texts(texts=text_chunks, embedding=embeddings)"
+            "Done vectorstore FAISS.from_texts(texts=text_chunks, embedding=embedding)"
         )
 
         return vectorstore
@@ -268,11 +269,11 @@ def get_vectorstore(
     # if vectorstore.lower() not in ['chroma']
     # TODO handle other cases
     logger.info(
-        "Doing vectorstore FAISS.from_texts(texts=text_chunks, embedding=embeddings)"
+        "Doing vectorstore FAISS.from_texts(texts=text_chunks, embedding=embedding)"
     )
-    vectorstore = FAISS.from_texts(texts=text_chunks, embedding=embeddings)
+    vectorstore = FAISS.from_texts(texts=text_chunks, embedding=embedding)
     logger.info(
-        "Done vectorstore FAISS.from_texts(texts=text_chunks, embedding=embeddings)"
+        "Done vectorstore FAISS.from_texts(texts=text_chunks, embedding=embedding)"
     )
 
     return vectorstore
@@ -308,9 +309,7 @@ def process_files(
     # wait for update before querying new ns.qa
     ns.ingest_done = False
 
-    logger.debug(f"{ns.files_uploaded}")
-
-    logger.info(f"ingest({ns.files_uploaded})...")
+    logger.debug(f"ns.files_uploaded: {ns.files_uploaded}")
 
     # imgs = [None] * 24
     # for img in progress.tqdm(imgs, desc="Loading from list"):
@@ -322,17 +321,25 @@ def process_files(
     # for img in progress.tqdm(img_set, desc="inner list"):
     # time.sleep(10.1)
 
-    # return f"done file(s): {ns.files_info}"
-    # return f"done file(s)"
+    # return "done..."
 
     documents = []
-    for file_path in progress.tqdm(ns.files_uploaded, desc="Reading file(s)"):
-        logger.debug(f"Doing {file_path}")
-        try:
-            documents.extend(load_single_document(f"{file_path}"))
-            logger.debug("Done reading files.")
-        except Exception as exc:
-            logger.error(f"{file_path}: {exc}")
+    if progress is None:
+        for file_path in ns.files_uploaded:
+            logger.debug(f"-Doing {file_path}")
+            try:
+                documents.extend(load_single_document(f"{file_path}"))
+                logger.debug("-Done reading files.")
+            except Exception as exc:
+                logger.error(f"-{file_path}: {exc}")
+    else:
+        for file_path in progress.tqdm(ns.files_uploaded, desc="Reading file(s)"):
+            logger.debug(f"Doing {file_path}")
+            try:
+                documents.extend(load_single_document(f"{file_path}"))
+                logger.debug("Done reading files.")
+            except Exception as exc:
+                logger.error(f"{file_path}: {exc}")
 
     text_splitter = RecursiveCharacterTextSplitter(
         chunk_size=ns.chunk_size, chunk_overlap=ns.chunk_overlap
@@ -340,30 +347,52 @@ def process_files(
     texts = text_splitter.split_documents(documents)
 
     logger.info(f"Loaded {len(ns.files_uploaded)} files ")
-    logger.info(f"Loaded {len(documents)} documents ")
-    logger.info(f"Split into {len(texts)} chunks of text")
-
-    # initilize if necessary
-    if ns.qa is None:
-        embeddings = SentenceTransformerEmbeddings(
-            model_name=ns.model_name, model_kwargs={"device": DEVICE}
-        )
+    logger.info(f"Loaded {len(documents)} document(s) ")
+    logger.info(f"Split into {len(texts)} chunk(s) of text")
+
+    # initialize if necessary
+    if ns.db is None:
+        logger.info(f"loading {ns.model_name:}")
+        for _ in progress.tqdm(range(1), desc="diggin..."):
+            embedding = SentenceTransformerEmbeddings(
+                model_name=ns.model_name, model_kwargs={"device": DEVICE}
+            )
 
-        ns.qa = Chroma(
-            # persist_directory=PERSIST_DIRECTORY,
-            embedding_function=embeddings,
-            # client_settings=CHROMA_SETTINGS,
-        )
+            logger.info("creating vectorstore")
+            ns.db = Chroma(
+                # persist_directory=PERSIST_DIRECTORY,
+                embedding_function=embedding,
+                # client_settings=CHROMA_SETTINGS,
+            )
+        logger.info("done creating vectorstore")
 
     total = ceil(len(texts) / 101)
-    # for text in progress.tqdm(
-    for idx, text in enumerate(progress.tqdm(
-        mit.chunked_even(texts, 101),
-        total=total,
-        desc="Processing docs",
-    )):
-        logger.debug(f"{idx + 1} of {total}")
-        ns.qa.add_documents(documents=text)
+    if progress is None:
+        # for text in progress.tqdm(
+        for idx, text in enumerate(mit.chunked_even(texts, 101)):
+            logger.debug(f"-{idx + 1} of {total}")
+            ns.db.add_documents(documents=text)
+    else:
+        # for text in progress.tqdm(
+        for idx, text in enumerate(progress.tqdm(
+            mit.chunked_even(texts, 101),
+            total=total,
+            desc="Processing docs",
+        )):
+            logger.debug(f"{idx + 1} of {total}")
+            ns.db.add_documents(documents=text)
+    logger.debug(f" done all {total}")
+
+    # ns.qa = load_qa()
+
+    llm = OpenAI(temperature=0, max_tokens=1024)  # type: ignore
+    retriever = ns.db.as_retriever()
+    ns.qa = RetrievalQA.from_chain_type(
+        llm=llm,
+        chain_type="stuff",
+        retriever=retriever,
+        # return_source_documents=True,
+    )
 
     ns.ingest_done = True
     _ = [
@@ -372,9 +401,55 @@ def process_files(
     ]
     ns.files_info = _
 
-    # ns.qa = load_qa()
+    logger.debug(f"{ns.ingest_done=}, exit process_files")
+    return f"done file(s): {dict(ns.files_info)}"
+
+
+def respond(message, chat_history):
+    """Gen response."""
+    logger.debug(f"{ns.files_uploaded=}")
+    if not ns.files_uploaded:  # no files processed yet
+        bot_message = "Upload some file(s) for processing first."
+        chat_history.append((message, bot_message))
+        return "", chat_history
 
-    return f"done file(s): {ns.files_info}"
+    logger.debug(f"{ns.ingest_done=}")
+    if not ns.ingest_done:  # embedding database not doen yet
+        bot_message = (
+            "Waiting for ingest (embedding) to finish, "
+            "be patient... You can switch the 'Upload files' "
+            "Tab to check"
+        )
+        chat_history.append((message, bot_message))
+        return "", chat_history
+
+    _ = """
+    if ns.qa is None:  # load qa one time
+        logger.info("Loading qa, need to do just one time.")
+        ns.qa = load_qa()
+        logger.info("Done loading qa, need to do just one time.")
+    # """
+    logger.debug(f"{ns.qa=}")
+    if ns.qa is None:
+        bot_message = "Looks like the bot is not ready. Try again later..."
+        chat_history.append((message, bot_message))
+        return "", chat_history
+
+    try:
+        res = ns.qa(message)
+        answer = res.get("result")
+        docs = res.get("source_documents")
+        if docs:
+            bot_message = f"{answer}\n({docs})"
+        else:
+            bot_message = f"{answer}"
+    except Exception as exc:
+        logger.error(exc)
+        bot_message = f"bummer! {exc}"
+
+    chat_history.append((message, bot_message))
+
+    return "", chat_history
 
 
 # pylint disable=unused-argument
@@ -424,9 +499,9 @@ def ingest(
     logger.info(f"Loaded {len(documents)} documents ")
     logger.info(f"Split into {len(texts)} chunks of text")
 
-    # Create embeddings
-    # embeddings = HuggingFaceInstructEmbeddings(
-    embeddings = SentenceTransformerEmbeddings(
+    # Create embedding
+    # embedding = HuggingFaceInstructEmbeddings(
+    embedding = SentenceTransformerEmbeddings(
         model_name=model_name, model_kwargs={"device": device}
     )
 
@@ -437,7 +512,7 @@ def ingest(
     # mit.chunked_even(texts, 100)
     db = Chroma(
         # persist_directory=PERSIST_DIRECTORY,
-        embedding_function=embeddings,
+        embedding_function=embedding,
         # client_settings=CHROMA_SETTINGS,
     )
     # for text in progress.tqdm(
@@ -448,7 +523,7 @@ def ingest(
     with about_time() as atime:  # type: ignore
         db = Chroma.from_documents(
             texts,
-            embeddings,
+            embedding,
             persist_directory=PERSIST_DIRECTORY,
             client_settings=CHROMA_SETTINGS,
         )
@@ -512,7 +587,14 @@ def gen_local_llm(model_id="TheBloke/vicuna-7B-1.1-HF"):
 
 
 def load_qa(device=None, model_name: str = MODEL_NAME):
-    """Gen qa."""
+    """Gen qa.
+
+    device = 'cpu'
+    model_name = "hkunlp/instructor-xl"
+    model_name = "hkunlp/instructor-large"
+    model_name = "hkunlp/instructor-base"
+    embedding = HuggingFaceInstructEmbeddings(
+    """
     logger.info("Doing qa")
     if device is None:
         if torch.cuda.is_available():
@@ -520,19 +602,14 @@ def load_qa(device=None, model_name: str = MODEL_NAME):
         else:
             device = "cpu"
 
-    # device = 'cpu'
-    # model_name = "hkunlp/instructor-xl"
-    # model_name = "hkunlp/instructor-large"
-    # model_name = "hkunlp/instructor-base"
-    # embeddings = HuggingFaceInstructEmbeddings(
-    embeddings = SentenceTransformerEmbeddings(
+    embedding = SentenceTransformerEmbeddings(
         model_name=model_name, model_kwargs={"device": device}
     )
     # xl 4.96G, large 3.5G,
 
     db = Chroma(
         persist_directory=PERSIST_DIRECTORY,
-        embedding_function=embeddings,
+        embedding_function=embedding,
         client_settings=CHROMA_SETTINGS,
     )
     retriever = db.as_retriever()
@@ -552,8 +629,7 @@ def load_qa(device=None, model_name: str = MODEL_NAME):
 
     return qa
 
-    # """
-
+    # TODO: conversation_chain
     # pylint: disable=unreachable
 
     # model = 'gpt-3.5-turbo', default text-davinci-003
@@ -615,7 +691,7 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         gr.Markdown(dedent(_))
 
     with gr.Tab("Upload files"):
-        # Upload files and generate embeddings database
+        # Upload files and generate vectorstore
         with gr.Row():
             file_output = gr.File()
             # file_output = gr.Text()
@@ -626,9 +702,10 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
                 file_count="multiple",
             )
         with gr.Row():
-            text2 = gr.Textbox("Progress/Log")
-            process_btn = gr.Button("Click to process files")
-        reset_btn = gr.Button("Reset everything")
+            text2 = gr.Textbox("Gen embedding")
+            process_btn = gr.Button("Click to embed")
+
+        # reset_btn = gr.Button("Reset everything", visibile=False)
 
     with gr.Tab("Query docs"):
         # interactive chat
@@ -643,21 +720,24 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         ns = deepcopy(ns_initial)
         return f"reset done: ns={ns}"
 
-    reset_btn.click(reset_all, [], text2)
+    # reset_btn.click(reset_all, [], text2)
 
     upload_button.upload(upload_files, upload_button, file_output)
     process_btn.click(process_files, [], text2)
 
     def respond(message, chat_history):
         """Gen response."""
+        logger.info(f"{ns.ingest_done=}")
         if ns.ingest_done is None:  # no files processed yet
             bot_message = "Upload some file(s) for processing first."
             chat_history.append((message, bot_message))
             return "", chat_history
 
+        logger.info(f"{ns.ingest_done=}")
         if not ns.ingest_done:  # embedding database not doen yet
             bot_message = (
                 "Waiting for ingest (embedding) to finish, "
+                f"({ns.ingest_done=})"
                 "be patient... You can switch the 'Upload files' "
                 "Tab to check"
             )
@@ -695,13 +775,6 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
     clear.click(lambda: None, None, chatbot, queue=False)
 
 if __name__ == "__main__":
-    # main()
-    try:
-        from google import colab  # noqa  # type: ignore
-
-        share = True  # start share when in colab
-    except Exception:
-        share = False
     demo.queue(concurrency_count=20).launch(share=share)
 
 _ = """
@@ -710,12 +783,12 @@ device = 'cpu'
 model_name = "hkunlp/instructor-xl"
 model_name = "hkunlp/instructor-large"
 model_name = "hkunlp/instructor-base"
-embeddings = HuggingFaceInstructEmbeddings(
+embedding = HuggingFaceInstructEmbeddings(
     model_name=,
     model_kwargs={"device": device}
 )
 # xl 4.96G, large 3.5G,
-db = Chroma(persist_directory=PERSIST_DIRECTORY, embedding_function=embeddings, client_settings=CHROMA_SETTINGS)
+db = Chroma(persist_directory=PERSIST_DIRECTORY, embedding_function=embedding, client_settings=CHROMA_SETTINGS)
 retriever = db.as_retriever()
 
 llm = gen_local_llm()  # "TheBloke/vicuna-7B-1.1-HF" 12G?

docs/test2.txt

@@ -0,0 +1,2 @@
+总　纲
+　　中国共产党是中国工人阶级的先锋队，同时是中国人民和中华民族的先锋队，是中国特色社会主义事业的领导核心，代表中国先进生产力的发展要求，代表中国先进文化的前进方向，代表中国最广大人民的根本利益。党的最高理想和最终目标是实现共产主义。

main.py

@@ -0,0 +1,50 @@
+"""Test."""
+# pylint: disable=invalid-name, unused-import, broad-except,
+from copy import deepcopy
+
+import gradio as gr
+from app import ingest, ns, ns_initial, process_files, upload_files, respond
+from load_api_key import load_api_key, pk_base, sk_base
+from loguru import logger
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    with gr.Tab("Upload files"):
+        # Upload files and generate vectorstore
+        with gr.Row():
+            file_output = gr.File()
+            # file_output = gr.Text()
+            # file_output = gr.DataFrame()
+            upload_button = gr.UploadButton(
+                "Click to upload",
+                # file_types=["*.pdf", "*.epub", "*.docx"],
+                file_count="multiple",
+            )
+        with gr.Row():
+            text2 = gr.Textbox("Gen embedding")
+            process_btn = gr.Button("Click to embed")
+
+        reset_btn = gr.Button("Reset everything", visible=False)
+
+    with gr.Tab("Query docs"):
+        # interactive chat
+        chatbot = gr.Chatbot()
+        msg = gr.Textbox(label="Query")
+        clear = gr.Button("Clear")
+
+    # actions
+    def reset_all():
+        """Reset ns."""
+        # global ns
+        globals().update(**{"ns": deepcopy(ns_initial)})
+        return f"reset done: ns={ns}"
+
+    reset_btn.click(reset_all, [], text2)
+
+    upload_button.upload(upload_files, upload_button, file_output)
+    process_btn.click(process_files, [], text2)
+
+    msg.submit(respond, [msg, chatbot], [msg, chatbot])
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+if __name__ == "__main__":
+    demo.queue(concurrency_count=20).launch()

requirements-freeze.txt

@@ -0,0 +1,179 @@
+about-time==4.2.1
+absl-py==0.11.0
+accelerate==0.19.0
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+analytics-python==1.4.post1
+anyio==3.7.0
+argilla==1.8.0
+astroid==2.15.5
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+backcall==0.2.0
+backoff==1.10.0
+bcrypt==4.0.1
+bitsandbytes==0.39.0
+black==23.3.0
+certifi==2023.5.7
+cffi==1.15.1
+chardet==5.1.0
+charset-normalizer==3.1.0
+chromadb==0.3.22
+click==8.1.3
+clickhouse-connect==0.5.25
+colorama==0.4.6
+commonmark==0.9.1
+contourpy==1.0.7
+cryptography==41.0.1
+cycler==0.11.0
+dataclasses-json==0.5.7
+decorator==5.1.1
+Deprecated==1.2.14
+dill==0.3.6
+docx2txt==0.8
+duckdb==0.8.0
+EbookLib==0.17.1
+epub2txt==0.1.6
+et-xmlfile==1.1.0
+exceptiongroup==1.1.1
+executing==1.2.0
+faiss-cpu==1.7.4
+fastapi==0.96.0
+ffmpy==0.3.0
+filelock==3.12.0
+fonttools==4.39.4
+frozenlist==1.3.3
+fsspec==2023.5.0
+gradio==3.35.2
+gradio_client==0.2.7
+greenlet==2.0.2
+h11==0.12.0
+hnswlib==0.7.0
+httpcore==0.12.3
+httptools==0.5.0
+httpx==0.16.1
+huggingface-hub==0.15.1
+idna==3.4
+InstructorEmbedding==1.0.1
+ipython==8.14.0
+isort==5.12.0
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+kiwisolver==1.4.4
+langchain==0.0.166
+lazy-object-proxy==1.9.0
+linkify-it-py==2.0.2
+llama-cpp-python==0.1.48
+llama-index==0.6.21.post1
+loguru==0.7.0
+logzero==1.7.0
+lxml==4.9.2
+lz4==4.3.2
+Markdown==3.4.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.3
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mccabe==0.7.0
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+monotonic==1.6
+more-itertools==9.1.0
+mpmath==1.3.0
+msg-parser==1.2.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+networkx==3.1
+nltk==3.8.1
+numexpr==2.8.4
+numpy==1.23.5
+olefile==0.46
+openai==0.27.8
+openapi-schema-pydantic==1.2.4
+openpyxl==3.1.2
+orjson==3.9.0
+packaging==23.1
+pandas==1.5.3
+paramiko==3.2.0
+parso==0.8.3
+pathspec==0.11.1
+pdfminer.six==20221105
+pickleshare==0.7.5
+Pillow==9.5.0
+platformdirs==3.5.1
+posthog==3.0.1
+prompt-toolkit==3.0.38
+protobuf==3.20.0
+psutil==5.9.5
+pure-eval==0.2.2
+pycparser==2.21
+pycryptodome==3.18.0
+pydantic==1.10.8
+pydub==0.25.1
+Pygments==2.15.1
+pylint==2.17.4
+PyNaCl==1.5.0
+pypandoc==1.11
+pyparsing==3.0.9
+pypdf==3.9.1
+PyPDF2==3.0.1
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-docx==0.8.11
+python-dotenv==1.0.0
+python-magic==0.4.27
+python-multipart==0.0.6
+python-pptx==0.6.21
+pytz==2023.3
+PyYAML==6.0
+regex==2023.6.3
+requests==2.31.0
+rfc3986==1.5.0
+rich==13.0.1
+scikit-learn==1.2.2
+scipy==1.10.1
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.15
+stack-data==0.6.2
+starlette==0.27.0
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.2
+threadpoolctl==3.1.0
+tiktoken==0.4.0
+tokenizers==0.13.3
+tomli==2.0.1
+tomlkit==0.11.8
+toolz==0.12.0
+torch==2.0.1
+torchvision==0.15.2
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.29.2
+typer==0.9.0
+typing-inspect==0.8.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+urllib3==1.26.6
+uvicorn==0.22.0
+watchfiles==0.19.0
+wcwidth==0.2.6
+websockets==11.0.3
+win32-setctime==1.1.0
+wrapt==1.14.1
+xlrd==2.0.1
+XlsxWriter==3.1.2
+yarl==1.9.2
+zstandard==0.21.0

requirements-win10-cpu.txt

@@ -0,0 +1,33 @@
+langchain==0.0.166
+chromadb==0.3.22
+llama-cpp-python==0.1.48
+urllib3==1.26.6
+pdfminer.six==20221105
+InstructorEmbedding
+
+# required by sentence-transformers
+# do not use the following in windows. it will cause
+# "Throws a silent error if function takes more than 5 seconds #3078" issue https://github.com/gradio-app/gradio/issues/3078
+# --extra-index-url https://download.pytorch.org/whl/cpu
+torch
+torchvision
+sentence-transformers
+faiss-cpu
+huggingface_hub
+transformers
+protobuf==3.20.0
+accelerate
+bitsandbytes
+# click
+openpyxl
+loguru
+gradio
+charset-normalizer
+PyPDF2
+epub2txt
+docx2txt
+
+about-time
+openai
+more-itertools
+# tqdm

requirements.txt

@@ -16,7 +16,7 @@ transformers
 protobuf==3.20.0
 accelerate
 bitsandbytes
-click
+# click
 openpyxl
 loguru
 gradio
@@ -28,4 +28,4 @@ docx2txt
 about-time
 openai
 more-itertools
-tqdm
+# tqdm