Update with h2oGPT hash 3513278043665f503945eb05d56c1ec1152d1006

generate.py

@@ -33,7 +33,6 @@ from typing import Union
 
 import fire
 import torch
-from peft import PeftModel
 from transformers import GenerationConfig, AutoModel, TextIteratorStreamer
 from accelerate import init_empty_weights, infer_auto_device_map
 
@@ -710,6 +709,7 @@ def get_model(
                 base_model,
                 **model_kwargs
             )
+            from peft import PeftModel  # loads cuda, so avoid in global scope
             model = PeftModel.from_pretrained(
                 model,
                 lora_weights,
@@ -727,6 +727,7 @@ def get_model(
                     base_model,
                     **model_kwargs
                 )
+                from peft import PeftModel  # loads cuda, so avoid in global scope
                 model = PeftModel.from_pretrained(
                     model,
                     lora_weights,
@@ -827,24 +828,27 @@ no_default_param_names = [
     'iinput_nochat',
 ]
 
+gen_hyper = ['temperature',
+             'top_p',
+             'top_k',
+             'num_beams',
+             'max_new_tokens',
+             'min_new_tokens',
+             'early_stopping',
+             'max_time',
+             'repetition_penalty',
+             'num_return_sequences',
+             'do_sample',
+             ]
+
 eval_func_param_names = ['instruction',
                          'iinput',
                          'context',
                          'stream_output',
                          'prompt_type',
-                         'prompt_dict',
-                         'temperature',
-                         'top_p',
-                         'top_k',
-                         'num_beams',
-                         'max_new_tokens',
-                         'min_new_tokens',
-                         'early_stopping',
-                         'max_time',
-                         'repetition_penalty',
-                         'num_return_sequences',
-                         'do_sample',
-                         'chat',
+                         'prompt_dict'] + \
+                        gen_hyper + \
+                        ['chat',
                          'instruction_nochat',
                          'iinput_nochat',
                          'langchain_mode',
@@ -900,6 +904,9 @@ def evaluate_from_str(
     # only used for submit_nochat_api
     user_kwargs['chat'] = False
     user_kwargs['stream_output'] = False
+    if 'langchain_mode' not in user_kwargs:
+        # if user doesn't specify, then assume disabled, not use default
+        user_kwargs['langchain_mode'] = 'Disabled'
 
     assert set(list(default_kwargs.keys())) == set(eval_func_param_names)
     # correct ordering.  Note some things may not be in default_kwargs, so can't be default of user_kwargs.get()
@@ -1083,7 +1090,6 @@ def evaluate(
                            db=db1,
                            user_path=user_path,
                            detect_user_path_changes_every_query=detect_user_path_changes_every_query,
-                           max_new_tokens=max_new_tokens,
                            cut_distanct=1.1 if langchain_mode in ['wiki_full'] else 1.64,  # FIXME, too arbitrary
                            use_openai_embedding=use_openai_embedding,
                            use_openai_model=use_openai_model,
@@ -1096,10 +1102,20 @@ def evaluate(
                            document_choice=document_choice,
                            db_type=db_type,
                            top_k_docs=top_k_docs,
+
+                           # gen_hyper:
+                           do_sample=do_sample,
                            temperature=temperature,
                            repetition_penalty=repetition_penalty,
                            top_k=top_k,
                            top_p=top_p,
+                           num_beams=num_beams,
+                           min_new_tokens=min_new_tokens,
+                           max_new_tokens=max_new_tokens,
+                           early_stopping=early_stopping,
+                           max_time=max_time,
+                           num_return_sequences=num_return_sequences,
+
                            prompt_type=prompt_type,
                            prompt_dict=prompt_dict,
                            n_jobs=n_jobs,

gpt_langchain.py

@@ -22,6 +22,7 @@ from langchain.embeddings import HuggingFaceInstructEmbeddings
 from tqdm import tqdm
 
 from enums import DocumentChoices
+from generate import gen_hyper
 from prompter import non_hf_types, PromptType
 from utils import wrapped_partial, EThread, import_matplotlib, sanitize_filename, makedirs, get_url, flatten_list, \
     get_device, ProgressParallel, remove, hash_file, clear_torch_cache
@@ -261,11 +262,17 @@ def get_answer_from_sources(chain, sources, question):
 
 def get_llm(use_openai_model=False, model_name=None, model=None,
             tokenizer=None, stream_output=False,
-            max_new_tokens=256,
+            do_sample=False,
             temperature=0.1,
-            repetition_penalty=1.0,
             top_k=40,
             top_p=0.7,
+            num_beams=1,
+            max_new_tokens=256,
+            min_new_tokens=1,
+            early_stopping=False,
+            max_time=180,
+            repetition_penalty=1.0,
+            num_return_sequences=1,
             prompt_type=None,
             prompt_dict=None,
             prompter=None,
@@ -312,10 +319,20 @@ def get_llm(use_openai_model=False, model_name=None, model=None,
                                                              load_in_8bit=load_8bit)
 
         max_max_tokens = tokenizer.model_max_length
-        gen_kwargs = dict(max_new_tokens=max_new_tokens,
+        gen_kwargs = dict(do_sample=do_sample,
+                          temperature=temperature,
+                          top_k=top_k,
+                          top_p=top_p,
+                          num_beams=num_beams,
+                          max_new_tokens=max_new_tokens,
+                          min_new_tokens=min_new_tokens,
+                          early_stopping=early_stopping,
+                          max_time=max_time,
+                          repetition_penalty=repetition_penalty,
+                          num_return_sequences=num_return_sequences,
                           return_full_text=True,
-                          early_stopping=False,
                           handle_long_generation='hole')
+        assert len(set(gen_hyper).difference(gen_kwargs.keys())) == 0
 
         if stream_output:
             skip_prompt = False
@@ -1235,11 +1252,17 @@ def _run_qa_db(query=None,
                show_rank=False,
                load_db_if_exists=False,
                db=None,
-               max_new_tokens=256,
+               do_sample=False,
                temperature=0.1,
-               repetition_penalty=1.0,
                top_k=40,
                top_p=0.7,
+               num_beams=1,
+               max_new_tokens=256,
+               min_new_tokens=1,
+               early_stopping=False,
+               max_time=180,
+               repetition_penalty=1.0,
+               num_return_sequences=1,
                langchain_mode=None,
                document_choice=[DocumentChoices.All_Relevant.name],
                n_jobs=-1,
@@ -1274,14 +1297,21 @@ def _run_qa_db(query=None,
             assert prompt_dict is not None  # should at least be {} or ''
         else:
             prompt_dict = ''
+    assert len(set(gen_hyper).difference(inspect.signature(get_llm).parameters)) == 0
     llm, model_name, streamer, prompt_type_out = get_llm(use_openai_model=use_openai_model, model_name=model_name,
                                                          model=model, tokenizer=tokenizer,
                                                          stream_output=stream_output,
-                                                         max_new_tokens=max_new_tokens,
+                                                         do_sample=do_sample,
                                                          temperature=temperature,
-                                                         repetition_penalty=repetition_penalty,
                                                          top_k=top_k,
                                                          top_p=top_p,
+                                                         num_beams=num_beams,
+                                                         max_new_tokens=max_new_tokens,
+                                                         min_new_tokens=min_new_tokens,
+                                                         early_stopping=early_stopping,
+                                                         max_time=max_time,
+                                                         repetition_penalty=repetition_penalty,
+                                                         num_return_sequences=num_return_sequences,
                                                          prompt_type=prompt_type,
                                                          prompt_dict=prompt_dict,
                                                          prompter=prompter,
@@ -1609,6 +1639,7 @@ def get_some_dbs_from_hf(dest='.', db_zips=None):
             assert os.path.isdir(os.path.join(dest, dir_expected)), "Missing path for %s" % dir_expected
             assert os.path.isdir(os.path.join(dest, dir_expected, 'index')), "Missing index in %s" % dir_expected
 
+
 def _create_local_weaviate_client():
     WEAVIATE_URL = os.getenv('WEAVIATE_URL', "http://localhost:8080")
     WEAVIATE_USERNAME = os.getenv('WEAVIATE_USERNAME')
@@ -1629,5 +1660,6 @@ def _create_local_weaviate_client():
         print(f"Failed to create Weaviate client: {e}")
         return None
 
+
 if __name__ == '__main__':
     pass

gradio_runner.py

@@ -649,7 +649,7 @@ def go_gradio(**kwargs):
                                    inputs=[fileup_output, my_db_state, add_to_shared_db_btn, add_to_my_db_btn,
                                            chunk, chunk_size],
                                    outputs=[add_to_shared_db_btn, add_to_my_db_btn, sources_text], queue=queue,
-                                   api_name='add_to_shared' if allow_api else None) \
+                                   api_name='add_to_shared' if allow_api and allow_upload_to_user_data else None) \
             .then(clear_file_list, outputs=fileup_output, queue=queue) \
             .then(update_radio_to_user, inputs=None, outputs=langchain_mode, queue=False)
 
@@ -664,7 +664,7 @@ def go_gradio(**kwargs):
                            inputs=[url_text, my_db_state, add_to_shared_db_btn, add_to_my_db_btn,
                                    chunk, chunk_size],
                            outputs=[add_to_shared_db_btn, add_to_my_db_btn, sources_text], queue=queue,
-                           api_name='add_url_to_shared' if allow_api else None) \
+                           api_name='add_url_to_shared' if allow_api and allow_upload_to_user_data else None) \
             .then(clear_textbox, outputs=url_text, queue=queue) \
             .then(update_radio_to_user, inputs=None, outputs=langchain_mode, queue=False)
 
@@ -673,7 +673,7 @@ def go_gradio(**kwargs):
                                  inputs=[user_text_text, my_db_state, add_to_shared_db_btn, add_to_my_db_btn,
                                          chunk, chunk_size],
                                  outputs=[add_to_shared_db_btn, add_to_my_db_btn, sources_text], queue=queue,
-                                 api_name='add_text_to_shared' if allow_api else None) \
+                                 api_name='add_text_to_shared' if allow_api and allow_upload_to_user_data else None) \
             .then(clear_textbox, outputs=user_text_text, queue=queue) \
             .then(update_radio_to_user, inputs=None, outputs=langchain_mode, queue=False)
 
@@ -695,7 +695,7 @@ def go_gradio(**kwargs):
                                inputs=[fileup_output, my_db_state, add_to_shared_db_btn, add_to_my_db_btn,
                                        chunk, chunk_size],
                                outputs=[my_db_state, add_to_shared_db_btn, add_to_my_db_btn, sources_text], queue=queue,
-                               api_name='add_to_my' if allow_api else None) \
+                               api_name='add_to_my' if allow_api and allow_upload_to_my_data else None) \
             .then(clear_file_list, outputs=fileup_output, queue=queue) \
             .then(update_radio_to_my, inputs=None, outputs=langchain_mode, queue=False)
         # .then(make_invisible, outputs=add_to_shared_db_btn, queue=queue)
@@ -706,7 +706,7 @@ def go_gradio(**kwargs):
                          inputs=[url_text, my_db_state, add_to_shared_db_btn, add_to_my_db_btn,
                                  chunk, chunk_size],
                          outputs=[my_db_state, add_to_shared_db_btn, add_to_my_db_btn, sources_text], queue=queue,
-                         api_name='add_url_to_my' if allow_api else None) \
+                         api_name='add_url_to_my' if allow_api and allow_upload_to_my_data else None) \
             .then(clear_textbox, outputs=url_text, queue=queue) \
             .then(update_radio_to_my, inputs=None, outputs=langchain_mode, queue=False)
 
@@ -715,7 +715,7 @@ def go_gradio(**kwargs):
                                inputs=[user_text_text, my_db_state, add_to_shared_db_btn, add_to_my_db_btn,
                                        chunk, chunk_size],
                                outputs=[my_db_state, add_to_shared_db_btn, add_to_my_db_btn, sources_text], queue=queue,
-                               api_name='add_txt_to_my' if allow_api else None) \
+                               api_name='add_txt_to_my' if allow_api and allow_upload_to_my_data else None) \
             .then(clear_textbox, outputs=user_text_text, queue=queue) \
             .then(update_radio_to_my, inputs=None, outputs=langchain_mode, queue=False)
 
@@ -1788,6 +1788,8 @@ def get_db(db1, langchain_mode, dbs=None):
 
 def get_source_files_given_langchain_mode(db1, langchain_mode='UserData', dbs=None):
     db = get_db(db1, langchain_mode, dbs=dbs)
+    if langchain_mode in ['ChatLLM', 'LLM'] or db is None:
+        return "Sources: N/A"
     return get_source_files(db=db, exceptions=None)
 
 

requirements.txt

@@ -56,7 +56,8 @@ einops==0.6.1
 instructorembedding==1.0.1
 
 # for gpt4all .env file, but avoid worrying about imports
-python-dotenv==1.0.0# optional for chat with PDF
+python-dotenv==1.0.0
+# optional for chat with PDF
 langchain==0.0.193
 pypdf==3.8.1
 tiktoken==0.3.3

utils.py

@@ -14,7 +14,6 @@ import time
 import traceback
 import zipfile
 from datetime import datetime
-from enum import Enum
 
 import filelock
 import requests, uuid