Upload 3 files

app.py

@@ -4,8 +4,8 @@ from tagger.utils import gradio_copy_text, COPY_ACTION_JS
 from tagger.tagger import convert_danbooru_to_e621_prompt, insert_recom_prompt
 from genimage import generate_image
 from llmdolphin import (get_llm_formats, get_dolphin_model_format,
-    get_dolphin_models, get_dolphin_model_info, select_dolphin_model,
-    select_dolphin_format, add_dolphin_models, get_dolphin_sysprompt,
+    get_dolphin_models, get_dolphin_model_info, select_dolphin_model, get_dolphin_loras, select_dolphin_lora,
+    add_dolphin_loras, select_dolphin_format, add_dolphin_models, get_dolphin_sysprompt,
     get_dolphin_sysprompt_mode, select_dolphin_sysprompt, get_dolphin_languages,
     select_dolphin_language, dolphin_respond, dolphin_parse, respond_playground)
 
@@ -27,7 +27,7 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
                     chat_submit = gr.Button("Send", scale=1, variant="primary")
                     chat_clear = gr.Button("Clear", scale=1, variant="secondary")
                 with gr.Accordion("Additional inputs", open=False):
-                    chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0][1]), label="Message format")
+                    chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0]), label="Message format")
                     chat_sysmsg = gr.Textbox(value=get_dolphin_sysprompt(), label="System message")
                     with gr.Row():
                         chat_tokens = gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max tokens")
@@ -35,13 +35,18 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
                         chat_topp = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p")
                         chat_topk = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k")
                         chat_rp = gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
+                    with gr.Accordion("Loras", open=True, visible=False):
+                        chat_lora = gr.Dropdown(choices=get_dolphin_loras(), value=get_dolphin_loras()[0], allow_custom_value=True, label="Lora")
+                        chat_lora_scale = gr.Slider(minimum=0.0, maximum=1.0, value=1.0, step=0.01, label="Lora scale")
+                        chat_add_lora_text = gr.Textbox(label="URL or Repo ID", placeholder="https://huggingface.co/ggml-org/LoRA-Qwen2.5-14B-Instruct-abliterated-v2-F16-GGUF/blob/main/LoRA-Qwen2.5-14B-Instruct-abliterated-v2-f16.gguf", lines=1)
+                        chat_add_lora_submit = gr.Button("Update lists of loras")
                 with gr.Accordion("Add models", open=False):
                     chat_add_text = gr.Textbox(label="URL or Repo ID", placeholder="https://huggingface.co/mradermacher/MagnumChronos-i1-GGUF/blob/main/MagnumChronos.i1-Q4_K_M.gguf", lines=1)
                     chat_add_format = gr.Dropdown(choices=get_llm_formats(), value=get_llm_formats()[0], label="Message format")
                     chat_add_submit = gr.Button("Update lists of models")
                 with gr.Accordion("Modes", open=True):
-                    chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0][1], allow_custom_value=True, label="Model")
-                    chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0][1]), label="Model info")
+                    chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0], allow_custom_value=True, label="Model")
+                    chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0]), label="Model info")
                     with gr.Row():
                         chat_mode = gr.Dropdown(choices=get_dolphin_sysprompt_mode(), value=get_dolphin_sysprompt_mode()[0], allow_custom_value=False, label="Mode")
                         chat_lang = gr.Dropdown(choices=get_dolphin_languages(), value="English", allow_custom_value=True, label="Output language")
@@ -68,9 +73,9 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
                     Don't worry about the strange appearance, **it's just a bug of Gradio!**""", elem_classes="title")
         pg_chatbot = gr.Chatbot(scale=1, show_copy_button=True, show_share_button=False)
         with gr.Accordion("Additional inputs", open=False):
-            pg_chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0][1], allow_custom_value=True, label="Model")
-            pg_chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0][1]), label="Model info")
-            pg_chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0][1]), label="Message format")
+            pg_chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0], allow_custom_value=True, label="Model")
+            pg_chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0]), label="Model info")
+            pg_chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0]), label="Message format")
             pg_chat_sysmsg = gr.Textbox(value="You are a helpful assistant.", label="System message")
             with gr.Row():
                 pg_chat_tokens = gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens")
@@ -78,6 +83,11 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
                 pg_chat_topp = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p")
                 pg_chat_topk = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k")
                 pg_chat_rp = gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
+                with gr.Accordion("Loras", open=True, visible=False):
+                    pg_chat_lora = gr.Dropdown(choices=get_dolphin_loras(), value=get_dolphin_loras()[0], allow_custom_value=True, label="Lora")
+                    pg_chat_lora_scale = gr.Slider(minimum=0.0, maximum=1.0, value=1.0, step=0.01, label="Lora scale")
+                    pg_chat_add_lora_text = gr.Textbox(label="URL or Repo ID", placeholder="https://huggingface.co/ggml-org/LoRA-Qwen2.5-14B-Instruct-abliterated-v2-F16-GGUF/blob/main/LoRA-Qwen2.5-14B-Instruct-abliterated-v2-f16.gguf", lines=1)
+                    pg_chat_add_lora_submit = gr.Button("Update lists of loras")
             with gr.Accordion("Add models", open=True):
                 pg_chat_add_text = gr.Textbox(label="URL or Repo ID", placeholder="https://huggingface.co/mradermacher/MagnumChronos-i1-GGUF/blob/main/MagnumChronos.i1-Q4_K_M.gguf", lines=1)
                 pg_chat_add_format = gr.Dropdown(choices=get_llm_formats(), value=get_llm_formats()[0], label="Message format")
@@ -90,7 +100,8 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
             #clear_btn="Clear",
             submit_btn="Send",
             #additional_inputs_accordion='gr.Accordion(label="Additional Inputs", open=False)',
-            additional_inputs=[pg_chat_model, pg_chat_sysmsg, pg_chat_tokens, pg_chat_temperature, pg_chat_topp, pg_chat_topk, pg_chat_rp, state],
+            additional_inputs=[pg_chat_model, pg_chat_sysmsg, pg_chat_tokens, pg_chat_temperature, pg_chat_topp, pg_chat_topk, pg_chat_rp,
+                               pg_chat_lora, pg_chat_lora_scale, state],
             chatbot=pg_chatbot
         )
     gr.LoginButton()
@@ -99,7 +110,7 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
     gr.on(
         triggers=[chat_msg.submit, chat_submit.click],
         fn=dolphin_respond,
-        inputs=[chat_msg, chatbot, chat_model, chat_sysmsg, chat_tokens, chat_temperature, chat_topp, chat_topk, chat_rp, state],
+        inputs=[chat_msg, chatbot, chat_model, chat_sysmsg, chat_tokens, chat_temperature, chat_topp, chat_topk, chat_rp, chat_lora, chat_lora_scale, state],
         outputs=[chatbot],
         queue=True,
         show_progress="full",
@@ -113,6 +124,8 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
     .success(lambda: None, None, chatbot, queue=False)
     chat_format.change(select_dolphin_format, [chat_format, state], [chat_format, state], queue=False)\
     .success(lambda: None, None, chatbot, queue=False)
+    chat_lora.change(select_dolphin_lora, [chat_lora, state], [chat_lora, state], queue=True, show_progress="full")\
+    .success(lambda: None, None, chatbot, queue=False)
     chat_mode.change(select_dolphin_sysprompt, [chat_mode, state], [chat_sysmsg, state], queue=False)
     chat_lang.change(select_dolphin_language, [chat_lang, state], [chat_sysmsg, state], queue=False)
     gr.on(
@@ -123,6 +136,14 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
         queue=True,
         trigger_mode="once",
     )
+    gr.on(
+        triggers=[chat_add_lora_text.submit, chat_add_lora_submit.click],
+        fn=add_dolphin_loras,
+        inputs=[chat_add_lora_text],
+        outputs=[chat_lora],
+        queue=True,
+        trigger_mode="once",
+    )
 
     copy_btn.click(gradio_copy_text, [output_text], js=COPY_ACTION_JS)
     copy_btn_pony.click(gradio_copy_text, [output_text_pony], js=COPY_ACTION_JS)
@@ -131,6 +152,7 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
 
     pg_chat_model.change(select_dolphin_model, [pg_chat_model, state], [pg_chat_model, pg_chat_format, pg_chat_model_info, state], queue=True, show_progress="full")
     pg_chat_format.change(select_dolphin_format, [pg_chat_format, state], [pg_chat_format, state], queue=False)
+    pg_chat_lora.change(select_dolphin_lora, [pg_chat_lora, state], [pg_chat_lora, state], queue=True, show_progress="full")
     gr.on(
         triggers=[pg_chat_add_text.submit, pg_chat_add_submit.click],
         fn=add_dolphin_models,
@@ -139,6 +161,15 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
         queue=True,
         trigger_mode="once",
     )
+    gr.on(
+        triggers=[pg_chat_add_lora_text.submit, pg_chat_add_lora_submit.click],
+        fn=add_dolphin_loras,
+        inputs=[pg_chat_add_lora_text],
+        outputs=[pg_chat_lora],
+        queue=True,
+        trigger_mode="once",
+    )
+
 
 if __name__ == "__main__":
     app.queue()

llmdolphin.py

@@ -4,6 +4,8 @@ from pathlib import Path
 import re
 import torch
 import gc
+import os
+import urllib
 from typing import Any
 from huggingface_hub import hf_hub_download, HfApi
 from llama_cpp import Llama
@@ -15,14 +17,17 @@ from ja_to_danbooru.ja_to_danbooru import jatags_to_danbooru_tags
 import wrapt_timeout_decorator
 from llama_cpp_agent.messages_formatter import MessagesFormatter
 from formatter import mistral_v1_formatter, mistral_v2_formatter, mistral_v3_tekken_formatter
-from llmenv import llm_models, llm_models_dir, llm_formats, llm_languages, dolphin_system_prompt
+from llmenv import llm_models, llm_models_dir, llm_loras, llm_loras_dir, llm_formats, llm_languages, dolphin_system_prompt
 import subprocess
 subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
 
 
-llm_models_tupled_list = []
+llm_models_list = []
+llm_loras_list = []
 default_llm_model_filename = list(llm_models.keys())[0]
+default_llm_lora_filename = list(llm_loras.keys())[0]
 device = "cuda" if torch.cuda.is_available() else "cpu"
+HF_TOKEN = os.getenv("HF_TOKEN", False)
 
 
 def to_list(s: str):
@@ -68,43 +73,170 @@ def is_japanese(s: str):
     return False
 
 
-def update_llm_model_tupled_list():
-    global llm_models_tupled_list
-    llm_models_tupled_list = []
-    for k, v in llm_models.items():
-        name = k
-        value = k
-        llm_models_tupled_list.append((name, value))
+def get_dir_size(path: str):
+    total = 0
+    with os.scandir(path) as it:
+        for entry in it:
+            if entry.is_file():
+                total += entry.stat().st_size
+            elif entry.is_dir():
+                total += get_dir_size(entry.path)
+    return total
+
+
+def get_dir_size_gb(path: str):
+    try:
+        size_gb = get_dir_size(path) / (1024 ** 3)
+        print(f"Dir size: {size_gb:.2f} GB ({path})")
+    except Exception as e:
+        size_gb = 999
+        print(f"Error while retrieving the used storage: {e}.")
+    finally:
+        return size_gb
+
+
+def clean_dir(path: str, size_gb: float, limit_gb: float):
+    try:
+        files = os.listdir(path)
+        files = [os.path.join(path, f) for f in files if f.endswith(".gguf") and default_llm_model_filename not in f and default_llm_lora_filename not in f]
+        files.sort(key=os.path.getatime, reverse=False)
+        req_bytes = int((size_gb - limit_gb) * (1024 ** 3))
+        for file in files:
+            if req_bytes < 0: break
+            size = os.path.getsize(file)
+            Path(file).unlink()
+            req_bytes -= size
+            print(f"Deleted: {file}")
+    except Exception as e:
+        print(e)
+
+
+def update_storage(path: str, limit_gb: float=50.0):
+    size_gb = get_dir_size_gb(path)
+    if size_gb > limit_gb:
+        print("Cleaning storage...")
+        clean_dir(path, size_gb, limit_gb)
+        #get_dir_size_gb(path)
+
+
+def split_hf_url(url: str):
+    try:
+        s = list(re.findall(r'^(?:https?://huggingface.co/)(?:(datasets|spaces)/)?(.+?/.+?)/\w+?/.+?/(?:(.+)/)?(.+?.\w+)(?:\?download=true)?$', url)[0])
+        if len(s) < 4: return "", "", "", ""
+        repo_id = s[1]
+        if s[0] == "datasets": repo_type = "dataset"
+        elif s[0] == "spaces": repo_type = "space"
+        else: repo_type = "model"
+        subfolder = urllib.parse.unquote(s[2]) if s[2] else None
+        filename = urllib.parse.unquote(s[3])
+        return repo_id, filename, subfolder, repo_type
+    except Exception as e:
+        print(e)
+
+
+def hf_url_exists(url: str):
+    hf_token = HF_TOKEN
+    repo_id, filename, subfolder, repo_type = split_hf_url(url)
+    api = HfApi(token=hf_token)
+    return api.file_exists(repo_id=repo_id, filename=filename, repo_type=repo_type, token=hf_token)
+
+
+def get_repo_type(repo_id: str):
+    try:
+        api = HfApi(token=HF_TOKEN)
+        if api.repo_exists(repo_id=repo_id, repo_type="dataset", token=HF_TOKEN): return "dataset"
+        elif api.repo_exists(repo_id=repo_id, repo_type="space", token=HF_TOKEN): return "space"
+        elif api.repo_exists(repo_id=repo_id, token=HF_TOKEN): return "model"
+        else: return None
+    except Exception as e:
+        print(e)
+        raise Exception(f"Repo not found: {repo_id} {e}")
+
+
+def get_hf_blob_url(repo_id: str, repo_type: str, path: str):
+    if repo_type == "model": return f"https://huggingface.co/{repo_id}/blob/main/{path}"
+    elif repo_type == "dataset": return f"https://huggingface.co/datasets/{repo_id}/blob/main/{path}"
+    elif repo_type == "space": return f"https://huggingface.co/spaces/{repo_id}/blob/main/{path}"
+
+
+def get_gguf_url(s: str):
+    def find_gguf(d: dict, keys: dict):
+        paths = []
+        for key, size in keys.items():
+            if size != 0: l = [p for p, s in d.items() if key.lower() in p.lower() and s < size]
+            else: l = [p for p in d.keys() if key.lower() in p.lower()]
+            if len(l) > 0: paths.append(l[0])
+        if len(paths) > 0: return paths[0]
+        return list(d.keys())[0]
+
+    try:
+        if s.lower().endswith(".gguf"): return s
+        repo_type = get_repo_type(s)
+        if repo_type is None: return s
+        repo_id = s
+        api = HfApi(token=HF_TOKEN)
+        gguf_dict = {i.path: i.size for i in api.list_repo_tree(repo_id=repo_id, repo_type=repo_type, recursive=True, token=HF_TOKEN) if i.path.endswith(".gguf")}
+        if len(gguf_dict) == 0: return s
+        return get_hf_blob_url(repo_id, repo_type, find_gguf(gguf_dict, {"Q5_K_M": 6000000000, "Q4_K_M": 0, "Q4": 0}))
+    except Exception as e:
+        print(e)
+        return s
+
+
+def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
+    hf_token = HF_TOKEN
+    repo_id, filename, subfolder, repo_type = split_hf_url(url)
+    try:
+        print(f"Downloading {url} to {directory}")
+        if subfolder is not None: path = hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder, repo_type=repo_type, local_dir=directory, token=hf_token)
+        else: path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type, local_dir=directory, token=hf_token)
+        return path
+    except Exception as e:
+        print(f"Failed to download: {e}")
+        return None
+
+
+def update_llm_model_list():
+    global llm_models_list
+    llm_models_list = []
+    for k in llm_models.keys():
+        llm_models_list.append(k)
     model_files = Path(llm_models_dir).glob('*.gguf')
     for path in model_files:
-        name = path.name
-        value = path.name
-        llm_models_tupled_list.append((name, value))
-    llm_models_tupled_list = list_uniq(llm_models_tupled_list)
-    return llm_models_tupled_list
-
-
-def download_llm_models():
-    global llm_models_tupled_list
-    llm_models_tupled_list = []
-    for k, v in llm_models.items():
-        try:
-            hf_hub_download(repo_id = v[0], filename = k, local_dir = llm_models_dir)
-        except Exception:
-            continue
-        name = k
-        value = k
-        llm_models_tupled_list.append((name, value))
+        llm_models_list.append(path.name)
+    llm_models_list = list_uniq(llm_models_list)
+    return llm_models_list
 
 
 def download_llm_model(filename: str):
-    if not filename in llm_models.keys(): return default_llm_model_filename
+    if filename not in llm_models.keys(): return default_llm_model_filename
     try:
-        hf_hub_download(repo_id = llm_models[filename][0], filename = filename, local_dir = llm_models_dir)
+        hf_hub_download(repo_id=llm_models[filename][0], filename=filename, local_dir=llm_models_dir, token=HF_TOKEN)
     except Exception as e:
         print(e)
         return default_llm_model_filename
-    update_llm_model_tupled_list()
+    update_llm_model_list()
+    return filename
+
+
+def update_llm_lora_list():
+    global llm_loras_list
+    llm_loras_list = list(llm_loras.keys()).copy()
+    model_files = Path(llm_loras_dir).glob('*.gguf')
+    for path in model_files:
+        llm_loras_list.append(path.name)
+    llm_loras_list = list_uniq([""] + llm_loras_list)
+    return llm_loras_list
+
+
+def download_llm_lora(filename: str):
+    if not filename in llm_loras.keys(): return ""
+    try:
+        download_hf_file(llm_loras_dir, llm_loras[filename])
+    except Exception as e:
+        print(e)
+        return ""
+    update_llm_lora_list()
     return filename
 
 
@@ -122,9 +254,18 @@ def select_dolphin_model(filename: str, state: dict, progress=gr.Progress(track_
     value = download_llm_model(filename)
     progress(1, desc="Model loaded.")
     md = get_dolphin_model_info(filename)
+    update_storage(llm_models_dir)
     return gr.update(value=value, choices=get_dolphin_models()), gr.update(value=get_dolphin_model_format(value)), gr.update(value=md), state
 
 
+def select_dolphin_lora(filename: str, state: dict, progress=gr.Progress(track_tqdm=True)):
+    progress(0, desc="Loading lora...")
+    value = download_llm_lora(filename)
+    progress(1, desc="Lora loaded.")
+    update_storage(llm_loras_dir)
+    return gr.update(value=value, choices=get_dolphin_loras()), state
+
+
 def select_dolphin_format(format_name: str, state: dict):
     set_state(state, "override_llm_format", llm_formats[format_name])
     return gr.update(value=format_name), state
@@ -134,7 +275,11 @@ download_llm_model(default_llm_model_filename)
 
 
 def get_dolphin_models():
-    return update_llm_model_tupled_list()
+    return update_llm_model_list()
+
+
+def get_dolphin_loras():
+    return update_llm_lora_list()
 
 
 def get_llm_formats():
@@ -157,33 +302,41 @@ def get_dolphin_model_format(filename: str):
 
 def add_dolphin_models(query: str, format_name: str):
     global llm_models
-    api = HfApi()
-    add_models = {}
-    format = llm_formats[format_name]
-    filename = ""
-    repo = ""
     try:
-        s = list(re.findall(r'^(?:https?://huggingface.co/)?(.+?/.+?)(?:/.*/(.+?.gguf).*?)?$', query)[0])
-        if s and  "" in s: s.remove("")
-        if len(s) == 1:
-            repo = s[0]
-            if not api.repo_exists(repo_id = repo): return gr.update()
-            files = api.list_repo_files(repo_id = repo)
-            for file in files:
-                if str(file).endswith(".gguf"): add_models[filename] = [repo, format]
-        elif len(s) >= 2:
-            repo = s[0]
-            filename = s[1]
-            if not api.repo_exists(repo_id = repo) or not api.file_exists(repo_id = repo, filename = filename): return gr.update()
-            add_models[filename] = [repo, format]
+        add_models = {}
+        format = llm_formats[format_name]
+        filename = ""
+        repo = ""
+        query = get_gguf_url(query)
+        if hf_url_exists(query):
+            s = list(re.findall(r'^https?://huggingface.co/(.+?/.+?)/(?:blob|resolve)/main/(.+.gguf)(?:\?download=true)?$', query)[0])
+            if len(s) == 2:
+                repo = s[0]
+                filename = s[1]
+                add_models[filename] = [repo, format]
         else: return gr.update()
     except Exception as e:
         print(e)
         return gr.update()
     llm_models = (llm_models | add_models).copy()
-    update_llm_model_tupled_list()
+    update_llm_model_list()
     choices = get_dolphin_models()
-    return gr.update(choices=choices, value=choices[-1][1])
+    return gr.update(choices=choices, value=choices[-1])
+
+
+def add_dolphin_loras(query: str):
+    global llm_loras
+    try:
+        add_loras = {}
+        query = get_gguf_url(query)
+        if hf_url_exists(query): add_loras[Path(query).name] = query
+    except Exception as e:
+        print(e)
+        return gr.update()
+    llm_loras = (llm_loras | add_loras).copy()
+    update_llm_lora_list()
+    choices = get_dolphin_loras()
+    return gr.update(choices=choices, value=choices[-1])
 
 
 def get_dolphin_sysprompt(state: dict={}):
@@ -221,6 +374,7 @@ def get_raw_prompt(msg: str):
     return re.sub(r'[*/:_"#\n]', ' ', ", ".join(m)).lower() if m else ""
 
 
+# https://llama-cpp-python.readthedocs.io/en/latest/api-reference/
 @torch.inference_mode()
 @spaces.GPU(duration=59)
 def dolphin_respond(
@@ -233,6 +387,8 @@ def dolphin_respond(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    lora: str = "",
+    lora_scale: float = 1.0,
     state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
@@ -244,12 +400,18 @@ def dolphin_respond(
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
+        kwargs = {}
+        if lora:
+            kwargs["lora_path"] = str(Path(f"{llm_loras_dir}/{lora}"))
+            kwargs["lora_scale"] = lora_scale
+        else:
+            kwargs["flash_attn"] = True
         llm = Llama(
             model_path=str(model_path),
-            flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
             n_ctx=8192, #8192
+            **kwargs,
         )
         provider = LlamaCppPythonProvider(llm)
 
@@ -339,6 +501,8 @@ def dolphin_respond_auto(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    lora: str = "",
+    lora_scale: float = 1.0,
     state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
@@ -351,12 +515,18 @@ def dolphin_respond_auto(
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
+        kwargs = {}
+        if lora:
+            kwargs["lora_path"] = str(Path(f"{llm_loras_dir}/{lora}"))
+            kwargs["lora_scale"] = lora_scale
+        else:
+            kwargs["flash_attn"] = True
         llm = Llama(
             model_path=str(model_path),
-            flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
             n_ctx=8192, #8192
+            **kwargs,
         )
         provider = LlamaCppPythonProvider(llm)
 
@@ -452,6 +622,8 @@ def respond_playground(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    lora: str = "",
+    lora_scale: float = 1.0,
     state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
@@ -462,12 +634,18 @@ def respond_playground(
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
+        kwargs = {}
+        if lora:
+            kwargs["lora_path"] = str(Path(f"{llm_loras_dir}/{lora}"))
+            kwargs["lora_scale"] = lora_scale
+        else:
+            kwargs["flash_attn"] = True
         llm = Llama(
             model_path=str(model_path),
-            flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
             n_ctx=8192, #8192
+            **kwargs,
         )
         provider = LlamaCppPythonProvider(llm)
 

llmenv.py

@@ -1,5 +1,7 @@
 from llama_cpp_agent import MessagesFormatterType
 from formatter import mistral_v1_formatter, mistral_v2_formatter, mistral_v3_tekken_formatter
+from pathlib import Path
+
 
 llm_models = {
     #"": ["", MessagesFormatterType.LLAMA_3],
@@ -96,6 +98,20 @@ llm_models = {
     #"": ["", MessagesFormatterType.OPEN_CHAT],
     #"": ["", MessagesFormatterType.CHATML],
     #"": ["", MessagesFormatterType.PHI_3],
+    "MN-12B-solracht-EXPERIMENTAL-011425.Q4_K_M.gguf": ["mradermacher/MN-12B-solracht-EXPERIMENTAL-011425-GGUF", MessagesFormatterType.MISTRAL],
+    "Llamaverse-3.1-8B-Instruct.Q5_K_M.gguf": ["mradermacher/Llamaverse-3.1-8B-Instruct-GGUF", MessagesFormatterType.LLAMA_3],
+    "Morphing-8B-Model_Stock.Q5_K_M.gguf": ["mradermacher/Morphing-8B-Model_Stock-GGUF", MessagesFormatterType.LLAMA_3],
+    "Not_Even_My_Final_Form-8B-Model_Stock.Q5_K_M.gguf": ["mradermacher/Not_Even_My_Final_Form-8B-Model_Stock-GGUF", MessagesFormatterType.LLAMA_3],
+    "Qwen2.5-7B-sft-ultrachat.Q5_K_M.gguf": ["mradermacher/Qwen2.5-7B-sft-ultrachat-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Kosmos-EVAA-Franken-Immersive-v40-8B.Q5_K_M.gguf": ["mradermacher/Kosmos-EVAA-Franken-Immersive-v40-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "light-7b-beta.Q5_K_M.gguf": ["mradermacher/light-7b-beta-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "light-3B-beta.Q5_K_M.gguf": ["mradermacher/light-3B-beta-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Magnolia-v4-12B.Q4_K_M.gguf": ["mradermacher/Magnolia-v4-12B-GGUF", MessagesFormatterType.MISTRAL],
+    "Darkest-muse-v1-lorablated-v2.i1-Q4_K_M.gguf": ["mradermacher/Darkest-muse-v1-lorablated-v2-i1-GGUF", MessagesFormatterType.ALPACA],
+    "Eunoia-Gemma-9B-o1-Indo.Q4_K_M.gguf": ["mradermacher/Eunoia-Gemma-9B-o1-Indo-GGUF", MessagesFormatterType.ALPACA],
+    "VISION-1.Q5_K_M.gguf": ["mradermacher/VISION-1-GGUF", MessagesFormatterType.LLAMA_3],
+    "RigoChat-7b-v2.i1-Q5_K_M.gguf": ["mradermacher/RigoChat-7b-v2-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "italy-10b-q5_k_m.gguf": ["ClaudioItaly/Italy-10B-Q5_K_M-GGUF", MessagesFormatterType.ALPACA],
     "SJT-14B.Q4_K_M.gguf": ["mradermacher/SJT-14B-GGUF", MessagesFormatterType.OPEN_CHAT],
     "Hermes-Llama-3.2-CoT-Summary.Q5_K_M.gguf": ["mradermacher/Hermes-Llama-3.2-CoT-Summary-GGUF", MessagesFormatterType.LLAMA_3],
     "Rombo-LLM-V2.5-Qwen-7b.Q5_K_M.gguf": ["mradermacher/Rombo-LLM-V2.5-Qwen-7b-GGUF", MessagesFormatterType.OPEN_CHAT],
@@ -2080,7 +2096,14 @@ llm_models = {
 }
 
 
+llm_loras_urls = [
+    "https://huggingface.co/ggml-org/LoRA-Qwen2.5-32B-Instruct-abliterated-F16-GGUF/blob/main/LoRA-Qwen2.5-32B-Instruct-abliterated-f16.gguf",
+]
+llm_loras = {str(Path(u).name): u for u in llm_loras_urls}
+
+
 llm_models_dir = "./llm_models"
+llm_loras_dir = "./llm_loras"
 
 
 llm_formats = {