Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,26 +1,15 @@
|
|
| 1 |
"""
|
| 2 |
-
app.py
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
New (Objective):
|
| 6 |
-
- Radio to choose "Extractive (safe)" vs "Generative (Evo)".
|
| 7 |
-
- Sliders for temperature and max tokens.
|
| 8 |
-
- We pass these to synthesize_with_evo().
|
| 9 |
"""
|
| 10 |
|
| 11 |
from pathlib import Path
|
| 12 |
from typing import List
|
| 13 |
-
|
| 14 |
import gradio as gr
|
| 15 |
|
| 16 |
from indexer import build_index
|
| 17 |
-
from rag_search import
|
| 18 |
-
RAGSearcher,
|
| 19 |
-
format_sources,
|
| 20 |
-
extract_links,
|
| 21 |
-
split_form_links,
|
| 22 |
-
links_markdown,
|
| 23 |
-
)
|
| 24 |
from evo_inference import synthesize_with_evo
|
| 25 |
from utils_lang import SUPPORTED, normalize_lang, L
|
| 26 |
|
|
@@ -28,103 +17,70 @@ _searcher = None
|
|
| 28 |
DATA_SEED_DIR = Path("data/seed")
|
| 29 |
ALLOWED_EXTS = {".txt", ".pdf", ".html", ".htm", ".xhtml"}
|
| 30 |
|
| 31 |
-
|
| 32 |
def ensure_searcher():
|
| 33 |
global _searcher
|
| 34 |
-
if _searcher is None:
|
| 35 |
-
_searcher = RAGSearcher()
|
| 36 |
return _searcher
|
| 37 |
|
| 38 |
-
|
| 39 |
def on_build_index():
|
| 40 |
try:
|
| 41 |
msg = build_index()
|
| 42 |
-
global _searcher
|
| 43 |
-
_searcher = None
|
| 44 |
except Exception as e:
|
| 45 |
msg = f"Error while building index: {e}"
|
| 46 |
return msg
|
| 47 |
|
| 48 |
-
|
| 49 |
def _save_files(files: List[gr.File]) -> List[str]:
|
| 50 |
-
saved =
|
| 51 |
-
|
| 52 |
-
if not files:
|
| 53 |
-
return saved
|
| 54 |
-
|
| 55 |
for f in files:
|
| 56 |
try:
|
| 57 |
path = None
|
| 58 |
-
if isinstance(f, dict) and "name" in f:
|
| 59 |
-
|
| 60 |
-
elif
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
path = Path(f)
|
| 64 |
-
if not path or not path.exists():
|
| 65 |
-
continue
|
| 66 |
-
if path.suffix.lower() not in ALLOWED_EXTS:
|
| 67 |
-
continue
|
| 68 |
dest = DATA_SEED_DIR / path.name
|
| 69 |
dest.write_bytes(path.read_bytes())
|
| 70 |
saved.append(path.name)
|
| 71 |
-
except
|
| 72 |
-
continue
|
| 73 |
return saved
|
| 74 |
|
| 75 |
-
|
| 76 |
-
def on_upload_and_reindex(files) -> str:
|
| 77 |
saved = _save_files(files or [])
|
| 78 |
-
if not saved:
|
| 79 |
-
return "No valid .txt/.pdf/.html uploaded."
|
| 80 |
status = on_build_index()
|
| 81 |
return f"Uploaded: {', '.join(saved)}\n{status}"
|
| 82 |
|
| 83 |
-
|
| 84 |
-
def on_ask(question: str, top_k: int, lang_code: str, mode: str, temp: float, max_tokens: int, history: list):
|
| 85 |
lang = normalize_lang(lang_code)
|
| 86 |
if not question or len(question.strip()) < 3:
|
| 87 |
-
return history, L(lang,
|
| 88 |
-
|
| 89 |
try:
|
| 90 |
searcher = ensure_searcher()
|
| 91 |
hits = searcher.search(question, k=int(top_k))
|
| 92 |
-
|
| 93 |
-
# Map UI mode label to function arg
|
| 94 |
-
use_mode = "generative" if mode.startswith("Generative") else "extractive"
|
| 95 |
-
|
| 96 |
-
answer = synthesize_with_evo(
|
| 97 |
-
question, lang, hits,
|
| 98 |
-
mode=use_mode,
|
| 99 |
-
max_new_tokens=int(max_tokens),
|
| 100 |
-
temperature=float(temp),
|
| 101 |
-
)
|
| 102 |
-
|
| 103 |
srcs = format_sources(hits)
|
| 104 |
-
if srcs.startswith("Sources:"):
|
| 105 |
-
srcs = srcs.replace("Sources:", f"{L(lang, 'sources')}:")
|
| 106 |
-
|
| 107 |
all_links = extract_links(hits)
|
| 108 |
form_links, other_links = split_form_links(all_links)
|
| 109 |
-
links_md = links_markdown(L(lang,
|
| 110 |
-
forms_md = links_markdown(L(lang,
|
| 111 |
-
|
| 112 |
except Exception as e:
|
| 113 |
-
answer = f"Error: {e}\n\n{L(lang,
|
| 114 |
-
srcs = f"{L(lang,
|
| 115 |
-
links_md = f"**{L(lang,
|
| 116 |
-
forms_md = f"**{L(lang,
|
| 117 |
-
|
| 118 |
history = history + [(question, answer)]
|
| 119 |
return history, answer, srcs, links_md, forms_md
|
| 120 |
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
gr.Markdown(f"# 🇲🇺 **{L('en','title')}** — Step 8")
|
| 124 |
|
| 125 |
with gr.Row():
|
| 126 |
-
build_btn = gr.Button(L("en",
|
| 127 |
-
status = gr.Markdown(L("en",
|
| 128 |
build_btn.click(fn=on_build_index, outputs=status)
|
| 129 |
|
| 130 |
with gr.Accordion("Admin: upload TXT / PDF / HTML and auto-reindex", open=False):
|
|
@@ -136,20 +92,10 @@ with gr.Blocks(title=L("en", "title")) as demo:
|
|
| 136 |
gr.Markdown("### Language / Langue / Langaz")
|
| 137 |
lang = gr.Dropdown(choices=list(SUPPORTED.keys()), value="en", label="EN / FR / MFE")
|
| 138 |
|
| 139 |
-
gr.Markdown("### Answer mode & controls")
|
| 140 |
-
with gr.Row():
|
| 141 |
-
mode = gr.Radio(
|
| 142 |
-
choices=["Extractive (safe)", "Generative (Evo)"],
|
| 143 |
-
value="Extractive (safe)",
|
| 144 |
-
label="Mode"
|
| 145 |
-
)
|
| 146 |
-
temp = gr.Slider(0.0, 1.2, value=0.4, step=0.05, label="Temperature (gen)")
|
| 147 |
-
max_tokens = gr.Slider(64, 384, value=192, step=16, label="Max new tokens (gen)")
|
| 148 |
-
|
| 149 |
gr.Markdown("### Ask a question / Posez une question / Poz enn kestyon")
|
| 150 |
with gr.Row():
|
| 151 |
-
q = gr.Textbox(placeholder=L("en",
|
| 152 |
-
k = gr.Slider(3, 12, step=1, value=6, label=L("en",
|
| 153 |
ask_btn = gr.Button("Ask / Demander / Rode")
|
| 154 |
|
| 155 |
chat = gr.Chatbot(label="Assistant", height=360)
|
|
@@ -158,15 +104,7 @@ with gr.Blocks(title=L("en", "title")) as demo:
|
|
| 158 |
links_md = gr.Markdown()
|
| 159 |
forms_md = gr.Markdown()
|
| 160 |
|
| 161 |
-
ask_btn.click(
|
| 162 |
-
|
| 163 |
-
inputs=[q, k, lang, mode, temp, max_tokens, chat],
|
| 164 |
-
outputs=[chat, answer_md, sources_md, links_md, forms_md],
|
| 165 |
-
)
|
| 166 |
-
q.submit(
|
| 167 |
-
fn=on_ask,
|
| 168 |
-
inputs=[q, k, lang, mode, temp, max_tokens, chat],
|
| 169 |
-
outputs=[chat, answer_md, sources_md, links_md, forms_md],
|
| 170 |
-
)
|
| 171 |
|
| 172 |
demo.launch()
|
|
|
|
| 1 |
"""
|
| 2 |
+
app.py — Step 7
|
| 3 |
+
- Admin can upload .txt/.pdf/.html to data/seed/ and reindex
|
| 4 |
+
- Q&A uses retrieval + Evo templated synthesis from earlier steps
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
"""
|
| 6 |
|
| 7 |
from pathlib import Path
|
| 8 |
from typing import List
|
|
|
|
| 9 |
import gradio as gr
|
| 10 |
|
| 11 |
from indexer import build_index
|
| 12 |
+
from rag_search import RAGSearcher, format_sources, extract_links, split_form_links, links_markdown
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
from evo_inference import synthesize_with_evo
|
| 14 |
from utils_lang import SUPPORTED, normalize_lang, L
|
| 15 |
|
|
|
|
| 17 |
DATA_SEED_DIR = Path("data/seed")
|
| 18 |
ALLOWED_EXTS = {".txt", ".pdf", ".html", ".htm", ".xhtml"}
|
| 19 |
|
|
|
|
| 20 |
def ensure_searcher():
|
| 21 |
global _searcher
|
| 22 |
+
if _searcher is None: _searcher = RAGSearcher()
|
|
|
|
| 23 |
return _searcher
|
| 24 |
|
|
|
|
| 25 |
def on_build_index():
|
| 26 |
try:
|
| 27 |
msg = build_index()
|
| 28 |
+
global _searcher; _searcher = None
|
|
|
|
| 29 |
except Exception as e:
|
| 30 |
msg = f"Error while building index: {e}"
|
| 31 |
return msg
|
| 32 |
|
|
|
|
| 33 |
def _save_files(files: List[gr.File]) -> List[str]:
|
| 34 |
+
saved=[]; DATA_SEED_DIR.mkdir(parents=True, exist_ok=True)
|
| 35 |
+
if not files: return saved
|
|
|
|
|
|
|
|
|
|
| 36 |
for f in files:
|
| 37 |
try:
|
| 38 |
path = None
|
| 39 |
+
if isinstance(f, dict) and "name" in f: path = Path(f["name"])
|
| 40 |
+
elif hasattr(f, "name"): path = Path(getattr(f, "name"))
|
| 41 |
+
elif isinstance(f, str): path = Path(f)
|
| 42 |
+
if not path or not path.exists(): continue
|
| 43 |
+
if path.suffix.lower() not in ALLOWED_EXTS: continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
dest = DATA_SEED_DIR / path.name
|
| 45 |
dest.write_bytes(path.read_bytes())
|
| 46 |
saved.append(path.name)
|
| 47 |
+
except: continue
|
|
|
|
| 48 |
return saved
|
| 49 |
|
| 50 |
+
def on_upload_and_reindex(files):
|
|
|
|
| 51 |
saved = _save_files(files or [])
|
| 52 |
+
if not saved: return "No valid .txt/.pdf/.html uploaded."
|
|
|
|
| 53 |
status = on_build_index()
|
| 54 |
return f"Uploaded: {', '.join(saved)}\n{status}"
|
| 55 |
|
| 56 |
+
def on_ask(question: str, top_k: int, lang_code: str, history: list):
|
|
|
|
| 57 |
lang = normalize_lang(lang_code)
|
| 58 |
if not question or len(question.strip()) < 3:
|
| 59 |
+
return history, L(lang,"intro_err"), f"{L(lang,'sources')}: (none)", "", ""
|
|
|
|
| 60 |
try:
|
| 61 |
searcher = ensure_searcher()
|
| 62 |
hits = searcher.search(question, k=int(top_k))
|
| 63 |
+
answer = synthesize_with_evo(question, lang, hits) # templated synthesis (Step 5)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
srcs = format_sources(hits)
|
| 65 |
+
if srcs.startswith("Sources:"): srcs = srcs.replace("Sources:", f"{L(lang,'sources')}:")
|
|
|
|
|
|
|
| 66 |
all_links = extract_links(hits)
|
| 67 |
form_links, other_links = split_form_links(all_links)
|
| 68 |
+
links_md = links_markdown(L(lang,"links"), other_links)
|
| 69 |
+
forms_md = links_markdown(L(lang,"forms"), form_links)
|
|
|
|
| 70 |
except Exception as e:
|
| 71 |
+
answer = f"Error: {e}\n\n{L(lang,'tip_build')}"
|
| 72 |
+
srcs = f"{L(lang,'sources')}: (none)"
|
| 73 |
+
links_md = f"**{L(lang,'links')}:** (none)"
|
| 74 |
+
forms_md = f"**{L(lang,'forms')}:** (none)"
|
|
|
|
| 75 |
history = history + [(question, answer)]
|
| 76 |
return history, answer, srcs, links_md, forms_md
|
| 77 |
|
| 78 |
+
with gr.Blocks(title=L("en","title")) as demo:
|
| 79 |
+
gr.Markdown(f"# 🇲🇺 **{L('en','title')}** — Step 7")
|
|
|
|
| 80 |
|
| 81 |
with gr.Row():
|
| 82 |
+
build_btn = gr.Button(L("en","build_btn"))
|
| 83 |
+
status = gr.Markdown(L("en","status_idle"))
|
| 84 |
build_btn.click(fn=on_build_index, outputs=status)
|
| 85 |
|
| 86 |
with gr.Accordion("Admin: upload TXT / PDF / HTML and auto-reindex", open=False):
|
|
|
|
| 92 |
gr.Markdown("### Language / Langue / Langaz")
|
| 93 |
lang = gr.Dropdown(choices=list(SUPPORTED.keys()), value="en", label="EN / FR / MFE")
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
gr.Markdown("### Ask a question / Posez une question / Poz enn kestyon")
|
| 96 |
with gr.Row():
|
| 97 |
+
q = gr.Textbox(placeholder=L("en","ask_placeholder"), label=L("en","ask_label"))
|
| 98 |
+
k = gr.Slider(3, 12, step=1, value=6, label=L("en","k_label"))
|
| 99 |
ask_btn = gr.Button("Ask / Demander / Rode")
|
| 100 |
|
| 101 |
chat = gr.Chatbot(label="Assistant", height=360)
|
|
|
|
| 104 |
links_md = gr.Markdown()
|
| 105 |
forms_md = gr.Markdown()
|
| 106 |
|
| 107 |
+
ask_btn.click(fn=on_ask, inputs=[q, k, lang, chat], outputs=[chat, answer_md, sources_md, links_md, forms_md])
|
| 108 |
+
q.submit(fn=on_ask, inputs=[q, k, lang, chat], outputs=[chat, answer_md, sources_md, links_md, forms_md])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
demo.launch()
|