Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import torch
|
3 |
+
from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer
|
4 |
+
from peft import PeftModel, PeftConfig
|
5 |
+
from huggingface_hub import hf_hub_download, login
|
6 |
+
import os
|
7 |
+
import json
|
8 |
+
|
9 |
+
# Retrieve the token from the environment variable
|
10 |
+
hf_token = os.environ.get("HF_TOKEN")
|
11 |
+
if hf_token:
|
12 |
+
login(token=hf_token)
|
13 |
+
print("Successfully logged in to Hugging Face Hub!")
|
14 |
+
else:
|
15 |
+
print("HF_TOKEN not found in environment variables. Cannot authenticate.")
|
16 |
+
|
17 |
+
DEFAULT_FORM_QUESTIONS = [
|
18 |
+
[
|
19 |
+
"SAMPLE",
|
20 |
+
"Does the algorithmic undecidability in the Halting Problem reflect metaphysical truths about reality and real limits of physical computation as suggested by Turing, or does it instead reveal nothing more than a boundary of the rules of language and the way we use language as suggested by Wittgenstein's rule-following paradox?",
|
21 |
+
"A synthetic response might suggest that the Halting Problem illuminates limits inherent in algorithmic processes, while Wittgenstein's insights highlight the role of language and social context in shaping our understanding of those limits - both offering valuable perspectives that contribute to a nuanced understanding of computation, knowledge, and our relationship to the world."
|
22 |
+
],
|
23 |
+
[
|
24 |
+
"TRAINING",
|
25 |
+
"Why does Spinoza consider doubt a signal of an inadequate idea?",
|
26 |
+
"Because doubt arises when two conflicting ideas about the same object are held, showing that neither is clear and distinct."
|
27 |
+
],
|
28 |
+
[
|
29 |
+
"TRAINING",
|
30 |
+
"How does Dewey describe the interplay between analysis and synthesis during judgment?",
|
31 |
+
"Analysis emphasizes significant traits while synthesis places them in an inclusive context; each perfects the other in a continuous spiral.",
|
32 |
+
|
33 |
+
],
|
34 |
+
["NEW", "Other", "", ""]
|
35 |
+
]
|
36 |
+
|
37 |
+
def generate_output(base_model_name, adapter_name, question, max_tokens, temp):
|
38 |
+
|
39 |
+
if adapter_name:
|
40 |
+
tokenizer = AutoTokenizer.from_pretrained(base_model_name)
|
41 |
+
base_model = AutoModelForCausalLM.from_pretrained(base_model_name)
|
42 |
+
model = PeftModel.from_pretrained(base_model, adapter_name)
|
43 |
+
pipe = pipeline(
|
44 |
+
task="text-generation",
|
45 |
+
model=model,
|
46 |
+
device_map="auto",
|
47 |
+
torch_dtype=torch.bfloat16,
|
48 |
+
tokenizer=tokenizer
|
49 |
+
)
|
50 |
+
else:
|
51 |
+
pipe = pipeline(
|
52 |
+
task="text-generation",
|
53 |
+
model=base_model_name,
|
54 |
+
device_map="auto",
|
55 |
+
torch_dtype=torch.bfloat16
|
56 |
+
)
|
57 |
+
|
58 |
+
messages = [
|
59 |
+
{
|
60 |
+
"role": "system",
|
61 |
+
"content": [{"type": "text", "text": "You are a helpful assistant who answers questions."}]
|
62 |
+
},
|
63 |
+
{
|
64 |
+
"role": "user",
|
65 |
+
"content": [{"type": "text", "text": question}]
|
66 |
+
}
|
67 |
+
]
|
68 |
+
|
69 |
+
chat_template = """
|
70 |
+
{% for message in messages %}
|
71 |
+
{% if message['role'] == 'system' %}
|
72 |
+
{{ message['content'][0]['text'] }}
|
73 |
+
|
74 |
+
{% elif message['role'] == 'user' %}
|
75 |
+
User: {{ message['content'][0]['text'] }}
|
76 |
+
|
77 |
+
{% elif message['role'] == 'assistant' %}
|
78 |
+
Assistant: {{ message['content'][0]['text'] }}
|
79 |
+
|
80 |
+
{% endif %}
|
81 |
+
{% endfor %}Assistant:"""
|
82 |
+
|
83 |
+
# Apply the chat template to format the input for the model.
|
84 |
+
# `tokenize=False` is used because the pipeline will handle tokenization.
|
85 |
+
# `add_generation_prompt=True` adds the necessary prompt for the model to generate a response.
|
86 |
+
prompt = pipe.tokenizer.apply_chat_template(
|
87 |
+
messages,
|
88 |
+
tokenize=False,
|
89 |
+
add_generation_prompt=True,
|
90 |
+
chat_template=chat_template
|
91 |
+
)
|
92 |
+
|
93 |
+
# Pass the formatted prompt to the pipeline.
|
94 |
+
# max_new_tokens limits the length of the generated answer.
|
95 |
+
outputs = pipe(prompt, max_new_tokens=max_tokens, temperature=temp)
|
96 |
+
|
97 |
+
# Extract the generated text. The output is a list of dictionaries.
|
98 |
+
# The generated text is typically found in 'generated_text' and needs to be cleaned to remove the input prompt.
|
99 |
+
generated_text = outputs[0]['generated_text'][len(prompt):].strip()
|
100 |
+
|
101 |
+
# generated text may include overflow of extra generated chat if first response less than max_new_tokens limit
|
102 |
+
generated_text_first_line = generated_text.splitlines()[0]
|
103 |
+
|
104 |
+
return generated_text_first_line
|
105 |
+
|
106 |
+
def run_challenge(
|
107 |
+
base_model,
|
108 |
+
adapter,
|
109 |
+
custom_adapter,
|
110 |
+
prompt_question,
|
111 |
+
custom_question,
|
112 |
+
max_tokens,
|
113 |
+
temp
|
114 |
+
):
|
115 |
+
|
116 |
+
if (base_model and (adapter or custom_adapter) and (prompt_question or custom_question) and max_tokens and temp):
|
117 |
+
|
118 |
+
if prompt_question == "Other":
|
119 |
+
prompt_question = custom_question
|
120 |
+
|
121 |
+
if adapter == "Other":
|
122 |
+
adapter = custom_adapter
|
123 |
+
|
124 |
+
# get base model output
|
125 |
+
output_base_model = generate_output(base_model, "", prompt_question, max_tokens, temp)
|
126 |
+
#output_base_model = "TEMP OUTAGE"
|
127 |
+
print(output_base_model)
|
128 |
+
|
129 |
+
# get adapter output
|
130 |
+
output_adapter = generate_output(base_model, adapter, prompt_question, max_tokens, temp)
|
131 |
+
#output_adapter = "TEMP OUTAGE"
|
132 |
+
print(output_adapter)
|
133 |
+
|
134 |
+
return output_base_model, output_adapter
|
135 |
+
|
136 |
+
else:
|
137 |
+
return "Incomplete form values", "Incomplete form values"
|
138 |
+
|
139 |
+
def handle_question_choice(choice):
|
140 |
+
if choice == "Other":
|
141 |
+
return gr.update(label="Enter new question", visible=True, scale=1) # Show the textbox
|
142 |
+
else:
|
143 |
+
return gr.update(visible=False) # Hide the textbox
|
144 |
+
|
145 |
+
def handle_question_training(choice):
|
146 |
+
for sublist in DEFAULT_FORM_QUESTIONS: # Outer loop: iterates through each sublist
|
147 |
+
if (sublist[0] == "TRAINING") and (sublist[1] == choice):
|
148 |
+
return gr.update(value="**Training Answer**: " + sublist[2], visible=True)
|
149 |
+
elif (sublist[0] == "SAMPLE") and (sublist[1] == choice):
|
150 |
+
return gr.update(value="**Sample Question Information (not used in training)**: " + sublist[2], visible=True)
|
151 |
+
return gr.update(value="", visible=False)
|
152 |
+
|
153 |
+
def handle_adapter_choice(choice):
|
154 |
+
if choice == "Other":
|
155 |
+
return gr.update(visible=True) # Show the textbox
|
156 |
+
else:
|
157 |
+
return gr.update(visible=False) # Hide the textbox
|
158 |
+
|
159 |
+
def get_base_model(adapter_repo):
|
160 |
+
"""
|
161 |
+
Get base model for adapter via hub
|
162 |
+
"""
|
163 |
+
try:
|
164 |
+
# Download adapter config from hub
|
165 |
+
config_path = hf_hub_download(repo_id=adapter_repo, filename="adapter_config.json")
|
166 |
+
|
167 |
+
with open(config_path, 'r') as f:
|
168 |
+
adapter_config = json.load(f)
|
169 |
+
|
170 |
+
base_model = adapter_config.get('base_model_name_or_path', '')
|
171 |
+
|
172 |
+
if base_model:
|
173 |
+
return base_model
|
174 |
+
else:
|
175 |
+
print(f"❌ Base model not found")
|
176 |
+
return "Base model not found"
|
177 |
+
|
178 |
+
except Exception as e:
|
179 |
+
print(f"Hub base model check failed: {e}")
|
180 |
+
return "Base model not found"
|
181 |
+
|
182 |
+
def handle_adapter_radio(choice):
|
183 |
+
if choice != "Other":
|
184 |
+
print("handle_adapter_radio")
|
185 |
+
print(choice)
|
186 |
+
model = get_base_model(choice)
|
187 |
+
return gr.update(value=model)
|
188 |
+
|
189 |
+
with gr.Blocks() as demo:
|
190 |
+
|
191 |
+
# Add a title and description for the app.
|
192 |
+
gr.Markdown("# Transformers Text-Generation Pipeline Q&A: Adapter vs. Base Model")
|
193 |
+
gr.Markdown("### A tool for experimenting with PEFT (Parameter-Efficient Fine-tuning) and LoRA (Low-Rank Adaptation) adapter performance.")
|
194 |
+
|
195 |
+
with gr.Row():
|
196 |
+
|
197 |
+
with gr.Column(scale=1):
|
198 |
+
|
199 |
+
with gr.Row():
|
200 |
+
|
201 |
+
prompt_adapter = gr.Radio(
|
202 |
+
[
|
203 |
+
"joshause/llama-3.2-1b-dewey-how-we-think-adapter",
|
204 |
+
"joshause/gemma-3-1b-pt-spinoza-treatise-emendation-intellect",
|
205 |
+
"Other"
|
206 |
+
],
|
207 |
+
label="Select an adapter or enter a new adapter",
|
208 |
+
scale=1
|
209 |
+
)
|
210 |
+
with gr.Group():
|
211 |
+
custom_adapter = gr.Textbox(label="Enter an adapter", scale=1, visible=False)
|
212 |
+
|
213 |
+
submit_adapter_button = gr.Button(
|
214 |
+
"Get Base Model",
|
215 |
+
scale=1,
|
216 |
+
variant="secondary",
|
217 |
+
visible=False
|
218 |
+
)
|
219 |
+
with gr.Row():
|
220 |
+
prompt_base_model = gr.Textbox(
|
221 |
+
label="Base model (auto-filled)",
|
222 |
+
max_lines=1,
|
223 |
+
interactive=False,
|
224 |
+
scale=1,
|
225 |
+
value=""
|
226 |
+
)
|
227 |
+
|
228 |
+
with gr.Row():
|
229 |
+
questions = [[sublist[0]+" QUESTION: "+sublist[1], sublist[1]] for sublist in DEFAULT_FORM_QUESTIONS]
|
230 |
+
prompt_question = gr.Radio(questions, label="Select a question or enter a new question", scale=1)
|
231 |
+
with gr.Group():
|
232 |
+
training_answer = gr.Markdown("", padding=True, visible=False)
|
233 |
+
custom_question = gr.Textbox(label="Enter new question", visible=False, scale=1)
|
234 |
+
|
235 |
+
with gr.Row():
|
236 |
+
|
237 |
+
challenge_submit_button = gr.Button(
|
238 |
+
"Run Comparison",
|
239 |
+
scale=1,
|
240 |
+
variant="primary"
|
241 |
+
)
|
242 |
+
|
243 |
+
with gr.Column(scale=1):
|
244 |
+
with gr.Row():
|
245 |
+
output_max_tokens = gr.Slider(
|
246 |
+
1, 512, step=1, value=128, label="Max new tokens", info="max_new_tokens in text-generation pipeline limits the length of the generated answer"
|
247 |
+
)
|
248 |
+
with gr.Row():
|
249 |
+
output_temp = gr.Slider(
|
250 |
+
0.1, 0.9, step=0.1, value=0.7, label="Temperature", info="temperature in text-generation pipeline parameter controls the randomness of the generated text"
|
251 |
+
)
|
252 |
+
with gr.Row():
|
253 |
+
output_adapter = gr.Textbox(
|
254 |
+
label="Adapter output",
|
255 |
+
show_label=True,
|
256 |
+
lines=4,
|
257 |
+
interactive=False,
|
258 |
+
show_copy_button=True,
|
259 |
+
)
|
260 |
+
with gr.Row():
|
261 |
+
output_base_model = gr.Textbox(
|
262 |
+
label="Base model output",
|
263 |
+
show_label=True,
|
264 |
+
lines=4,
|
265 |
+
interactive=False,
|
266 |
+
show_copy_button=True,
|
267 |
+
)
|
268 |
+
|
269 |
+
prompt_adapter.change(
|
270 |
+
fn=handle_adapter_choice,
|
271 |
+
inputs=prompt_adapter,
|
272 |
+
outputs=custom_adapter
|
273 |
+
).then(
|
274 |
+
fn=handle_adapter_choice,
|
275 |
+
inputs=prompt_adapter,
|
276 |
+
outputs=submit_adapter_button
|
277 |
+
).then(
|
278 |
+
fn=handle_adapter_radio,
|
279 |
+
inputs=prompt_adapter,
|
280 |
+
outputs=prompt_base_model
|
281 |
+
)
|
282 |
+
|
283 |
+
prompt_question.change(
|
284 |
+
fn=handle_question_choice,
|
285 |
+
inputs=prompt_question,
|
286 |
+
outputs=custom_question
|
287 |
+
).then(
|
288 |
+
fn=handle_question_training,
|
289 |
+
inputs=prompt_question,
|
290 |
+
outputs=training_answer
|
291 |
+
)
|
292 |
+
|
293 |
+
submit_adapter_button.click(
|
294 |
+
fn=get_base_model,
|
295 |
+
inputs=custom_adapter,
|
296 |
+
outputs=prompt_base_model
|
297 |
+
)
|
298 |
+
|
299 |
+
challenge_submit_button.click(
|
300 |
+
fn=run_challenge,
|
301 |
+
inputs=[
|
302 |
+
prompt_base_model,
|
303 |
+
prompt_adapter,
|
304 |
+
custom_adapter,
|
305 |
+
prompt_question,
|
306 |
+
custom_question,
|
307 |
+
output_max_tokens,
|
308 |
+
output_temp
|
309 |
+
],
|
310 |
+
outputs=[
|
311 |
+
output_base_model,
|
312 |
+
output_adapter
|
313 |
+
]
|
314 |
+
)
|
315 |
+
|
316 |
+
if __name__ == "__main__":
|
317 |
+
demo.launch()
|