joshause commited on
Commit
87156fc
·
verified ·
1 Parent(s): 85d9bf7

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +317 -0
app.py ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer
4
+ from peft import PeftModel, PeftConfig
5
+ from huggingface_hub import hf_hub_download, login
6
+ import os
7
+ import json
8
+
9
+ # Retrieve the token from the environment variable
10
+ hf_token = os.environ.get("HF_TOKEN")
11
+ if hf_token:
12
+ login(token=hf_token)
13
+ print("Successfully logged in to Hugging Face Hub!")
14
+ else:
15
+ print("HF_TOKEN not found in environment variables. Cannot authenticate.")
16
+
17
+ DEFAULT_FORM_QUESTIONS = [
18
+ [
19
+ "SAMPLE",
20
+ "Does the algorithmic undecidability in the Halting Problem reflect metaphysical truths about reality and real limits of physical computation as suggested by Turing, or does it instead reveal nothing more than a boundary of the rules of language and the way we use language as suggested by Wittgenstein's rule-following paradox?",
21
+ "A synthetic response might suggest that the Halting Problem illuminates limits inherent in algorithmic processes, while Wittgenstein's insights highlight the role of language and social context in shaping our understanding of those limits - both offering valuable perspectives that contribute to a nuanced understanding of computation, knowledge, and our relationship to the world."
22
+ ],
23
+ [
24
+ "TRAINING",
25
+ "Why does Spinoza consider doubt a signal of an inadequate idea?",
26
+ "Because doubt arises when two conflicting ideas about the same object are held, showing that neither is clear and distinct."
27
+ ],
28
+ [
29
+ "TRAINING",
30
+ "How does Dewey describe the interplay between analysis and synthesis during judgment?",
31
+ "Analysis emphasizes significant traits while synthesis places them in an inclusive context; each perfects the other in a continuous spiral.",
32
+
33
+ ],
34
+ ["NEW", "Other", "", ""]
35
+ ]
36
+
37
+ def generate_output(base_model_name, adapter_name, question, max_tokens, temp):
38
+
39
+ if adapter_name:
40
+ tokenizer = AutoTokenizer.from_pretrained(base_model_name)
41
+ base_model = AutoModelForCausalLM.from_pretrained(base_model_name)
42
+ model = PeftModel.from_pretrained(base_model, adapter_name)
43
+ pipe = pipeline(
44
+ task="text-generation",
45
+ model=model,
46
+ device_map="auto",
47
+ torch_dtype=torch.bfloat16,
48
+ tokenizer=tokenizer
49
+ )
50
+ else:
51
+ pipe = pipeline(
52
+ task="text-generation",
53
+ model=base_model_name,
54
+ device_map="auto",
55
+ torch_dtype=torch.bfloat16
56
+ )
57
+
58
+ messages = [
59
+ {
60
+ "role": "system",
61
+ "content": [{"type": "text", "text": "You are a helpful assistant who answers questions."}]
62
+ },
63
+ {
64
+ "role": "user",
65
+ "content": [{"type": "text", "text": question}]
66
+ }
67
+ ]
68
+
69
+ chat_template = """
70
+ {% for message in messages %}
71
+ {% if message['role'] == 'system' %}
72
+ {{ message['content'][0]['text'] }}
73
+
74
+ {% elif message['role'] == 'user' %}
75
+ User: {{ message['content'][0]['text'] }}
76
+
77
+ {% elif message['role'] == 'assistant' %}
78
+ Assistant: {{ message['content'][0]['text'] }}
79
+
80
+ {% endif %}
81
+ {% endfor %}Assistant:"""
82
+
83
+ # Apply the chat template to format the input for the model.
84
+ # `tokenize=False` is used because the pipeline will handle tokenization.
85
+ # `add_generation_prompt=True` adds the necessary prompt for the model to generate a response.
86
+ prompt = pipe.tokenizer.apply_chat_template(
87
+ messages,
88
+ tokenize=False,
89
+ add_generation_prompt=True,
90
+ chat_template=chat_template
91
+ )
92
+
93
+ # Pass the formatted prompt to the pipeline.
94
+ # max_new_tokens limits the length of the generated answer.
95
+ outputs = pipe(prompt, max_new_tokens=max_tokens, temperature=temp)
96
+
97
+ # Extract the generated text. The output is a list of dictionaries.
98
+ # The generated text is typically found in 'generated_text' and needs to be cleaned to remove the input prompt.
99
+ generated_text = outputs[0]['generated_text'][len(prompt):].strip()
100
+
101
+ # generated text may include overflow of extra generated chat if first response less than max_new_tokens limit
102
+ generated_text_first_line = generated_text.splitlines()[0]
103
+
104
+ return generated_text_first_line
105
+
106
+ def run_challenge(
107
+ base_model,
108
+ adapter,
109
+ custom_adapter,
110
+ prompt_question,
111
+ custom_question,
112
+ max_tokens,
113
+ temp
114
+ ):
115
+
116
+ if (base_model and (adapter or custom_adapter) and (prompt_question or custom_question) and max_tokens and temp):
117
+
118
+ if prompt_question == "Other":
119
+ prompt_question = custom_question
120
+
121
+ if adapter == "Other":
122
+ adapter = custom_adapter
123
+
124
+ # get base model output
125
+ output_base_model = generate_output(base_model, "", prompt_question, max_tokens, temp)
126
+ #output_base_model = "TEMP OUTAGE"
127
+ print(output_base_model)
128
+
129
+ # get adapter output
130
+ output_adapter = generate_output(base_model, adapter, prompt_question, max_tokens, temp)
131
+ #output_adapter = "TEMP OUTAGE"
132
+ print(output_adapter)
133
+
134
+ return output_base_model, output_adapter
135
+
136
+ else:
137
+ return "Incomplete form values", "Incomplete form values"
138
+
139
+ def handle_question_choice(choice):
140
+ if choice == "Other":
141
+ return gr.update(label="Enter new question", visible=True, scale=1) # Show the textbox
142
+ else:
143
+ return gr.update(visible=False) # Hide the textbox
144
+
145
+ def handle_question_training(choice):
146
+ for sublist in DEFAULT_FORM_QUESTIONS: # Outer loop: iterates through each sublist
147
+ if (sublist[0] == "TRAINING") and (sublist[1] == choice):
148
+ return gr.update(value="**Training Answer**: " + sublist[2], visible=True)
149
+ elif (sublist[0] == "SAMPLE") and (sublist[1] == choice):
150
+ return gr.update(value="**Sample Question Information (not used in training)**: " + sublist[2], visible=True)
151
+ return gr.update(value="", visible=False)
152
+
153
+ def handle_adapter_choice(choice):
154
+ if choice == "Other":
155
+ return gr.update(visible=True) # Show the textbox
156
+ else:
157
+ return gr.update(visible=False) # Hide the textbox
158
+
159
+ def get_base_model(adapter_repo):
160
+ """
161
+ Get base model for adapter via hub
162
+ """
163
+ try:
164
+ # Download adapter config from hub
165
+ config_path = hf_hub_download(repo_id=adapter_repo, filename="adapter_config.json")
166
+
167
+ with open(config_path, 'r') as f:
168
+ adapter_config = json.load(f)
169
+
170
+ base_model = adapter_config.get('base_model_name_or_path', '')
171
+
172
+ if base_model:
173
+ return base_model
174
+ else:
175
+ print(f"❌ Base model not found")
176
+ return "Base model not found"
177
+
178
+ except Exception as e:
179
+ print(f"Hub base model check failed: {e}")
180
+ return "Base model not found"
181
+
182
+ def handle_adapter_radio(choice):
183
+ if choice != "Other":
184
+ print("handle_adapter_radio")
185
+ print(choice)
186
+ model = get_base_model(choice)
187
+ return gr.update(value=model)
188
+
189
+ with gr.Blocks() as demo:
190
+
191
+ # Add a title and description for the app.
192
+ gr.Markdown("# Transformers Text-Generation Pipeline Q&A: Adapter vs. Base Model")
193
+ gr.Markdown("### A tool for experimenting with PEFT (Parameter-Efficient Fine-tuning) and LoRA (Low-Rank Adaptation) adapter performance.")
194
+
195
+ with gr.Row():
196
+
197
+ with gr.Column(scale=1):
198
+
199
+ with gr.Row():
200
+
201
+ prompt_adapter = gr.Radio(
202
+ [
203
+ "joshause/llama-3.2-1b-dewey-how-we-think-adapter",
204
+ "joshause/gemma-3-1b-pt-spinoza-treatise-emendation-intellect",
205
+ "Other"
206
+ ],
207
+ label="Select an adapter or enter a new adapter",
208
+ scale=1
209
+ )
210
+ with gr.Group():
211
+ custom_adapter = gr.Textbox(label="Enter an adapter", scale=1, visible=False)
212
+
213
+ submit_adapter_button = gr.Button(
214
+ "Get Base Model",
215
+ scale=1,
216
+ variant="secondary",
217
+ visible=False
218
+ )
219
+ with gr.Row():
220
+ prompt_base_model = gr.Textbox(
221
+ label="Base model (auto-filled)",
222
+ max_lines=1,
223
+ interactive=False,
224
+ scale=1,
225
+ value=""
226
+ )
227
+
228
+ with gr.Row():
229
+ questions = [[sublist[0]+" QUESTION: "+sublist[1], sublist[1]] for sublist in DEFAULT_FORM_QUESTIONS]
230
+ prompt_question = gr.Radio(questions, label="Select a question or enter a new question", scale=1)
231
+ with gr.Group():
232
+ training_answer = gr.Markdown("", padding=True, visible=False)
233
+ custom_question = gr.Textbox(label="Enter new question", visible=False, scale=1)
234
+
235
+ with gr.Row():
236
+
237
+ challenge_submit_button = gr.Button(
238
+ "Run Comparison",
239
+ scale=1,
240
+ variant="primary"
241
+ )
242
+
243
+ with gr.Column(scale=1):
244
+ with gr.Row():
245
+ output_max_tokens = gr.Slider(
246
+ 1, 512, step=1, value=128, label="Max new tokens", info="max_new_tokens in text-generation pipeline limits the length of the generated answer"
247
+ )
248
+ with gr.Row():
249
+ output_temp = gr.Slider(
250
+ 0.1, 0.9, step=0.1, value=0.7, label="Temperature", info="temperature in text-generation pipeline parameter controls the randomness of the generated text"
251
+ )
252
+ with gr.Row():
253
+ output_adapter = gr.Textbox(
254
+ label="Adapter output",
255
+ show_label=True,
256
+ lines=4,
257
+ interactive=False,
258
+ show_copy_button=True,
259
+ )
260
+ with gr.Row():
261
+ output_base_model = gr.Textbox(
262
+ label="Base model output",
263
+ show_label=True,
264
+ lines=4,
265
+ interactive=False,
266
+ show_copy_button=True,
267
+ )
268
+
269
+ prompt_adapter.change(
270
+ fn=handle_adapter_choice,
271
+ inputs=prompt_adapter,
272
+ outputs=custom_adapter
273
+ ).then(
274
+ fn=handle_adapter_choice,
275
+ inputs=prompt_adapter,
276
+ outputs=submit_adapter_button
277
+ ).then(
278
+ fn=handle_adapter_radio,
279
+ inputs=prompt_adapter,
280
+ outputs=prompt_base_model
281
+ )
282
+
283
+ prompt_question.change(
284
+ fn=handle_question_choice,
285
+ inputs=prompt_question,
286
+ outputs=custom_question
287
+ ).then(
288
+ fn=handle_question_training,
289
+ inputs=prompt_question,
290
+ outputs=training_answer
291
+ )
292
+
293
+ submit_adapter_button.click(
294
+ fn=get_base_model,
295
+ inputs=custom_adapter,
296
+ outputs=prompt_base_model
297
+ )
298
+
299
+ challenge_submit_button.click(
300
+ fn=run_challenge,
301
+ inputs=[
302
+ prompt_base_model,
303
+ prompt_adapter,
304
+ custom_adapter,
305
+ prompt_question,
306
+ custom_question,
307
+ output_max_tokens,
308
+ output_temp
309
+ ],
310
+ outputs=[
311
+ output_base_model,
312
+ output_adapter
313
+ ]
314
+ )
315
+
316
+ if __name__ == "__main__":
317
+ demo.launch()