Add 'load first example' button
Browse files
app.py
CHANGED
@@ -154,6 +154,20 @@ def format_stats_display(judgment_times, num_judgments, num_skips):
|
|
154 |
|
155 |
return stats
|
156 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
157 |
def handle_left_better(prompt, completion_1_display, completion_2_display, current_idx, instruction, completion_1, completion_2,
|
158 |
start_time, judgment_times, num_judgments, num_skips):
|
159 |
"""Handle when user selects left completion as better"""
|
@@ -173,15 +187,15 @@ def handle_left_better(prompt, completion_1_display, completion_2_display, curre
|
|
173 |
# Get new random example
|
174 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
175 |
|
|
|
|
|
|
|
176 |
# Update stats display
|
177 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
178 |
|
179 |
message = "β
Annotation saved! Left completion selected as better." if success else "β
Left completion selected (save failed - check console)"
|
180 |
gr.Info(message)
|
181 |
|
182 |
-
# Reset timer for new example
|
183 |
-
new_start_time = time.time()
|
184 |
-
|
185 |
return (
|
186 |
new_prompt,
|
187 |
new_comp_1,
|
@@ -216,15 +230,15 @@ def handle_right_better(prompt, completion_1_display, completion_2_display, curr
|
|
216 |
# Get new random example
|
217 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
218 |
|
|
|
|
|
|
|
219 |
# Update stats display
|
220 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
221 |
|
222 |
message = "β
Annotation saved! Right completion selected as better." if success else "β
Right completion selected (save failed - check console)"
|
223 |
gr.Info(message)
|
224 |
|
225 |
-
# Reset timer for new example
|
226 |
-
new_start_time = time.time()
|
227 |
-
|
228 |
return (
|
229 |
new_prompt,
|
230 |
new_comp_1,
|
@@ -280,11 +294,23 @@ if HF_TOKEN:
|
|
280 |
else:
|
281 |
print("Warning: No HF_TOKEN found. Annotations will not be saved.")
|
282 |
|
283 |
-
|
284 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
285 |
|
286 |
# Create Gradio interface
|
287 |
-
|
|
|
288 |
gr.Markdown(f"""
|
289 |
# π― AI Alignment: Binary Preference Annotation
|
290 |
|
@@ -297,29 +323,34 @@ This simulates the data annotation process used in RLHF (Reinforcement Learning
|
|
297 |
""")
|
298 |
|
299 |
# State to track current example and its components
|
300 |
-
current_idx = gr.State(
|
301 |
-
current_instruction = gr.State(
|
302 |
-
current_completion_1 = gr.State(
|
303 |
-
current_completion_2 = gr.State(
|
304 |
|
305 |
# State to track timing and statistics
|
306 |
-
start_time = gr.State(
|
307 |
judgment_times = gr.State([]) # List of times taken for each judgment
|
308 |
num_judgments = gr.State(0) # Number of judgments made
|
309 |
num_skips = gr.State(0) # Number of examples skipped
|
310 |
|
311 |
-
#
|
312 |
-
|
|
|
|
|
|
|
313 |
|
314 |
-
# Display completions side by side
|
315 |
-
|
|
|
316 |
with gr.Column():
|
317 |
-
completion_1_display = gr.Markdown(
|
318 |
with gr.Column():
|
319 |
-
completion_2_display = gr.Markdown(
|
320 |
|
321 |
-
# Action buttons
|
322 |
-
|
|
|
323 |
left_better_btn = gr.Button("π Left is Better", variant="primary", size="lg")
|
324 |
skip_btn = gr.Button("βοΈ Skip This Example", variant="secondary", size="lg")
|
325 |
right_better_btn = gr.Button("π Right is Better", variant="primary", size="lg")
|
@@ -333,7 +364,15 @@ This simulates the data annotation process used in RLHF (Reinforcement Learning
|
|
333 |
# Statistics display
|
334 |
stats_display = gr.Markdown("π **Session Statistics:** No judgments made yet.", label="Performance Stats")
|
335 |
|
336 |
-
# Wire up the
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
337 |
left_better_btn.click(
|
338 |
handle_left_better,
|
339 |
inputs=[prompt_display, completion_1_display, completion_2_display, current_idx, current_instruction, current_completion_1, current_completion_2,
|
|
|
154 |
|
155 |
return stats
|
156 |
|
157 |
+
def load_first_example():
|
158 |
+
"""Load the first example and start the annotation interface"""
|
159 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2 = get_random_example()
|
160 |
+
start_time = time.time()
|
161 |
+
|
162 |
+
return (
|
163 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2,
|
164 |
+
start_time,
|
165 |
+
gr.update(visible=False), # Hide load button
|
166 |
+
gr.update(visible=True), # Show prompt
|
167 |
+
gr.update(visible=True), # Show completion row
|
168 |
+
gr.update(visible=True), # Show action buttons
|
169 |
+
)
|
170 |
+
|
171 |
def handle_left_better(prompt, completion_1_display, completion_2_display, current_idx, instruction, completion_1, completion_2,
|
172 |
start_time, judgment_times, num_judgments, num_skips):
|
173 |
"""Handle when user selects left completion as better"""
|
|
|
187 |
# Get new random example
|
188 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
189 |
|
190 |
+
# Reset timer for new example
|
191 |
+
new_start_time = time.time()
|
192 |
+
|
193 |
# Update stats display
|
194 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
195 |
|
196 |
message = "β
Annotation saved! Left completion selected as better." if success else "β
Left completion selected (save failed - check console)"
|
197 |
gr.Info(message)
|
198 |
|
|
|
|
|
|
|
199 |
return (
|
200 |
new_prompt,
|
201 |
new_comp_1,
|
|
|
230 |
# Get new random example
|
231 |
new_prompt, new_comp_1, new_comp_2, new_idx, new_instruction, new_completion_1, new_completion_2 = get_random_example()
|
232 |
|
233 |
+
# Reset timer for new example
|
234 |
+
new_start_time = time.time()
|
235 |
+
|
236 |
# Update stats display
|
237 |
stats_display = format_stats_display(judgment_times, num_judgments, num_skips)
|
238 |
|
239 |
message = "β
Annotation saved! Right completion selected as better." if success else "β
Right completion selected (save failed - check console)"
|
240 |
gr.Info(message)
|
241 |
|
|
|
|
|
|
|
242 |
return (
|
243 |
new_prompt,
|
244 |
new_comp_1,
|
|
|
294 |
else:
|
295 |
print("Warning: No HF_TOKEN found. Annotations will not be saved.")
|
296 |
|
297 |
+
def load_first_example():
|
298 |
+
"""Load the first example and start the annotation interface"""
|
299 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2 = get_random_example()
|
300 |
+
start_time = time.time()
|
301 |
+
|
302 |
+
return (
|
303 |
+
prompt, comp_1, comp_2, idx, instruction, completion_1, completion_2,
|
304 |
+
start_time,
|
305 |
+
gr.update(visible=False), # Hide load button
|
306 |
+
gr.update(visible=True), # Show prompt
|
307 |
+
gr.update(visible=True), # Show completion row
|
308 |
+
gr.update(visible=True), # Show action buttons
|
309 |
+
)
|
310 |
|
311 |
# Create Gradio interface
|
312 |
+
|
313 |
+
with gr.Blocks(title="AI Alignment: Binary Preference Annotation", css=".square-button { height: 80px !important; }") as demo:
|
314 |
gr.Markdown(f"""
|
315 |
# π― AI Alignment: Binary Preference Annotation
|
316 |
|
|
|
323 |
""")
|
324 |
|
325 |
# State to track current example and its components
|
326 |
+
current_idx = gr.State(0)
|
327 |
+
current_instruction = gr.State("")
|
328 |
+
current_completion_1 = gr.State("")
|
329 |
+
current_completion_2 = gr.State("")
|
330 |
|
331 |
# State to track timing and statistics
|
332 |
+
start_time = gr.State(0.0) # When current example was loaded
|
333 |
judgment_times = gr.State([]) # List of times taken for each judgment
|
334 |
num_judgments = gr.State(0) # Number of judgments made
|
335 |
num_skips = gr.State(0) # Number of examples skipped
|
336 |
|
337 |
+
# Load first example button (shown initially)
|
338 |
+
load_first_btn = gr.Button("π Load First Example", variant="primary", size="lg", elem_classes="square-button")
|
339 |
+
|
340 |
+
# Display prompt (hidden initially)
|
341 |
+
prompt_display = gr.Markdown("", label="Prompt", visible=False)
|
342 |
|
343 |
+
# Display completions side by side (hidden initially)
|
344 |
+
completion_row = gr.Row(visible=False)
|
345 |
+
with completion_row:
|
346 |
with gr.Column():
|
347 |
+
completion_1_display = gr.Markdown("", label="Completion A (Left)")
|
348 |
with gr.Column():
|
349 |
+
completion_2_display = gr.Markdown("", label="Completion B (Right)")
|
350 |
|
351 |
+
# Action buttons (hidden initially)
|
352 |
+
action_buttons = gr.Row(visible=False)
|
353 |
+
with action_buttons:
|
354 |
left_better_btn = gr.Button("π Left is Better", variant="primary", size="lg")
|
355 |
skip_btn = gr.Button("βοΈ Skip This Example", variant="secondary", size="lg")
|
356 |
right_better_btn = gr.Button("π Right is Better", variant="primary", size="lg")
|
|
|
364 |
# Statistics display
|
365 |
stats_display = gr.Markdown("π **Session Statistics:** No judgments made yet.", label="Performance Stats")
|
366 |
|
367 |
+
# Wire up the load first example button
|
368 |
+
load_first_btn.click(
|
369 |
+
load_first_example,
|
370 |
+
inputs=[],
|
371 |
+
outputs=[prompt_display, completion_1_display, completion_2_display, current_idx, current_instruction, current_completion_1, current_completion_2,
|
372 |
+
start_time, load_first_btn, prompt_display, completion_row, action_buttons]
|
373 |
+
)
|
374 |
+
|
375 |
+
# Wire up the action buttons
|
376 |
left_better_btn.click(
|
377 |
handle_left_better,
|
378 |
inputs=[prompt_display, completion_1_display, completion_2_display, current_idx, current_instruction, current_completion_1, current_completion_2,
|