Spaces:
Runtime error
Runtime error
Commit
·
fdbad51
1
Parent(s):
c9babee
Added adult example. Tighten up presentation.
Browse files
app.py
CHANGED
|
@@ -8,7 +8,7 @@ from matplotlib.ticker import MaxNLocator
|
|
| 8 |
from transformers import pipeline
|
| 9 |
|
| 10 |
MODEL_NAMES = ["bert-base-uncased", "roberta-base", "bert-large-uncased", "roberta-large"]
|
| 11 |
-
OWN_MODEL_NAME = 'add-
|
| 12 |
|
| 13 |
DECIMAL_PLACES = 1
|
| 14 |
EPS = 1e-5 # to avoid /0 errors
|
|
@@ -308,7 +308,7 @@ place_example = [
|
|
| 308 |
'PLACE',
|
| 309 |
"False",
|
| 310 |
1,
|
| 311 |
-
'She
|
| 312 |
]
|
| 313 |
|
| 314 |
|
|
@@ -353,13 +353,13 @@ def your_fn():
|
|
| 353 |
demo = gr.Blocks()
|
| 354 |
with demo:
|
| 355 |
gr.Markdown("# Spurious Correlation Evaluation for Pre-trained LLMs")
|
| 356 |
-
gr.Markdown("Find
|
| 357 |
|
| 358 |
-
gr.Markdown("See why this happens how in our paper, [Selection Bias Induced Spurious Correlations in Large Language Models](https://arxiv.org/pdf/2207.08982.pdf), presented at [
|
| 359 |
|
| 360 |
|
| 361 |
gr.Markdown("## Instructions for this Demo")
|
| 362 |
-
gr.Markdown("1) Click on one of the examples below (where we sweep through a spectrum of `places`, `
|
| 363 |
gr.Markdown("2) Check out the pre-populated fields as you scroll down to the ['Hit Submit...'] button!")
|
| 364 |
gr.Markdown("3) Repeat steps (1) and (2) with more pre-populated inputs or with your own values in the input fields!")
|
| 365 |
|
|
@@ -377,17 +377,17 @@ with demo:
|
|
| 377 |
gr.Markdown(
|
| 378 |
"<-- x-axis sorted in order of increasing self-identified female participation (see [bburky](http://bburky.com/subredditgenderratios/)): ")
|
| 379 |
|
| 380 |
-
your_gen = gr.Button('
|
| 381 |
gr.Markdown("<-- x-axis dates, with your own model loaded! (If first time, try another example, it can take a while to load new model.)")
|
| 382 |
|
| 383 |
gr.Markdown("## Input fields")
|
| 384 |
gr.Markdown(
|
| 385 |
-
f"A) Pick a spectrum of comma separated values for text injection and x-axis
|
| 386 |
|
| 387 |
with gr.Row():
|
| 388 |
x_axis = gr.Textbox(
|
| 389 |
-
lines=
|
| 390 |
-
label="A)
|
| 391 |
)
|
| 392 |
|
| 393 |
|
|
@@ -398,15 +398,15 @@ with demo:
|
|
| 398 |
model_name = gr.Radio(
|
| 399 |
MODEL_NAMES + [OWN_MODEL_NAME],
|
| 400 |
type="value",
|
| 401 |
-
label="B)
|
| 402 |
)
|
| 403 |
own_model_name = gr.Textbox(
|
| 404 |
-
label="C) If you selected an 'add-
|
| 405 |
)
|
| 406 |
|
| 407 |
gr.Markdown("D) Pick if you want to the predictions normalied to these gendered terms only.")
|
| 408 |
gr.Markdown("E) Also tell the demo what special token you will use in your input text, that you would like replaced with the spectrum of values you listed above.")
|
| 409 |
-
gr.Markdown("And F) the degree of polynomial fit used for high-lighting
|
| 410 |
|
| 411 |
|
| 412 |
with gr.Row():
|
|
@@ -416,11 +416,11 @@ with demo:
|
|
| 416 |
type="index",
|
| 417 |
)
|
| 418 |
place_holder = gr.Textbox(
|
| 419 |
-
label="E) Special token place-holder
|
| 420 |
)
|
| 421 |
n_fit = gr.Dropdown(
|
| 422 |
list(range(1, 5)),
|
| 423 |
-
label="F) Degree of polynomial fit
|
| 424 |
type="value",
|
| 425 |
)
|
| 426 |
|
|
@@ -429,8 +429,8 @@ with demo:
|
|
| 429 |
|
| 430 |
with gr.Row():
|
| 431 |
input_text = gr.Textbox(
|
| 432 |
-
lines=
|
| 433 |
-
label="G) Input text
|
| 434 |
)
|
| 435 |
|
| 436 |
gr.Markdown("## Outputs!")
|
|
|
|
| 8 |
from transformers import pipeline
|
| 9 |
|
| 10 |
MODEL_NAMES = ["bert-base-uncased", "roberta-base", "bert-large-uncased", "roberta-large"]
|
| 11 |
+
OWN_MODEL_NAME = 'add-a-model'
|
| 12 |
|
| 13 |
DECIMAL_PLACES = 1
|
| 14 |
EPS = 1e-5 # to avoid /0 errors
|
|
|
|
| 308 |
'PLACE',
|
| 309 |
"False",
|
| 310 |
1,
|
| 311 |
+
'She became an adult in PLACE.'
|
| 312 |
]
|
| 313 |
|
| 314 |
|
|
|
|
| 353 |
demo = gr.Blocks()
|
| 354 |
with demo:
|
| 355 |
gr.Markdown("# Spurious Correlation Evaluation for Pre-trained LLMs")
|
| 356 |
+
gr.Markdown("Find spurious correlations between seemingly independent variables (for example between `gender` and `time`) in almost any BERT-like LLM on Hugging Face, below.")
|
| 357 |
|
| 358 |
+
gr.Markdown("See why this happens how in our paper, [Selection Bias Induced Spurious Correlations in Large Language Models](https://arxiv.org/pdf/2207.08982.pdf), presented at [ICML 2022 Workshop on Spurious Correlations, Invariance, and Stability](https://sites.google.com/view/scis-workshop/home).")
|
| 359 |
|
| 360 |
|
| 361 |
gr.Markdown("## Instructions for this Demo")
|
| 362 |
+
gr.Markdown("1) Click on one of the examples below (where we sweep through a spectrum of `places`, `dates` and `subreddits`) to pre-populate the input fields.")
|
| 363 |
gr.Markdown("2) Check out the pre-populated fields as you scroll down to the ['Hit Submit...'] button!")
|
| 364 |
gr.Markdown("3) Repeat steps (1) and (2) with more pre-populated inputs or with your own values in the input fields!")
|
| 365 |
|
|
|
|
| 377 |
gr.Markdown(
|
| 378 |
"<-- x-axis sorted in order of increasing self-identified female participation (see [bburky](http://bburky.com/subredditgenderratios/)): ")
|
| 379 |
|
| 380 |
+
your_gen = gr.Button('Add-a-model example inputs')
|
| 381 |
gr.Markdown("<-- x-axis dates, with your own model loaded! (If first time, try another example, it can take a while to load new model.)")
|
| 382 |
|
| 383 |
gr.Markdown("## Input fields")
|
| 384 |
gr.Markdown(
|
| 385 |
+
f"A) Pick a spectrum of comma separated values for text injection and x-axis.")
|
| 386 |
|
| 387 |
with gr.Row():
|
| 388 |
x_axis = gr.Textbox(
|
| 389 |
+
lines=3,
|
| 390 |
+
label="A) Comma separated values for text injection and x-axis",
|
| 391 |
)
|
| 392 |
|
| 393 |
|
|
|
|
| 398 |
model_name = gr.Radio(
|
| 399 |
MODEL_NAMES + [OWN_MODEL_NAME],
|
| 400 |
type="value",
|
| 401 |
+
label="B) BERT-like model.",
|
| 402 |
)
|
| 403 |
own_model_name = gr.Textbox(
|
| 404 |
+
label="C) If you selected an 'add-a-model' model, put any Hugging Face pipeline model name (that supports the fill-mask task) here.",
|
| 405 |
)
|
| 406 |
|
| 407 |
gr.Markdown("D) Pick if you want to the predictions normalied to these gendered terms only.")
|
| 408 |
gr.Markdown("E) Also tell the demo what special token you will use in your input text, that you would like replaced with the spectrum of values you listed above.")
|
| 409 |
+
gr.Markdown("And F) the degree of polynomial fit used for high-lighting potential spurious association.")
|
| 410 |
|
| 411 |
|
| 412 |
with gr.Row():
|
|
|
|
| 416 |
type="index",
|
| 417 |
)
|
| 418 |
place_holder = gr.Textbox(
|
| 419 |
+
label="E) Special token place-holder",
|
| 420 |
)
|
| 421 |
n_fit = gr.Dropdown(
|
| 422 |
list(range(1, 5)),
|
| 423 |
+
label="F) Degree of polynomial fit",
|
| 424 |
type="value",
|
| 425 |
)
|
| 426 |
|
|
|
|
| 429 |
|
| 430 |
with gr.Row():
|
| 431 |
input_text = gr.Textbox(
|
| 432 |
+
lines=2,
|
| 433 |
+
label="G) Input text with pronouns and place-holder token",
|
| 434 |
)
|
| 435 |
|
| 436 |
gr.Markdown("## Outputs!")
|