Commit
·
22f82e7
1
Parent(s):
15ae508
Text changes, added FAQ
Browse files- about.py +35 -29
- app.py +36 -34
- constants.py +1 -0
about.py
CHANGED
@@ -1,12 +1,9 @@
|
|
1 |
-
from constants import TERMS_URL
|
2 |
|
3 |
ABOUT_INTRO = f"""
|
4 |
## About this challenge
|
5 |
|
6 |
-
### Register [here](https://datapoints.ginkgo.bio/ai-competitions/2025-abdev-competition)
|
7 |
-
|
8 |
-
- Task: Predict 5 antibody developability properties based on the assays in the [GDPa1 dataset](https://huggingface.co/datasets/ginkgo-datapoints/GDPa1)
|
9 |
-
- Submissions are scored on a private held-out test set.
|
10 |
|
11 |
#### What is antibody developability and why is it important?
|
12 |
|
@@ -23,7 +20,7 @@ For each of these 6 prizes, participants have the choice between **$10k in data
|
|
23 |
See the FAQs below or the [competition terms]({TERMS_URL}) for more details.
|
24 |
"""
|
25 |
|
26 |
-
ABOUT_TEXT = """
|
27 |
|
28 |
#### How to participate?
|
29 |
|
@@ -33,25 +30,7 @@ ABOUT_TEXT = """
|
|
33 |
4. **Choose a validation track**. You must first upload predictions on the validation set before submitting predictions on the private test set:
|
34 |
- **Track 1**: If you already have a developability model, you can submit your predictions for the GDPa1 public dataset.
|
35 |
- **Track 2**: If you don't have a model, train one using cross-validation on the GDPa1 dataset and submit your predictions under the "Cross-validation" option.
|
36 |
-
5. **Submit your predictions** as a CSV
|
37 |
-
You do **not** need to predict all 5 properties — each property has its own leaderboard and prize.
|
38 |
-
If you click the "Anonymous" checkbox, your predictions will not be displayed alongside your Hugging Face username but with a random ID.
|
39 |
-
6. **Final test submission**: Download test sequences from the "✉️ Submit" tab and upload predictions.
|
40 |
-
|
41 |
-
The validation set results will appear on the leaderboard after a few minutes. The private test set results will not appear on the leaderboard, and will be used to determine the winners at the close of the competition.
|
42 |
-
We may release private test set results at intermediate points during the competition.
|
43 |
-
|
44 |
-
There is an example submission file on the "✉️ Submit" tab. When you are ready to submit your predictions for the private test set, download the test set sequences from the "✉️ Submit" tab and follow the same process.
|
45 |
-
|
46 |
-
For the cross-validation metrics (if training only on the GDPa1 dataset), use the `"hierarchical_cluster_IgG_isotype_stratified_fold"` column to split the dataset into folds and make predictions for each of the folds.
|
47 |
-
Submit a CSV file in the same format but also containing the `"hierarchical_cluster_IgG_isotype_stratified_fold"` column.
|
48 |
-
There is also an example cross-validation submission file on the "✉️ Submit" tab, and we will be releasing a full cross-validation code tutorial shortly.
|
49 |
-
|
50 |
-
#### How to evaluate?
|
51 |
-
|
52 |
-
You can easily calculate the Spearman correlation coefficient on the GDPa1 dataset yourself before uploading to the leaderboard.
|
53 |
-
Simply use the `spearmanr(predictions, targets, nan_policy='omit')` function from `scipy.stats`.
|
54 |
-
For the heldout private set, we will calculate these results privately at the end of the competition (and possibly at other points throughout the competition) - but there will not be "rolling results" on the private test set.
|
55 |
|
56 |
#### How to contribute?
|
57 |
|
@@ -62,8 +41,8 @@ We'd like to add some more existing models to the leaderboard. Some examples of
|
|
62 |
|
63 |
If you would like to collaborate with others, start a discussion on the "Community" tab at the top of this page.
|
64 |
"""
|
65 |
-
|
66 |
-
# Note(Lood): Add another note of "many models are trained on different datasets, and differing train/test splits, so this is a consistent way of comparing for a heldout set"
|
67 |
FAQS = {
|
68 |
"Is there a fee to enter?": "No. Participation is free of charge.",
|
69 |
"Who can participate?": "Anyone. We encourage academic labs, individuals, and especially industry teams who use developability models in production.",
|
@@ -80,10 +59,12 @@ FAQS = {
|
|
80 |
"No. This is just a predictive competition, which will be judged according to the correlation between predictions and experimental values. There may be a generative round in the future."
|
81 |
),
|
82 |
"Can I participate anonymously?": (
|
83 |
-
"Yes! Please still create an anonymous Hugging Face account so that we can uniquely associate submissions
|
|
|
|
|
84 |
),
|
85 |
"How is intellectual property handled?": (
|
86 |
-
"Participants retain IP rights to the methods they use and develop during the tournament. Read more details in our terms here
|
87 |
),
|
88 |
"Do I need to submit my code / methods in order to participate?": (
|
89 |
"No, there are no requirements to submit code / methods and submitted predictions remain private. "
|
@@ -91,6 +72,11 @@ FAQS = {
|
|
91 |
"Top performing participants will be requested to identify themselves at the end of the tournament. "
|
92 |
"There will be one prize for the best open-source model, which will require code / methods to be available."
|
93 |
),
|
|
|
|
|
|
|
|
|
|
|
94 |
"How often does the leaderboard update?": (
|
95 |
"The leaderboard should reflect new submissions within a minute of submitting. Note that the leaderboard will not show the results on the private test set, these will be calculated once at the end of the tournament (and possibly at another occasion before that)."
|
96 |
),
|
@@ -128,3 +114,23 @@ FAQS = {
|
|
128 |
"If there are any intermediate releases of private test set results, these will not affect the final ranking."
|
129 |
),
|
130 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from constants import ABOUT_TAB_NAME, ASSAY_LIST, TERMS_URL
|
2 |
|
3 |
ABOUT_INTRO = f"""
|
4 |
## About this challenge
|
5 |
|
6 |
+
### Register [here](https://datapoints.ginkgo.bio/ai-competitions/2025-abdev-competition) on the Ginkgo website before submitting
|
|
|
|
|
|
|
7 |
|
8 |
#### What is antibody developability and why is it important?
|
9 |
|
|
|
20 |
See the FAQs below or the [competition terms]({TERMS_URL}) for more details.
|
21 |
"""
|
22 |
|
23 |
+
ABOUT_TEXT = f"""
|
24 |
|
25 |
#### How to participate?
|
26 |
|
|
|
30 |
4. **Choose a validation track**. You must first upload predictions on the validation set before submitting predictions on the private test set:
|
31 |
- **Track 1**: If you already have a developability model, you can submit your predictions for the GDPa1 public dataset.
|
32 |
- **Track 2**: If you don't have a model, train one using cross-validation on the GDPa1 dataset and submit your predictions under the "Cross-validation" option.
|
33 |
+
5. **Submit your predictions** as a CSV on the "✉️ Submit" tab on this page (you are currently on the "{ABOUT_TAB_NAME}" tab).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
34 |
|
35 |
#### How to contribute?
|
36 |
|
|
|
41 |
|
42 |
If you would like to collaborate with others, start a discussion on the "Community" tab at the top of this page.
|
43 |
"""
|
44 |
+
|
45 |
+
# Note(Lood): Significance: Add another note of "many models are trained on different datasets, and differing train/test splits, so this is a consistent way of comparing for a heldout set"
|
46 |
FAQS = {
|
47 |
"Is there a fee to enter?": "No. Participation is free of charge.",
|
48 |
"Who can participate?": "Anyone. We encourage academic labs, individuals, and especially industry teams who use developability models in production.",
|
|
|
59 |
"No. This is just a predictive competition, which will be judged according to the correlation between predictions and experimental values. There may be a generative round in the future."
|
60 |
),
|
61 |
"Can I participate anonymously?": (
|
62 |
+
"Yes! Please still create an anonymous Hugging Face account so that we can uniquely associate submissions and add an email on the [registration page](https://datapoints.ginkgo.bio/ai-competitions/2025-abdev-competition) so that we can contact participants throughout the competition."
|
63 |
+
"Note that top participants will need to identify themselves at the end of the tournament to receive prizes / recognition. "
|
64 |
+
"If there are any concerns about anonymity, please contact us at [email protected] - you can even send us a CSV of submissions from a burner email if necessary! 🥷"
|
65 |
),
|
66 |
"How is intellectual property handled?": (
|
67 |
+
f"Participants retain IP rights to the methods they use and develop during the tournament. Read more details in our terms [here]({TERMS_URL})."
|
68 |
),
|
69 |
"Do I need to submit my code / methods in order to participate?": (
|
70 |
"No, there are no requirements to submit code / methods and submitted predictions remain private. "
|
|
|
72 |
"Top performing participants will be requested to identify themselves at the end of the tournament. "
|
73 |
"There will be one prize for the best open-source model, which will require code / methods to be available."
|
74 |
),
|
75 |
+
"How exactly can I evaluate my model?": (
|
76 |
+
"You can easily calculate the Spearman correlation coefficient on the GDPa1 dataset yourself before uploading to the leaderboard. "
|
77 |
+
"Simply use the `spearmanr(predictions, targets, nan_policy='omit')` function from `scipy.stats`. "
|
78 |
+
"For the heldout private set, we will calculate these Spearman correlations privately at the end of the competition (and possibly at other points throughout the competition) - but there will not be 'rolling results' on the private test set to prevent test set leakage."
|
79 |
+
),
|
80 |
"How often does the leaderboard update?": (
|
81 |
"The leaderboard should reflect new submissions within a minute of submitting. Note that the leaderboard will not show the results on the private test set, these will be calculated once at the end of the tournament (and possibly at another occasion before that)."
|
82 |
),
|
|
|
114 |
"If there are any intermediate releases of private test set results, these will not affect the final ranking."
|
115 |
),
|
116 |
}
|
117 |
+
|
118 |
+
SUBMIT_INTRUCTIONS = f"""
|
119 |
+
# Antibody Developability Submission
|
120 |
+
Upload a CSV to get a score!
|
121 |
+
List of valid property names: `{', '.join(ASSAY_LIST)}`.
|
122 |
+
|
123 |
+
You do **not** need to predict all 5 properties — each property has its own leaderboard and prize.
|
124 |
+
|
125 |
+
## Instructions
|
126 |
+
1. **Submit your predictions** as a CSV with `antibody_name` + one column per property you are predicting (e.g. `"antibody_name,Titer,PR_CHO"` if your model predicts Titer and Polyreactivity).
|
127 |
+
2. **Final test submission**: Download test sequences from the example files below and upload predictions.
|
128 |
+
|
129 |
+
The validation set results should appear on the leaderboard after a few seconds. The **private test set results will not appear on the leaderboards**, and will be used to determine the winners at the close of the competition.
|
130 |
+
We may release private test set results at intermediate points during the competition.
|
131 |
+
|
132 |
+
## Cross-validation
|
133 |
+
|
134 |
+
For the cross-validation metrics (if training only on the GDPa1 dataset), use the `"hierarchical_cluster_IgG_isotype_stratified_fold"` column to split the dataset into folds and make predictions for each of the folds.
|
135 |
+
Submit a CSV file in the same format but also containing the `"hierarchical_cluster_IgG_isotype_stratified_fold"` column.
|
136 |
+
"""
|
app.py
CHANGED
@@ -11,9 +11,10 @@ from constants import (
|
|
11 |
ASSAY_DESCRIPTION,
|
12 |
EXAMPLE_FILE_DICT,
|
13 |
LEADERBOARD_DISPLAY_COLUMNS,
|
|
|
14 |
TERMS_URL,
|
15 |
)
|
16 |
-
from about import ABOUT_INTRO, ABOUT_TEXT, FAQS
|
17 |
from submit import make_submission
|
18 |
|
19 |
def format_leaderboard_table(df_results: pd.DataFrame, assay: str | None = None):
|
@@ -68,33 +69,35 @@ with gr.Blocks() as demo:
|
|
68 |
|
69 |
timer.tick(fn=update_current_dataframe, outputs=data_version)
|
70 |
|
71 |
-
|
72 |
-
|
73 |
-
|
|
|
|
|
|
|
|
|
74 |
|
75 |
-
|
76 |
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
|
|
|
|
82 |
gr.Image(
|
83 |
value="./assets/competition_logo.jpg",
|
84 |
show_label=False,
|
85 |
-
# elem_classes=["resized-image"],
|
86 |
show_download_button=False,
|
87 |
-
width="
|
88 |
)
|
|
|
|
|
|
|
89 |
gr.Markdown(ABOUT_INTRO)
|
90 |
gr.Image(value="./assets/prediction_explainer.png", show_label=False, show_download_button=False, width="50vw")
|
91 |
gr.Markdown(ABOUT_TEXT)
|
92 |
-
gr.Markdown("### FAQs")
|
93 |
-
for i, (question, answer) in enumerate(FAQS.items()):
|
94 |
-
# Would love to make questions bold but accordion doesn't support it
|
95 |
-
question = f"{i+1}. {question}"
|
96 |
-
with gr.Accordion(question, open=False):
|
97 |
-
gr.Markdown(f"*{answer}*") # Italics for answers
|
98 |
|
99 |
# Procedurally make these 5 tabs
|
100 |
for i, assay in enumerate(ASSAY_LIST):
|
@@ -124,16 +127,7 @@ with gr.Blocks() as demo:
|
|
124 |
data_version.change(fn=refresh_overall_leaderboard, outputs=lb)
|
125 |
|
126 |
with gr.TabItem("✉️ Submit", elem_id="boundary-benchmark-tab-table"):
|
127 |
-
gr.Markdown(
|
128 |
-
f"""
|
129 |
-
# Antibody Developability Submission
|
130 |
-
Upload a CSV to get a score!
|
131 |
-
List of valid property names: `{', '.join(ASSAY_LIST)}`.
|
132 |
-
|
133 |
-
Please use your Hugging Face account name to submit your model - we use this to track separate submissions, and if you would like to remain anonymous please set up an anonymous huggingface account.
|
134 |
-
Your submission will be evaluated and added to the leaderboard.
|
135 |
-
"""
|
136 |
-
)
|
137 |
submission_type_state = gr.State(value="GDPa1")
|
138 |
download_file_state = gr.State(value=EXAMPLE_FILE_DICT["GDPa1"])
|
139 |
|
@@ -142,7 +136,12 @@ with gr.Blocks() as demo:
|
|
142 |
username_input = gr.Textbox(
|
143 |
label="Username",
|
144 |
placeholder="Enter your Hugging Face username",
|
145 |
-
info="This will be used to
|
|
|
|
|
|
|
|
|
|
|
146 |
)
|
147 |
model_name_input = gr.Textbox(
|
148 |
label="Model Name",
|
@@ -155,11 +154,6 @@ with gr.Blocks() as demo:
|
|
155 |
info="Describe your model, training data, or methodology.",
|
156 |
lines=3,
|
157 |
)
|
158 |
-
anonymous_checkbox = gr.Checkbox(
|
159 |
-
label="Anonymous",
|
160 |
-
value=False,
|
161 |
-
info="If checked, your username will be replaced with an anonymous username on the leaderboard.",
|
162 |
-
)
|
163 |
registration_code = gr.Textbox(
|
164 |
label="Registration Code",
|
165 |
placeholder="Enter your registration code",
|
@@ -231,6 +225,14 @@ with gr.Blocks() as demo:
|
|
231 |
inputs=[message],
|
232 |
outputs=[message],
|
233 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
234 |
# Footnote
|
235 |
gr.Markdown(
|
236 |
f"""
|
|
|
11 |
ASSAY_DESCRIPTION,
|
12 |
EXAMPLE_FILE_DICT,
|
13 |
LEADERBOARD_DISPLAY_COLUMNS,
|
14 |
+
ABOUT_TAB_NAME,
|
15 |
TERMS_URL,
|
16 |
)
|
17 |
+
from about import ABOUT_INTRO, ABOUT_TEXT, FAQS, SUBMIT_INTRUCTIONS
|
18 |
from submit import make_submission
|
19 |
|
20 |
def format_leaderboard_table(df_results: pd.DataFrame, assay: str | None = None):
|
|
|
69 |
|
70 |
timer.tick(fn=update_current_dataframe, outputs=data_version)
|
71 |
|
72 |
+
## Header
|
73 |
+
|
74 |
+
with gr.Row():
|
75 |
+
with gr.Column(scale=6): # bigger text area
|
76 |
+
gr.Markdown(
|
77 |
+
f"""
|
78 |
+
## Welcome to the Ginkgo Antibody Developability Benchmark!
|
79 |
|
80 |
+
**Beta version, not publicly launched yet**
|
81 |
|
82 |
+
Participants can submit their model to the leaderboards by simply uploading a CSV file (see the "✉️ Submit" tab).
|
83 |
+
|
84 |
+
You can **predict any or all of the 5 properties**, and each property has its own leaderboard.
|
85 |
+
See more details in the "{ABOUT_TAB_NAME}" tab.
|
86 |
+
"""
|
87 |
+
)
|
88 |
+
with gr.Column(scale=2): # smaller side column for logo
|
89 |
gr.Image(
|
90 |
value="./assets/competition_logo.jpg",
|
91 |
show_label=False,
|
|
|
92 |
show_download_button=False,
|
93 |
+
width="25vw", # Take up the width of the column (2/8 = 1/4)
|
94 |
)
|
95 |
+
|
96 |
+
with gr.Tabs(elem_classes="tab-buttons"):
|
97 |
+
with gr.TabItem(ABOUT_TAB_NAME, elem_id="abdev-benchmark-tab-table"):
|
98 |
gr.Markdown(ABOUT_INTRO)
|
99 |
gr.Image(value="./assets/prediction_explainer.png", show_label=False, show_download_button=False, width="50vw")
|
100 |
gr.Markdown(ABOUT_TEXT)
|
|
|
|
|
|
|
|
|
|
|
|
|
101 |
|
102 |
# Procedurally make these 5 tabs
|
103 |
for i, assay in enumerate(ASSAY_LIST):
|
|
|
127 |
data_version.change(fn=refresh_overall_leaderboard, outputs=lb)
|
128 |
|
129 |
with gr.TabItem("✉️ Submit", elem_id="boundary-benchmark-tab-table"):
|
130 |
+
gr.Markdown(SUBMIT_INTRUCTIONS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
131 |
submission_type_state = gr.State(value="GDPa1")
|
132 |
download_file_state = gr.State(value=EXAMPLE_FILE_DICT["GDPa1"])
|
133 |
|
|
|
136 |
username_input = gr.Textbox(
|
137 |
label="Username",
|
138 |
placeholder="Enter your Hugging Face username",
|
139 |
+
info="This will be used to identify valid submissions, and to update your results if you submit again.",
|
140 |
+
)
|
141 |
+
anonymous_checkbox = gr.Checkbox(
|
142 |
+
label="Anonymous",
|
143 |
+
value=False,
|
144 |
+
info="If checked, your username will be replaced with an anonymous hash on the leaderboard.",
|
145 |
)
|
146 |
model_name_input = gr.Textbox(
|
147 |
label="Model Name",
|
|
|
154 |
info="Describe your model, training data, or methodology.",
|
155 |
lines=3,
|
156 |
)
|
|
|
|
|
|
|
|
|
|
|
157 |
registration_code = gr.Textbox(
|
158 |
label="Registration Code",
|
159 |
placeholder="Enter your registration code",
|
|
|
225 |
inputs=[message],
|
226 |
outputs=[message],
|
227 |
)
|
228 |
+
with gr.Tab("❔ FAQs"):
|
229 |
+
gr.Markdown("# Frequently Asked Questions")
|
230 |
+
for i, (question, answer) in enumerate(FAQS.items()):
|
231 |
+
# Would love to make questions bold but accordion doesn't support it
|
232 |
+
question = f"{i+1}. {question}"
|
233 |
+
with gr.Accordion(question, open=False):
|
234 |
+
gr.Markdown(f"*{answer}*") # Italics for answers
|
235 |
+
|
236 |
# Footnote
|
237 |
gr.Markdown(
|
238 |
f"""
|
constants.py
CHANGED
@@ -28,6 +28,7 @@ ASSAY_EMOJIS = {
|
|
28 |
"Tm2": "🌡️",
|
29 |
"Titer": "🧪",
|
30 |
}
|
|
|
31 |
|
32 |
REGISTRATION_CODE = "GINKGO-ABDEV-2025"
|
33 |
TERMS_URL = "https://euphsfcyogalqiqsawbo.supabase.co/storage/v1/object/public/gdpweb/pdfs/2025%20Ginkgo%20Antibody%20Developability%20Prediction%20Competition%202025-08-28-v2.pdf"
|
|
|
28 |
"Tm2": "🌡️",
|
29 |
"Titer": "🧪",
|
30 |
}
|
31 |
+
ABOUT_TAB_NAME = "ℹ️ About"
|
32 |
|
33 |
REGISTRATION_CODE = "GINKGO-ABDEV-2025"
|
34 |
TERMS_URL = "https://euphsfcyogalqiqsawbo.supabase.co/storage/v1/object/public/gdpweb/pdfs/2025%20Ginkgo%20Antibody%20Developability%20Prediction%20Competition%202025-08-28-v2.pdf"
|