Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Changed.
Browse files- app.py +15 -10
- src/about.py +123 -128
- src/display/css_html_js.py +79 -27
app.py
CHANGED
|
@@ -173,18 +173,23 @@ def gate_submission(oauth_token: gr.OAuthToken | None):
|
|
| 173 |
|
| 174 |
|
| 175 |
def get_theme():
|
| 176 |
-
# return gr.themes.Soft(
|
| 177 |
-
# primary_hue=gr.themes.colors.blue,
|
| 178 |
-
# secondary_hue=gr.themes.colors.sky,
|
| 179 |
-
# neutral_hue=gr.themes.colors.gray,
|
| 180 |
-
# ).set(
|
| 181 |
-
# body_background_fill="#FFFFFF",
|
| 182 |
-
# panel_background_fill="#f3f4f6",
|
| 183 |
-
# )
|
| 184 |
return "light"
|
| 185 |
|
| 186 |
|
| 187 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
with blocks:
|
| 189 |
|
| 190 |
with gr.Tabs(elem_classes="tab-buttons") as tabs:
|
|
@@ -214,7 +219,7 @@ with blocks:
|
|
| 214 |
|
| 215 |
login_box = gr.Group(visible=True)
|
| 216 |
with login_box:
|
| 217 |
-
gr.Markdown("Please sign in with Hugging Face to submit")
|
| 218 |
gr.LoginButton()
|
| 219 |
|
| 220 |
submit_panel = gr.Group(visible=False)
|
|
|
|
| 173 |
|
| 174 |
|
| 175 |
def get_theme():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
return "light"
|
| 177 |
|
| 178 |
|
| 179 |
+
# Force light theme even if HF user prefers dark
|
| 180 |
+
_force_light_js = (
|
| 181 |
+
"() => {\n"
|
| 182 |
+
" try {\n"
|
| 183 |
+
" document.body.classList.remove('dark');\n"
|
| 184 |
+
" document.documentElement.classList.remove('dark');\n"
|
| 185 |
+
" document.documentElement.setAttribute('data-theme','light');\n"
|
| 186 |
+
" // Some HF shells add data-color-mode\n"
|
| 187 |
+
" document.documentElement.setAttribute('data-color-mode','light');\n"
|
| 188 |
+
" } catch (e) {}\n"
|
| 189 |
+
"}"
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
blocks = gr.Blocks(css=custom_css, theme=get_theme(), js=_force_light_js)
|
| 193 |
with blocks:
|
| 194 |
|
| 195 |
with gr.Tabs(elem_classes="tab-buttons") as tabs:
|
|
|
|
| 219 |
|
| 220 |
login_box = gr.Group(visible=True)
|
| 221 |
with login_box:
|
| 222 |
+
gr.Markdown("Please sign in with Hugging Face to submit", elem_classes="markdown-text")
|
| 223 |
gr.LoginButton()
|
| 224 |
|
| 225 |
submit_panel = gr.Group(visible=False)
|
src/about.py
CHANGED
|
@@ -3,162 +3,157 @@ PAPER_URL = "https://arxiv.org/abs/2507.13337"
|
|
| 3 |
|
| 4 |
WHAT_IS_F1_HTML = f"""
|
| 5 |
<!DOCTYPE html>
|
| 6 |
-
<html lang
|
| 7 |
<body>
|
| 8 |
-
<main class
|
| 9 |
-
<header class
|
| 10 |
-
<h1 class
|
| 11 |
</header>
|
| 12 |
<section>
|
| 13 |
-
<p class
|
| 14 |
-
<p class
|
| 15 |
-
<p class
|
| 16 |
-
|
| 17 |
-
<!--
|
| 18 |
-
<div class
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
<
|
| 25 |
-
<
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
</div>
|
| 32 |
|
| 33 |
-
<!-- Tabbed Problem Viewer -->
|
| 34 |
-
<div class
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
</nav>
|
| 41 |
</div>
|
| 42 |
-
<div class
|
| 43 |
-
<
|
| 44 |
-
|
| 45 |
-
<p class="mb-2"><strong>Objective:</strong> Compute the sum of all weights of sets S⊆V such that the graph G[S], induced over S, is a collection of disjoint paths and cycles.</p>
|
| 46 |
-
</div>
|
| 47 |
-
<div id="problem-tab-content-tier1" class="problem-tab-content f1-problem-box hidden">
|
| 48 |
-
<p class="font-bold text-lg mb-2">Maximal-Union-of-Paths-and-Cycles</p>
|
| 49 |
-
<p class="mb-2"><strong>Objective:</strong> Compute the sum of all weights of sets S⊆V such that the graph G[S], induced over S, is a collection of disjoint paths and cycles, and S is maximal with respect to this property.</p>
|
| 50 |
-
</div>
|
| 51 |
-
<div id="problem-tab-content-tier2" class="problem-tab-content f1-problem-box hidden">
|
| 52 |
-
<p class="font-bold text-lg mb-2">Maximal-Union-of-Cycles</p>
|
| 53 |
-
<p class="mb-2"><strong>Objective:</strong> Compute the sum of all weights of sets S⊆V such that the graph G[S], induced over S, is a collection of disjoint cycles, and S is maximal with respect to this property.</p>
|
| 54 |
-
</div>
|
| 55 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
</div>
|
| 57 |
|
| 58 |
-
<p class
|
| 59 |
-
<p class
|
| 60 |
</section>
|
| 61 |
<section>
|
| 62 |
-
<h2 class
|
| 63 |
-
<
|
| 64 |
-
|
| 65 |
-
<p class
|
| 66 |
-
<p class
|
| 67 |
-
<p class="mb-2"><strong>Objective:</strong> Compute the sum of all weights of sets S⊆V such that the graph G[S], induced over S, does not contain any cycle of length four.</p>
|
| 68 |
-
<p class="text-sm text-gray-600"><strong>Notation:</strong> The weight of a set of vertices S is defined as w(S) ≜ ∑<sub>v∈S</sub>w(v). The final result should be returned modulo 10<sup>9</sup>+7.</p>
|
| 69 |
</div>
|
| 70 |
-
<p class
|
| 71 |
-
<blockquote class
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
<img src="/file=assets/bag_modifications.png" alt="An illustration of local modifications to bags (dashed boxes)" class="max-w-full md:max-w-2xl mx-auto rounded-lg shadow-md">
|
| 77 |
-
<figcaption class="f1-figcaption">An illustration of local modifications to bags: Introduce, Forget, and Join.</figcaption>
|
| 78 |
</figure>
|
| 79 |
-
<p class
|
| 80 |
-
<figure class
|
| 81 |
-
<video class
|
| 82 |
-
<source src
|
| 83 |
Your browser does not support the video tag.
|
| 84 |
</video>
|
| 85 |
-
<figcaption class
|
| 86 |
</figure>
|
| 87 |
-
<p class
|
| 88 |
-
</section>
|
| 89 |
-
<section>
|
| 90 |
-
<h2 class="text-3xl font-bold text-gray-900 f1-h2">Guiding Principles</h2>
|
| 91 |
-
<ul class="list-disc list-inside space-y-4">
|
| 92 |
-
<li class="f1-li"><strong>An In-Distribution Benchmark for Reasoning.</strong> Unlike <a href="https://arcprize.org/arc-agi" target="_blank" rel="noopener noreferrer" class="f1-a">benchmarks</a> that test for out-of-distribution generalisation, FormulaOne presents problems that are squarely <strong>in-distribution</strong> for models trained on code. Essentially, dynamic programming on graphs is the “bread and butter” of algorithmic programming. Thus, models’ current failure on FormulaOne highlights a fundamental deficit in deep, multi-step reasoning, rather than a lack of domain exposure.</li>
|
| 93 |
-
<li class="f1-li"><strong>An Unbounded Environment for Reinforcement Learning.</strong> The MSO framework allows for the generation of a nearly infinite stream of algorithmic problems with verifiable solutions, making it an ideal environment for training and evaluating agents with Reinforcement Learning with Verifiable Rewards (RLVR).</li>
|
| 94 |
-
<li class="f1-li"><strong>Probing the Frontiers of Complexity Theory.</strong> Many problems in our dataset are related to central conjectures in fine-grained complexity, such as the <a href="https://en.wikipedia.org/wiki/Exponential_time_hypothesis" target="_blank" rel="noopener noreferrer" class="f1-a">Strong Exponential Time Hypothesis</a> (SETH). If a model were to discover a significantly faster algorithm for one of these problems, that would constitute a significant contribution to theoretical computer science.</li>
|
| 95 |
-
</ul>
|
| 96 |
</section>
|
| 97 |
-
<section id
|
| 98 |
-
<h2 class
|
| 99 |
-
<p class
|
| 100 |
-
<p class
|
| 101 |
-
<ul class
|
| 102 |
-
<li class
|
| 103 |
-
<li class
|
| 104 |
-
<li class
|
| 105 |
</ul>
|
| 106 |
-
<p class
|
| 107 |
-
<p class
|
| 108 |
-
<h3 class
|
| 109 |
-
<p class
|
| 110 |
-
<figure class
|
| 111 |
-
<img src
|
| 112 |
-
<figcaption class
|
| 113 |
</figure>
|
| 114 |
-
<p class
|
| 115 |
-
<figure class
|
| 116 |
-
<img src
|
| 117 |
-
<figcaption class
|
| 118 |
</figure>
|
| 119 |
-
<p class
|
| 120 |
</section>
|
| 121 |
</main>
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
<script>
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
t.content.classList.add('hidden');
|
| 143 |
-
}}
|
| 144 |
-
}});
|
| 145 |
-
tab.btn.classList.add(...activeClasses.split(' '));
|
| 146 |
-
tab.btn.classList.remove(...inactiveClasses.split(' '));
|
| 147 |
-
tab.content.classList.remove('hidden');
|
| 148 |
-
}});
|
| 149 |
-
}}
|
| 150 |
-
}});
|
| 151 |
-
|
| 152 |
-
// Set initial active tab
|
| 153 |
-
if (problemTabs.length > 0 && problemTabs[0].btn) {{
|
| 154 |
-
problemTabs[0].btn.click();
|
| 155 |
-
}}
|
| 156 |
-
}}, 100); // 100ms delay to ensure elements are in the DOM
|
| 157 |
</script>
|
| 158 |
</body>
|
| 159 |
</html>
|
| 160 |
"""
|
| 161 |
|
|
|
|
| 162 |
EVALUATION_QUEUE_TEXT = """
|
| 163 |
## Submitting to the FormulaOne Leaderboard
|
| 164 |
|
|
|
|
| 3 |
|
| 4 |
WHAT_IS_F1_HTML = f"""
|
| 5 |
<!DOCTYPE html>
|
| 6 |
+
<html lang=\"en\">
|
| 7 |
<body>
|
| 8 |
+
<main class=\"f1-container\">
|
| 9 |
+
<header class=\"text-center mb-12\">
|
| 10 |
+
<h1 class=\"f1-h1\">FormulaOne</h1>
|
| 11 |
</header>
|
| 12 |
<section>
|
| 13 |
+
<p class=\"text-lg mb-4 f1-p\">Frontier AI models have recently demonstrated strong performance on mathematical and algorithmic benchmarks, including earning <a href=\"https://deepmind.google/discover/blog/advanced-version-of-gemini-with-deep-think-officially-achieves-gold-medal-standard-at-the-international-mathematical-olympiad/\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">gold medals in olympiads</a>, and attaining <a href=\"https://arxiv.org/html/2502.06807v1\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">top percentile ratings</a> in competitive programming contests. How well do such benchmarks capture the true depth of algorithmic reasoning, as it arises in real-world research problems?</p>
|
| 14 |
+
<p class=\"text-lg mb-4 f1-p\">We believe that existing benchmarks fail to capture the deep reasoning skills required for complex, research-level algorithmic problems. To address this gap, <a href=\"{PAPER_URL}\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">we introduce <strong>FormulaOne</strong></a>.</p>
|
| 15 |
+
<p class=\"mb-4 f1-p\"><strong>FormulaOne</strong> consists of 220 novel dynamic programming problems over graphs. The problems are organised into three categories, ranging from moderate difficulty and all the way up to research-level.</p>
|
| 16 |
+
|
| 17 |
+
<!-- Beautiful comparison table -->
|
| 18 |
+
<div class=\"f1-table-wrapper\">
|
| 19 |
+
<table class=\"f1-table\" aria-label=\"Comparison of FormulaOne tiers\">
|
| 20 |
+
<thead>
|
| 21 |
+
<tr>
|
| 22 |
+
<th scope=\"col\">Attribute</th>
|
| 23 |
+
<th scope=\"col\">Warmup</th>
|
| 24 |
+
<th scope=\"col\">Tier 1</th>
|
| 25 |
+
<th scope=\"col\">Tier 2</th>
|
| 26 |
+
</tr>
|
| 27 |
+
</thead>
|
| 28 |
+
<tbody>
|
| 29 |
+
<tr>
|
| 30 |
+
<td><strong>Problems</strong></td>
|
| 31 |
+
<td>100</td>
|
| 32 |
+
<td>100</td>
|
| 33 |
+
<td>20</td>
|
| 34 |
+
</tr>
|
| 35 |
+
<tr>
|
| 36 |
+
<td><strong>Difficulty</strong></td>
|
| 37 |
+
<td>Moderate</td>
|
| 38 |
+
<td>Challenging</td>
|
| 39 |
+
<td>Research‑level</td>
|
| 40 |
+
</tr>
|
| 41 |
+
<tr>
|
| 42 |
+
<td><strong>Description</strong></td>
|
| 43 |
+
<td>A set of 100 “easier” problems.</td>
|
| 44 |
+
<td>A set of 100 challenging problems.</td>
|
| 45 |
+
<td>A set of 20 highly challenging problems.</td>
|
| 46 |
+
</tr>
|
| 47 |
+
</tbody>
|
| 48 |
+
</table>
|
| 49 |
</div>
|
| 50 |
|
| 51 |
+
<!-- Tabbed Problem Viewer with bottom tabs -->
|
| 52 |
+
<div class=\"f1-tabs\" id=\"f1-problem-tabs\">
|
| 53 |
+
<div class=\"f1-tabs-header\">Examples of FormulaOne problems</div>
|
| 54 |
+
<div class=\"f1-tabs-content\">
|
| 55 |
+
<div id=\"f1-tab-warmup\" class=\"f1-problem-box\" role=\"tabpanel\">
|
| 56 |
+
<p class=\"mb-2\"><strong>Union of Paths and Cycles</strong></p>
|
| 57 |
+
<p class=\"mb-2 f1-p\"><strong>Objective:</strong> Compute the sum of all weights of sets \( S\\subseteq V \) such that the induced subgraph \( G[S] \) is a disjoint union of paths and cycles.</p>
|
|
|
|
| 58 |
</div>
|
| 59 |
+
<div id=\"f1-tab-tier1\" class=\"f1-problem-box\" role=\"tabpanel\" hidden>
|
| 60 |
+
<p class=\"mb-2\"><strong>Maximal Union of Paths and Cycles</strong></p>
|
| 61 |
+
<p class=\"mb-2 f1-p\"><strong>Objective:</strong> Compute the sum of all weights of sets \( S\\subseteq V \) such that \( G[S] \) is a disjoint union of paths and cycles, and \( S \) is maximal with respect to this property.</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
</div>
|
| 63 |
+
<div id=\"f1-tab-tier2\" class=\"f1-problem-box\" role=\"tabpanel\" hidden>
|
| 64 |
+
<p class=\"mb-2\"><strong>Maximal Union of Cycles</strong></p>
|
| 65 |
+
<p class=\"mb-2 f1-p\"><strong>Objective:</strong> Compute the sum of all weights of sets \( S\\subseteq V \) such that \( G[S] \) is a disjoint union of cycles, and \( S \) is maximal with respect to this property.</p>
|
| 66 |
+
</div>
|
| 67 |
+
</div>
|
| 68 |
+
<div class=\"f1-tabs-bar\" role=\"tablist\" aria-label=\"Example tiers\">
|
| 69 |
+
<button class=\"f1-tab-btn is-active\" data-target=\"f1-tab-warmup\" role=\"tab\" aria-controls=\"f1-tab-warmup\" aria-selected=\"true\">Warmup</button>
|
| 70 |
+
<button class=\"f1-tab-btn\" data-target=\"f1-tab-tier1\" role=\"tab\" aria-controls=\"f1-tab-tier1\">Tier 1</button>
|
| 71 |
+
<button class=\"f1-tab-btn\" data-target=\"f1-tab-tier2\" role=\"tab\" aria-controls=\"f1-tab-tier2\">Tier 2</button>
|
| 72 |
+
</div>
|
| 73 |
</div>
|
| 74 |
|
| 75 |
+
<p class=\"mt-6 mb-4 f1-p\">The latter category is incredibly demanding, requiring resolution of many points of uncertainty, and involving an array of reasoning steps, including topological and geometric insight, knowledge of mathematical domains such as extremal graph theory and logic, combinatorial considerations, precise implementation, and more.</p>
|
| 76 |
+
<p class=\"f1-p\">Despite <a href=\"https://epoch.ai/frontiermath\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">impressive</a> <a href=\"https://artificialanalysis.ai/evaluations/gpqa-diamond\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">performance</a> on existing benchmarks, presently <strong>no model solves even a single FormulaOne Tier 2 problem</strong>.<sup><a href=\"#evaluation\" class=\"f1-a\">1</a></sup></p>
|
| 77 |
</section>
|
| 78 |
<section>
|
| 79 |
+
<h2 class=\"f1-h2\">An “Infinite Well” of Problems</h2>
|
| 80 |
+
<div class=\"f1-problem-box\">
|
| 81 |
+
<p class=\"mb-2 f1-p\"><strong>Input:</strong> A tree‑like graph \( G=(V,E) \), a tree decomposition \( T \) of \( G \), and a weight function \( w:V\\to\\mathbb{{N}} \).</p>
|
| 82 |
+
<p class=\"mb-2 f1-p\"><strong>Objective:</strong> Compute the sum of all weights of sets \( S\\subseteq V \) such that the induced subgraph \( G[S] \) does not contain any 4‑cycle \( C_4 \).</p>
|
| 83 |
+
<p class=\"text-sm text-gray-600 f1-p\"><strong>Notation:</strong> The weight of a set of vertices \( S \) is defined as \( w(S) \\coloneqq \\sum_{{v\\in S}} w(v) \). The final result should be returned modulo \( 10^9+7 \).</p>
|
|
|
|
|
|
|
| 84 |
</div>
|
| 85 |
+
<p class=\"mb-4 f1-p\">While the problems are often natural to state, their solutions are far from obvious. The solvability of this vast class of problems is guaranteed by an algorithmic <strong>meta-theorem</strong> due to <a href=\"https://en.wikipedia.org/wiki/Courcelle%27s_theorem\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">Courcelle</a>, which broadly states:</p>
|
| 86 |
+
<blockquote class=\"my-6 f1-blockquote\">“For every sufficiently tree-like graph, any problem definable in an expressive formal logic — Monadic Second-Order (MSO) logic — can be solved by a dynamic programming algorithm that operates in time linear in the order of the graph.”</blockquote>
|
| 87 |
+
<p class=\"f1-p\">The key is to use a structure known as a tree decomposition, which organises the graph’s vertices into a series of overlapping sets, or “bags”, that are themselves arranged in a tree.</p>
|
| 88 |
+
<figure class=\"f1-figure\">
|
| 89 |
+
<img src=\"/file=assets/bag_modifications.png\" alt=\"An illustration of local modifications to bags (dashed boxes)\" class=\"max-w-full md:max-w-2xl mx-auto rounded-lg shadow-md\">
|
| 90 |
+
<figcaption class=\"f1-figcaption\">An illustration of local modifications to bags: Introduce, Forget, and Join.</figcaption>
|
|
|
|
|
|
|
| 91 |
</figure>
|
| 92 |
+
<p class=\"mb-4 f1-p\">An algorithm can then traverse this tree of bags, solving the problem piece by piece using dynamic programming. This process involves designing a “state” that summarises all necessary information about the partial solution within a bag, and then defining how this state transforms as vertices are introduced, forgotten, or bags are merged.</p>
|
| 93 |
+
<figure class=\"f1-figure\">
|
| 94 |
+
<video class=\"w-full max-w-2xl mx-auto rounded-lg shadow-lg\" autoplay loop muted playsinline>
|
| 95 |
+
<source src=\"/file=assets/dp_animation.mp4\" type=\"video/mp4\">
|
| 96 |
Your browser does not support the video tag.
|
| 97 |
</video>
|
| 98 |
+
<figcaption class=\"f1-figcaption\">Animation showing the design of a compressed dynamic programming state-space.</figcaption>
|
| 99 |
</figure>
|
| 100 |
+
<p class=\"f1-p\">The deceptive simplicity of the problem statements belies the <strong>extraordinary difficulty</strong> of discovering the correct dynamic programming solution. This process is riddled with subtle combinatorial and logical pitfalls, demanding a profound understanding of the problem’s underlying structure. For a detailed walkthrough of the fifteen interdependent reasoning steps required to solve a single hard problem – <strong>Maximal-Cluster-Graph</strong> – <a href=\"https://arxiv.org/pdf/2507.13337#appendix.A\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">see the appendix of our paper</a>.</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
</section>
|
| 102 |
+
<section id=\"evaluation\">
|
| 103 |
+
<h2 class=\"f1-h2\">Evaluation</h2>
|
| 104 |
+
<p class=\"mb-4 f1-p\">To give models the best possible chance of success, we provide a generous few-shot prompt that covers a broad array of the ideas and techniques involved in solving these problems. All models were evaluated using their highest available reasoning settings and with the maximum context length permitted.</p>
|
| 105 |
+
<p class=\"mb-4 f1-p\">Each submitted solution is subjected to a rigorous and automated <a href=\"https://arxiv.org/pdf/2507.13337#section.4\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">test suite</a> that measures three key aspects of its validity:</p>
|
| 106 |
+
<ul class=\"list-disc list-inside space-y-2 mb-6\">
|
| 107 |
+
<li class=\"f1-li\"><strong>Correctness:</strong> The output of the submitted algorithm must be correct on all graphs.</li>
|
| 108 |
+
<li class=\"f1-li\"><strong>Consistency:</strong> The solution must produce the same output for a given graph, regardless of the specific structure of its tree decomposition.</li>
|
| 109 |
+
<li class=\"f1-li\"><strong>Efficiency:</strong> The solution must be truly <a href=\"https://en.wikipedia.org/wiki/Parameterized_complexity\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">fixed-parameter linear</a>.</li>
|
| 110 |
</ul>
|
| 111 |
+
<p class=\"mb-4 f1-p\">To support research and encourage community contributions, the <code>FormulaOne-Warmup</code> dataset is released as a public resource for training and fine-tuning models. The complete test suite for all 100 Warmup problems is available, alongside a standalone evaluation environment, in our public GitHub repository: <a href=\"https://github.com/double-ai/formulaone-dataset/tree/main\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"f1-a\">https://github.com/double-ai/formulaone-dataset/tree/main</a>.</p>
|
| 112 |
+
<p class=\"f1-p\">In contrast, to maintain the integrity of the core benchmark, only a minimal subset of tests is released for the <code>FormulaOne Tier 1</code> and <code>Tier 2</code> problems.</p>
|
| 113 |
+
<h3 class=\"text-2xl font-bold text-gray-900 mt-8 mb-4\">Model Accuracy</h3>
|
| 114 |
+
<p class=\"mb-4 f1-p\">On the <strong>FormulaOne-Warmup</strong> problems, frontier models perform reasonably well. This confirms they have a foundational capability for these types of algorithmic tasks.</p>
|
| 115 |
+
<figure class=\"f1-figure\">
|
| 116 |
+
<img src=\"/file=assets/warmup_performance.png\" alt=\"Plot showing model performance on FormulaOne-Warmup\" class=\"max-w-full md:max-w-2xl mx-auto rounded-lg shadow-md\">
|
| 117 |
+
<figcaption class=\"f1-figcaption\">Performance of frontier models on the FormulaOne-Warmup dataset.</figcaption>
|
| 118 |
</figure>
|
| 119 |
+
<p class=\"mb-4 f1-p\">However, as the reasoning depth increases in <strong>FormulaOne Tier 1</strong>, and solutions require the discovery and integration of novel and more complex state representations, model performance drops off sharply.</p>
|
| 120 |
+
<figure class=\"f1-figure\">
|
| 121 |
+
<img src=\"/file=assets/tier1_performance.png\" alt=\"Plot showing model performance on FormulaOne Tier 1\" class=\"max-w-full md:max-w-2xl mx-auto rounded-lg shadow-md\">
|
| 122 |
+
<figcaption class=\"f1-figcaption\">Figure 1: Performance of frontier reasoning models on the FormulaOne dataset.</figcaption>
|
| 123 |
</figure>
|
| 124 |
+
<p class=\"f1-p\">This trend culminates in <strong>FormulaOne Tier 2</strong>, where the difficulty is characteristic of exploratory research problems. On this set of 20 problems, no current frontier model solves even a single one. This result starkly illustrates the gap that remains between high performance on existing benchmarks and the deep algorithmic reasoning required for truly complex problems.</p>
|
| 125 |
</section>
|
| 126 |
</main>
|
| 127 |
+
|
| 128 |
+
<!-- MathJax for LaTeX rendering -->
|
| 129 |
+
<script id=\"MathJax-script\" async src=\"https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js\"></script>
|
| 130 |
+
|
| 131 |
<script>
|
| 132 |
+
// Initialize bottom tabs and retrigger MathJax when switching
|
| 133 |
+
setTimeout(() => {{
|
| 134 |
+
const container = document.getElementById('f1-problem-tabs');
|
| 135 |
+
if (!container) return;
|
| 136 |
+
const buttons = container.querySelectorAll('.f1-tab-btn');
|
| 137 |
+
const panels = Array.from(container.querySelectorAll('[role="tabpanel"]'));
|
| 138 |
+
|
| 139 |
+
function activate(targetId) {{
|
| 140 |
+
panels.forEach(p => p.hidden = (p.id !== targetId));
|
| 141 |
+
buttons.forEach(b => b.classList.toggle('is-active', b.dataset.target === targetId));
|
| 142 |
+
if (window.MathJax && window.MathJax.typesetPromise) {{
|
| 143 |
+
window.MathJax.typesetPromise();
|
| 144 |
+
}}
|
| 145 |
+
}}
|
| 146 |
+
|
| 147 |
+
buttons.forEach(btn => btn.addEventListener('click', () => activate(btn.dataset.target)));
|
| 148 |
+
// Initial state
|
| 149 |
+
activate('f1-tab-warmup');
|
| 150 |
+
}}, 120);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
</script>
|
| 152 |
</body>
|
| 153 |
</html>
|
| 154 |
"""
|
| 155 |
|
| 156 |
+
|
| 157 |
EVALUATION_QUEUE_TEXT = """
|
| 158 |
## Submitting to the FormulaOne Leaderboard
|
| 159 |
|
src/display/css_html_js.py
CHANGED
|
@@ -1,26 +1,41 @@
|
|
| 1 |
custom_css = """
|
|
|
|
| 2 |
.markdown-text {
|
| 3 |
font-size: 16px !important;
|
|
|
|
|
|
|
|
|
|
| 4 |
}
|
|
|
|
|
|
|
| 5 |
.banner_image { width: 75% !important; align-self: center !important; }
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
button[role="tab"] {
|
| 8 |
-
font-family: 'Exo 2', system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important;
|
| 9 |
letter-spacing: 0.25px;
|
| 10 |
font-weight: 600;
|
| 11 |
font-size: 18px !important;
|
| 12 |
}
|
|
|
|
|
|
|
| 13 |
#models-to-add-text { font-size: 18px !important; }
|
| 14 |
#citation-button span { font-size: 16px !important; }
|
| 15 |
#citation-button textarea { font-size: 16px !important; }
|
| 16 |
#citation-button > label > button { margin: 6px; transform: scale(1.3); }
|
|
|
|
|
|
|
| 17 |
#leaderboard-table { margin-top: 15px }
|
| 18 |
#leaderboard-table-lite { margin-top: 15px }
|
| 19 |
#search-bar-table-box > div:first-child { background: none; border: none; }
|
| 20 |
#search-bar { padding: 0px; }
|
| 21 |
#leaderboard-table td:nth-child(2),
|
| 22 |
#leaderboard-table th:nth-child(2) { max-width: 400px; overflow: auto; white-space: nowrap; }
|
| 23 |
-
|
| 24 |
#scale-logo { border-style: none !important; box-shadow: none; display: block; margin-left: auto; margin-right: auto; max-width: 600px; }
|
| 25 |
#scale-logo .download { display: none; }
|
| 26 |
#filter_type{ border: 0; padding-left: 0; padding-top: 0; }
|
|
@@ -33,38 +48,75 @@ button[role="tab"] {
|
|
| 33 |
#filter-columns-size{ border:0; padding:0.5; }
|
| 34 |
#box-filter > .form{ border: 0 }
|
| 35 |
|
| 36 |
-
/*
|
| 37 |
-
|
| 38 |
-
.f1-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
.f1-p, .f1-li { line-height: 1.75; color: #374151; }
|
| 40 |
.f1-a { color: #2563eb; text-decoration: none; font-weight: 500; }
|
| 41 |
.f1-a:hover { text-decoration: underline; }
|
| 42 |
-
.f1-blockquote { border-left: 4px solid #d1d5db; padding-left: 1rem; margin-left: 0; font-style: italic; color: #4b5563; }
|
| 43 |
-
.f1-problem-box { background-color: #f9fafb; border: 1px solid #e5e7eb; border-radius: 0.
|
| 44 |
-
.f1-
|
| 45 |
-
.f1-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
.f1-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
|
| 59 |
-
/*
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
background: #ffffff !important;
|
| 62 |
color: #374151 !important;
|
| 63 |
border: 1px solid #d1d5db !important;
|
| 64 |
}
|
| 65 |
-
div[data-testid="login-button"] > button:hover
|
| 66 |
-
|
| 67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
"""
|
| 69 |
|
| 70 |
get_window_url_params = """
|
|
|
|
| 1 |
custom_css = """
|
| 2 |
+
/* Typography + readable width */
|
| 3 |
.markdown-text {
|
| 4 |
font-size: 16px !important;
|
| 5 |
+
line-height: 1.75 !important;
|
| 6 |
+
max-width: 900px; /* constrain paragraphs */
|
| 7 |
+
margin: 0 auto; /* center */
|
| 8 |
}
|
| 9 |
+
|
| 10 |
+
/* Banner/image helpers */
|
| 11 |
.banner_image { width: 75% !important; align-self: center !important; }
|
| 12 |
+
|
| 13 |
+
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Exo+2:wght@500;600&display=swap');
|
| 14 |
+
|
| 15 |
+
:root { font-family: Inter, system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; }
|
| 16 |
+
|
| 17 |
+
/* Tabs (top) */
|
| 18 |
button[role="tab"] {
|
| 19 |
+
font-family: 'Exo 2', Inter, system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important;
|
| 20 |
letter-spacing: 0.25px;
|
| 21 |
font-weight: 600;
|
| 22 |
font-size: 18px !important;
|
| 23 |
}
|
| 24 |
+
.tab-buttons button { font-size: 20px; }
|
| 25 |
+
|
| 26 |
#models-to-add-text { font-size: 18px !important; }
|
| 27 |
#citation-button span { font-size: 16px !important; }
|
| 28 |
#citation-button textarea { font-size: 16px !important; }
|
| 29 |
#citation-button > label > button { margin: 6px; transform: scale(1.3); }
|
| 30 |
+
|
| 31 |
+
/* Leaderboard spacing */
|
| 32 |
#leaderboard-table { margin-top: 15px }
|
| 33 |
#leaderboard-table-lite { margin-top: 15px }
|
| 34 |
#search-bar-table-box > div:first-child { background: none; border: none; }
|
| 35 |
#search-bar { padding: 0px; }
|
| 36 |
#leaderboard-table td:nth-child(2),
|
| 37 |
#leaderboard-table th:nth-child(2) { max-width: 400px; overflow: auto; white-space: nowrap; }
|
| 38 |
+
|
| 39 |
#scale-logo { border-style: none !important; box-shadow: none; display: block; margin-left: auto; margin-right: auto; max-width: 600px; }
|
| 40 |
#scale-logo .download { display: none; }
|
| 41 |
#filter_type{ border: 0; padding-left: 0; padding-top: 0; }
|
|
|
|
| 48 |
#filter-columns-size{ border:0; padding:0.5; }
|
| 49 |
#box-filter > .form{ border: 0 }
|
| 50 |
|
| 51 |
+
/* ---------- "What is FormulaOne" (HTML) ---------- */
|
| 52 |
+
/* Constrain width of the whole page section */
|
| 53 |
+
.f1-container { max-width: 900px; margin: 0 auto; padding: 0 1rem; }
|
| 54 |
+
|
| 55 |
+
/* Heading spacing fix */
|
| 56 |
+
.f1-h1 { font-weight: 700; font-size: 2.5rem; line-height: 1.1; color: #111827; text-align: center; margin-bottom: 1.25rem !important; }
|
| 57 |
+
|
| 58 |
+
.f1-h2 { font-weight: 700; border-bottom: 1px solid #e5e7eb; padding-bottom: 0.5rem; margin-top: 2.25rem; margin-bottom: 1rem; color: #111827; font-size: 1.75rem; line-height: 2.1rem; }
|
| 59 |
.f1-p, .f1-li { line-height: 1.75; color: #374151; }
|
| 60 |
.f1-a { color: #2563eb; text-decoration: none; font-weight: 500; }
|
| 61 |
.f1-a:hover { text-decoration: underline; }
|
| 62 |
+
.f1-blockquote { border-left: 4px solid #d1d5db; padding-left: 1rem; margin-left: 0; font-style: italic; color: #4b5563; background: #fafafa; }
|
| 63 |
+
.f1-problem-box { background-color: #f9fafb; border: 1px solid #e5e7eb; border-radius: 0.75rem; padding: 1.25rem 1.25rem; margin-top: 1.25rem; margin-bottom: 1.25rem; box-shadow: 0 1px 2px 0 rgb(0 0 0 / 0.05); }
|
| 64 |
+
.f1-figure { margin-top: 1.25rem; margin-bottom: 1.25rem; text-align: center; }
|
| 65 |
+
.f1-figcaption { margin-top: 0.5rem; font-size: 0.9rem; color: #6b7280; font-style: italic; }
|
| 66 |
+
|
| 67 |
+
/* Beautiful table for Warmup / Tier 1 / Tier 2 */
|
| 68 |
+
.f1-table-wrapper { margin: 1.5rem 0; overflow-x: auto; }
|
| 69 |
+
.f1-table { width: 100%; border-collapse: separate; border-spacing: 0; background: #fff; border: 1px solid #e5e7eb; border-radius: 0.75rem; box-shadow: 0 1px 2px 0 rgba(0,0,0,0.04); }
|
| 70 |
+
.f1-table thead th { background: #f3f4f6; color: #111827; font-weight: 700; text-align: left; padding: 0.85rem 1rem; border-bottom: 1px solid #e5e7eb; }
|
| 71 |
+
.f1-table tbody td { padding: 0.75rem 1rem; color: #374151; border-bottom: 1px solid #f1f5f9; }
|
| 72 |
+
.f1-table tbody tr:nth-child(odd) td { background: #fafafa; }
|
| 73 |
+
.f1-table tbody tr:last-child td { border-bottom: none; }
|
| 74 |
+
.f1-table th:first-child, .f1-table td:first-child { border-top-left-radius: 0.75rem; }
|
| 75 |
+
.f1-table th:last-child, .f1-table td:last-child { border-top-right-radius: 0.75rem; }
|
| 76 |
+
|
| 77 |
+
/* Bottom tabs: "Examples of FormulaOne problems" */
|
| 78 |
+
.f1-tabs { border: 1px solid #e5e7eb; border-radius: 0.75rem; background: #fff; box-shadow: 0 1px 2px 0 rgba(0,0,0,0.04); margin-top: 1.25rem; }
|
| 79 |
+
.f1-tabs .f1-tabs-header { padding: 1rem 1rem 0 1rem; font-weight: 700; color: #111827; font-size: 1.1rem; }
|
| 80 |
+
.f1-tabs .f1-tabs-content { padding: 1rem; }
|
| 81 |
+
.f1-tabs .f1-tabs-bar { display: flex; gap: 0.5rem; justify-content: center; border-top: 1px solid #e5e7eb; padding: 0.6rem; background: #fafafa; border-bottom-left-radius: 0.75rem; border-bottom-right-radius: 0.75rem; }
|
| 82 |
+
.f1-tabs .f1-tab-btn { appearance: none; border: 1px solid #d1d5db; background: #ffffff; padding: 0.45rem 0.9rem; border-radius: 999px; font-weight: 600; font-size: 0.95rem; color: #374151; cursor: pointer; transition: all 120ms ease; }
|
| 83 |
+
.f1-tabs .f1-tab-btn:hover { transform: translateY(-1px); box-shadow: 0 1px 2px rgba(0,0,0,0.06); }
|
| 84 |
+
.f1-tabs .f1-tab-btn.is-active { background: #2563eb; color: white; border-color: #2563eb; box-shadow: 0 1px 2px rgba(37,99,235,.35); }
|
| 85 |
+
|
| 86 |
+
/* Code blocks in citation look cleaner */
|
| 87 |
+
#citation-block { border-radius: 0.5rem; }
|
| 88 |
|
| 89 |
+
/* Nice cards used elsewhere */
|
| 90 |
+
.f1-category-card { display: block; border-radius: 0.5rem; border: 1px solid #e5e7eb; padding: 1rem; background-color: white; box-shadow: 0 1px 2px 0 rgb(0 0 0 / 0.05); }
|
| 91 |
+
|
| 92 |
+
/* Login Button — force light */
|
| 93 |
+
div[data-testid="login-button"] > button,
|
| 94 |
+
button.sso-button,
|
| 95 |
+
[data-testid="login-button"] button,
|
| 96 |
+
button[data-testid="oauth-login"],
|
| 97 |
+
.gr-login button {
|
| 98 |
background: #ffffff !important;
|
| 99 |
color: #374151 !important;
|
| 100 |
border: 1px solid #d1d5db !important;
|
| 101 |
}
|
| 102 |
+
div[data-testid="login-button"] > button:hover,
|
| 103 |
+
button.sso-button:hover,
|
| 104 |
+
[data-testid="login-button"] button:hover,
|
| 105 |
+
button[data-testid="oauth-login"]:hover,
|
| 106 |
+
.gr-login button:hover { background: #f9fafb !important; }
|
| 107 |
+
|
| 108 |
+
/* Leaderboard controls should stretch full width */
|
| 109 |
+
#formulaone-leaderboard-tab-table .gr-row,
|
| 110 |
+
#formulaone-leaderboard-tab-table .gr-column { width: 100% !important; }
|
| 111 |
+
#formulaone-leaderboard-tab-table input[type="text"],
|
| 112 |
+
#formulaone-leaderboard-tab-table select,
|
| 113 |
+
#formulaone-leaderboard-tab-table .wrap,
|
| 114 |
+
#formulaone-leaderboard-tab-table .wrap-inner,
|
| 115 |
+
#formulaone-leaderboard-tab-table .container { width: 100% !important; max-width: 100% !important; }
|
| 116 |
+
|
| 117 |
+
/* Light theme enforcement if outer shell is dark */
|
| 118 |
+
html.dark, body.dark { background: #ffffff !important; color-scheme: light; }
|
| 119 |
+
html.dark *, body.dark * { --tw-ring-color: rgba(37, 99, 235, 0.4); }
|
| 120 |
"""
|
| 121 |
|
| 122 |
get_window_url_params = """
|