add intel CPU to leaderboard (#32)

- add intel to leaderboard (0471f33e960a94686e670c0c70fb4c2cd8c5ae42)
- add intel to leaderboard (591a3e40a5c2aa1bcc27c2e9464dbf45366f6c70)
- intel results accesible in the leaderboard (003f467675456e4814fd68f1fd6fcb4b875b967d)
- add intel results to leaderboard (9f82a2bedf611b6097a56acf8504e93c5ae7a1e5)
- add intel results to leaderboard (d2401bdfdf5d857e255a1877f54b9fd846ad1b11)
- add intel results to leaderboard (39105fc16c53a5878e618dd07993eaba296d2696)
- fix hardware name (d7880b24c12052284d96ea76ef7a07039401d569)
- add documentation about the intel hardware (504caea55fac11d7f1170bf6d26f2b7b153e609b)
- add documentation about the intel hardware (4aa590a679e0c52ee0f3a4f187792a366d5b1299)

.gitignore

@@ -4,5 +4,7 @@ __pycache__/
 *ipynb
 .vscode/
 
+work-in-progress/
+
 dataset/
 .venv

app.py

@@ -4,6 +4,7 @@ from src.assets import custom_css
 
 # from src.attention import create_attn_plots
 from src.content import ABOUT, CITATION_BUTTON, CITATION_BUTTON_LABEL, LOGO, TITLE
+from src.hardware import load_hardware_configs
 from src.leaderboard import create_leaderboard_table
 from src.llm_perf import get_llm_perf_df
 from src.map import create_lat_score_mem_plot
@@ -13,14 +14,7 @@ from src.panel import (
     create_select_callback,
 )
 
-# from custom_kernels import create_quant_krnl_plots
-
-MACHINE_TO_HARDWARE = {
-    "1xA10": "A10-24GB-150W 🖥️",
-    "1xA100": "A100-80GB-275W 🖥️",
-    "1xT4": "T4-16GB-70W 🖥️",
-    # "1xH100": "H100-80GB-700W 🖥️",
-}
+configs = load_hardware_configs("hardware.yml")
 
 
 demo = gr.Blocks(css=custom_css)
@@ -29,12 +23,19 @@ with demo:
     gr.HTML(TITLE, elem_classes="title")
     ####################### HARDWARE TABS #######################
     with gr.Tabs(elem_classes="tabs"):
-        for id, (machine, hardware) in enumerate(MACHINE_TO_HARDWARE.items()):
-            with gr.TabItem(hardware, id=id):
-                ####################### CONTROL PANEL #######################
+        for id, config in enumerate(configs):
+            with gr.TabItem(config.description, id=id):
+                ####################### HARDWARE DETAILS #######################
+                if config.detail:
+                    gr.Markdown(config.detail, elem_classes="descriptive-text")
+
+
+
+                # ####################### CONTROL PANEL #######################
                 (
                     filter_button,
                     machine_textbox,
+                    subsets_values,
                     score_slider,
                     memory_slider,
                     backend_checkboxes,
@@ -42,17 +43,18 @@ with demo:
                     optimization_checkboxes,
                     quantization_checkboxes,
                     kernels_checkboxes,
-                ) = create_control_panel(machine=machine)
+                ) = create_control_panel(machine=config.machine, subsets=config.subsets, hardware_provider=config.hardware_provider)
                 ####################### HARDWARE SUBTABS #######################
                 with gr.Tabs(elem_classes="subtabs"):
-                    open_llm_perf_df = get_llm_perf_df(machine=machine)
+                    open_llm_perf_df = get_llm_perf_df(machine=config.machine, subsets=config.subsets)
                     ####################### LEADERBOARD TAB #######################
                     with gr.TabItem("Leaderboard 🏅", id=0):
                         search_bar, columns_checkboxes, leaderboard_table = (
                             create_leaderboard_table(open_llm_perf_df)
                         )
-                    with gr.TabItem("Find Your Best Model 🧭", id=1):
-                        lat_score_mem_plot = create_lat_score_mem_plot(open_llm_perf_df)
+                    if config.hardware_provider != "intel": # TODO intel CPU does not measure the memory requirements correctly, so disable the graph feature until we fix the underlying issue
+                        with gr.TabItem("Find Your Best Model 🧭", id=1):
+                            lat_score_mem_plot = create_lat_score_mem_plot(open_llm_perf_df)
                     ###################### ATTENTIONS SPEEDUP TAB #######################
                     # with gr.TabItem("Attention 📈", id=2):
                     #     attn_prefill_plot, attn_decode_plot = create_attn_plots(
@@ -69,6 +71,7 @@ with demo:
                     filter_button,
                     # inputs
                     machine_textbox,
+                    subsets_values,
                     score_slider,
                     memory_slider,
                     backend_checkboxes,
@@ -91,6 +94,7 @@ with demo:
                 create_select_callback(
                     # inputs
                     machine_textbox,
+                    subsets_values,
                     # interactive
                     columns_checkboxes,
                     search_bar,
@@ -99,7 +103,7 @@ with demo:
                 )
 
         ####################### ABOUT TAB #######################
-        with gr.TabItem("About 📖", id=3):
+        with gr.TabItem("About 📖", id=len(configs)):
             gr.Markdown(ABOUT, elem_classes="descriptive-text")
     ####################### CITATION
     with gr.Row():

hardware.yml

@@ -0,0 +1,46 @@
+- machine: 1xA10
+  description: A10-24GB-150W 🖥️
+  hardware_provider: nvidia
+  hardware_type: gpu
+  subsets:
+    - unquantized
+    - awq
+    - bnb
+    - gptq
+  backends:
+    - pytorch
+
+- machine: 1xA100
+  description: A100-80GB-275W 🖥️
+  hardware_provider: nvidia
+  hardware_type: gpu
+  subsets:
+    - unquantized
+    - awq
+    - bnb
+    - gptq
+  backends:
+    - pytorch
+
+- machine: 1xT4
+  description: T4-16GB-70W 🖥️
+  hardware_provider: nvidia
+  hardware_type: gpu
+  subsets:
+    - unquantized
+    - awq
+    - bnb
+    - gptq
+  backends:
+    - pytorch
+
+- machine: 32vCPU-C7i
+  description: Intel-Xeon-SPR-385W 🖥️
+  detail: |
+    We tested the [32vCPU AWS C7i](https://aws.amazon.com/ec2/instance-types/c7i/) instance for the benchmark.
+  hardware_provider: intel
+  hardware_type: cpu
+  subsets:
+    - unquantized
+  backends:
+    - pytorch

src/hardware.py

@@ -0,0 +1,26 @@
+from typing import Any, Dict, List
+
+import yaml
+
+
+class HardwareConfig:
+    def __init__(self, data: Dict[str, Any]):
+        self.machine = data["machine"]
+        self.description = data["description"]
+        self.hardware_provider = data["hardware_provider"]
+        self.hardware_type = data["hardware_type"]
+        self.subsets = data["subsets"]
+        self.backends = data["backends"]
+        self.detail = data.get("detail", None)
+
+    def __repr__(self):
+        return (
+            f"HardwareConfig(machine='{self.machine}', description='{self.description}', "
+            f"hardware_provider={self.hardware_provider}, hardware_type={self.hardware_type}, subsets={self.subsets}, backends={self.backends})"
+        )
+
+
+def load_hardware_configs(file_path: str) -> List[HardwareConfig]:
+    with open(file_path, "r") as file:
+        data = yaml.safe_load(file)
+    return [HardwareConfig(config) for config in data]

src/llm_perf.py

@@ -1,7 +1,10 @@
 import os
+from typing import List
 
 import pandas as pd
 
+from src.hardware import HardwareConfig
+
 from .utils import process_kernels, process_quantizations
 
 DATASET_DIRECTORY = "dataset"
@@ -28,13 +31,12 @@ COLUMNS_MAPPING = {
     "#Params (B)": "Params (B)",
 }
 SORTING_COLUMNS = ["Open LLM Score (%)", "Decode (tokens/s)", "Prefill (s)"]
-SUBSETS = ["unquantized", "awq", "bnb", "gptq"]
 SORTING_ASCENDING = [False, True, False]
 
 
-def get_raw_llm_perf_df(machine: str = "1xA10"):
+def get_raw_llm_perf_df(machine: str, subsets: List[str]):
     dfs = []
-    for subset in SUBSETS:
+    for subset in subsets:
         try:
             dfs.append(
                 pd.read_csv(
@@ -110,14 +112,14 @@ def processed_llm_perf_df(llm_perf_df):
     return llm_perf_df
 
 
-def get_llm_perf_df(machine: str = "1xA10"):
+def get_llm_perf_df(machine: str, subsets: List[str]):
     if not os.path.exists(DATASET_DIRECTORY):
         os.makedirs(DATASET_DIRECTORY)
 
     if os.path.exists(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv"):
         llm_perf_df = pd.read_csv(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv")
     else:
-        llm_perf_df = get_raw_llm_perf_df(machine)
+        llm_perf_df = get_raw_llm_perf_df(machine, subsets)
         llm_perf_df = processed_llm_perf_df(llm_perf_df)
         llm_perf_df.to_csv(f"{DATASET_DIRECTORY}/llm-perf-leaderboard-{machine}.csv", index=False)
 

src/panel.py

@@ -1,3 +1,5 @@
+from typing import List
+
 import gradio as gr
 
 from src.leaderboard import get_leaderboard_df
@@ -8,9 +10,26 @@ from src.llm_perf import get_llm_perf_df
 from src.map import get_lat_score_mem_fig
 
 
-def create_control_panel(machine: str):
+def create_control_panel(machine: str, subsets: List[str], hardware_provider: str):
     # controls
     machine_textbox = gr.Textbox(value=machine, visible=False)
+    subsets_values = gr.State(value=subsets)
+
+
+    if hardware_provider == "nvidia":
+        backends = ["pytorch"]
+        attention_implementations = ["Eager", "SDPA", "FAv2"]
+        quantizations = ["Unquantized", "BnB.4bit", "BnB.8bit", "AWQ.4bit", "GPTQ.4bit"]
+        kernels = ["No Kernel", "GPTQ.ExllamaV1", "GPTQ.ExllamaV2", "AWQ.GEMM", "AWQ.GEMV"]
+    elif hardware_provider == "intel":
+        backends = ["pytorch", "onnxruntime", "openvino"]
+        attention_implementations = ["Eager"]
+        quantizations = ["Unquantized"]
+        kernels = ["No Kernel"]
+    else:
+        raise ValueError(f"Unknown hardware provider: {hardware_provider}")
+
+
     with gr.Accordion("Control Panel 🎛️", open=False, elem_id="control-panel"):
         with gr.Row():
             with gr.Column(scale=2, variant="panel"):
@@ -32,8 +51,8 @@ def create_control_panel(machine: str):
             with gr.Column(scale=1, variant="panel"):
                 backend_checkboxes = gr.CheckboxGroup(
                     label="Backends 🏭",
-                    choices=["pytorch"],
-                    value=["pytorch"],
+                    choices=backends,
+                    value=backends,
                     info="☑️ Select the backends",
                     elem_id="backend-checkboxes",
                 )
@@ -49,8 +68,8 @@ def create_control_panel(machine: str):
             with gr.Column(scale=1, variant="panel"):
                 optimization_checkboxes = gr.CheckboxGroup(
                     label="Attentions 👁️",
-                    choices=["Eager", "SDPA", "FAv2"],
-                    value=["Eager", "SDPA", "FAv2"],
+                    choices=attention_implementations,
+                    value=attention_implementations,
                     info="☑️ Select the optimization",
                     elem_id="optimization-checkboxes",
                 )
@@ -58,20 +77,8 @@ def create_control_panel(machine: str):
             with gr.Column(scale=1, variant="panel"):
                 quantization_checkboxes = gr.CheckboxGroup(
                     label="Quantizations 🗜️",
-                    choices=[
-                        "Unquantized",
-                        "BnB.4bit",
-                        "BnB.8bit",
-                        "AWQ.4bit",
-                        "GPTQ.4bit",
-                    ],
-                    value=[
-                        "Unquantized",
-                        "BnB.4bit",
-                        "BnB.8bit",
-                        "AWQ.4bit",
-                        "GPTQ.4bit",
-                    ],
+                    choices=quantizations,
+                    value=quantizations,
                     info="☑️ Select the quantization schemes",
                     elem_id="quantization-checkboxes",
                     elem_classes="boxed-option",
@@ -79,20 +86,8 @@ def create_control_panel(machine: str):
             with gr.Column(scale=1, variant="panel"):
                 kernels_checkboxes = gr.CheckboxGroup(
                     label="Kernels ⚛️",
-                    choices=[
-                        "No Kernel",
-                        "GPTQ.ExllamaV1",
-                        "GPTQ.ExllamaV2",
-                        "AWQ.GEMM",
-                        "AWQ.GEMV",
-                    ],
-                    value=[
-                        "No Kernel",
-                        "GPTQ.ExllamaV1",
-                        "GPTQ.ExllamaV2",
-                        "AWQ.GEMM",
-                        "AWQ.GEMV",
-                    ],
+                    choices=kernels,
+                    value=kernels,
                     info="☑️ Select the custom kernels",
                     elem_id="kernel-checkboxes",
                     elem_classes="boxed-option",
@@ -107,6 +102,7 @@ def create_control_panel(machine: str):
     return (
         filter_button,
         machine_textbox,
+        subsets_values,
         score_slider,
         memory_slider,
         backend_checkboxes,
@@ -119,6 +115,7 @@ def create_control_panel(machine: str):
 
 def filter_rows_fn(
     machine,
+    subsets,
     # inputs
     score,
     memory,
@@ -131,7 +128,7 @@ def filter_rows_fn(
     columns,
     search,
 ):
-    llm_perf_df = get_llm_perf_df(machine=machine)
+    llm_perf_df = get_llm_perf_df(machine=machine, subsets=subsets)
     # print(attentions)
     # print(llm_perf_df["Attention 👁️"].unique())
     filtered_llm_perf_df = llm_perf_df[
@@ -145,7 +142,7 @@ def filter_rows_fn(
         & (llm_perf_df["Memory (MB)"] <= memory)
     ]
     selected_filtered_llm_perf_df = select_columns_fn(
-        machine, columns, search, filtered_llm_perf_df
+        machine, subsets, columns, search, filtered_llm_perf_df
     )
     selected_filtered_lat_score_mem_fig = get_lat_score_mem_fig(filtered_llm_perf_df)
     # filtered_bt_prefill_fig = get_bt_prefill_fig(filtered_df)
@@ -172,6 +169,7 @@ def create_control_callback(
     filter_button,
     # fixed
     machine_textbox,
+    subsets_textbox,
     # inputs
     score_slider,
     memory_slider,
@@ -198,6 +196,7 @@ def create_control_callback(
         inputs=[
             # fixed
             machine_textbox,
+            subsets_textbox,
             # inputs
             score_slider,
             memory_slider,
@@ -223,9 +222,9 @@ def create_control_callback(
     )
 
 
-def select_columns_fn(machine, columns, search, llm_perf_df=None):
+def select_columns_fn(machine, subsets, columns, search, llm_perf_df=None):
     if llm_perf_df is None:
-        llm_perf_df = get_llm_perf_df(machine=machine)
+        llm_perf_df = get_llm_perf_df(machine=machine, subsets=subsets)
 
     selected_leaderboard_df = get_leaderboard_df(llm_perf_df)
     selected_leaderboard_df = selected_leaderboard_df[
@@ -239,6 +238,7 @@ def select_columns_fn(machine, columns, search, llm_perf_df=None):
 def create_select_callback(
     # fixed
     machine_textbox,
+    subsets_values,
     # interactive
     columns_checkboxes,
     search_bar,
@@ -247,11 +247,11 @@ def create_select_callback(
 ):
     columns_checkboxes.change(
         fn=select_columns_fn,
-        inputs=[machine_textbox, columns_checkboxes, search_bar],
+        inputs=[machine_textbox, subsets_values, columns_checkboxes, search_bar],
         outputs=[leaderboard_table],
     )
     search_bar.change(
         fn=select_columns_fn,
-        inputs=[machine_textbox, columns_checkboxes, search_bar],
+        inputs=[machine_textbox, subsets_values, columns_checkboxes, search_bar],
         outputs=[leaderboard_table],
     )