Merge branch 'main' into pr/15

app.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 import os
 import datetime
 import socket

backend-cli.py

@@ -6,6 +6,7 @@ import argparse
 
 import socket
 import random
+import threading
 from datetime import datetime
 
 from src.backend.run_eval_suite import run_evaluation
@@ -16,7 +17,7 @@ from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
 from src.envs import QUEUE_REPO, RESULTS_REPO, API
-from src.utils import my_snapshot_download
+from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus
 
 from src.leaderboard.read_evals import get_raw_eval_results
 
@@ -123,7 +124,17 @@ def request_to_result_name(request: EvalRequest) -> str:
 
 
 def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[int] = None) -> dict:
-    batch_size = 4
+    batch_size = 1
+    batch_size = eval_request.batch_size
+
+    init_gpu_info = analyze_gpu_stats(parse_nvidia_smi())
+    # if init_gpu_info['Mem(M)'] > 500:
+    #     assert False, f"This machine is not empty: {init_gpu_info}"
+    gpu_stats_list = []
+    stop_event = threading.Event()
+    monitor_thread = threading.Thread(target=monitor_gpus, args=(stop_event, 5, gpu_stats_list))
+    monitor_thread.start()
+
     try:
         results = run_evaluation(
             eval_request=eval_request,
@@ -150,6 +161,20 @@ def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[in
             raise
 
     # print("RESULTS", results)
+    stop_event.set()
+    monitor_thread.join()
+    gpu_info = analyze_gpu_stats(gpu_stats_list)
+    for task_name in results['results'].keys():
+        for key, value in gpu_info.items():
+            if "GPU" not in key:
+                results['results'][task_name][f"{key},none"] = int(value)
+            else:
+                results['results'][task_name][f"{key},none"] = value
+
+        results['results'][task_name]['batch_size,none'] = batch_size
+        results['results'][task_name]['precision,none'] = eval_request.precision
+    print(f"gpu_stats_list: {gpu_stats_list}")
+    print("GPU Usage:", gpu_info)
 
     dumped = json.dumps(results, indent=2, default=lambda o: "<not serializable>")
     # print(dumped)
@@ -396,9 +421,9 @@ def get_args():
     parser = argparse.ArgumentParser(description="Run the backend")
     parser.add_argument("--debug", action="store_true", help="Run in debug mode")
     # debug parameters
-    parser.add_argument("--task", type=str, default="selfcheckgpt", help="Task to debug")
-    parser.add_argument("--model", type=str, default="facebook/opt-1.3b", help="Model to debug")
-    parser.add_argument("--precision", type=str, default="float16", help="Precision to debug")
+    parser.add_argument("--task", type=str, default="selfcheckgpt,mmlu", help="Task to debug")
+    parser.add_argument("--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1,mistralai/Mixtral-8x7B-v0.1", help="Model to debug")
+    parser.add_argument("--precision", type=str, default="float32,float16,8bit,4bit", help="Precision to debug")
     parser.add_argument("--inference-framework", type=str, default="hf-chat", help="Inference framework to debug")
     parser.add_argument("--limit", type=int, default=None, help="Limit for the number of samples")
     return parser.parse_args()
@@ -409,23 +434,31 @@ if __name__ == "__main__":
     local_debug = args.debug
     # debug specific task by ping
     if local_debug:
-        debug_model_names = [args.model]  # Use model from arguments
-        debug_task_name = args.task  # Use task from arguments
+        # debug_model_names = [args.model]  # Use model from arguments
+        # debug_task_name = [args.task]  # Use task from arguments
+        debug_model_names = args.model.split(",")
+        debug_task_name = args.task.split(",")
+        precisions = args.precision.split(",")
+        print(f"debug_model_names: {debug_model_names}, debug_task_name: {debug_task_name}, precisions: {precisions}")
         task_lst = TASKS_HARNESS.copy()
-        for task in task_lst:
+        for precision in precisions:
             for debug_model_name in debug_model_names:
-                task_name = task.benchmark
-                if task_name != debug_task_name:
-                    continue
-                eval_request = EvalRequest(
-                    model=debug_model_name, 
-                    private=False, 
-                    status="", 
-                    json_filepath="", 
-                    precision=args.precision,  # Use precision from arguments
-                    inference_framework=args.inference_framework  # Use inference framework from arguments
-                )
-                results = process_evaluation(task, eval_request, limit=args.limit)
+                for task in task_lst:
+                    task_name = task.benchmark
+                    if task_name not in debug_task_name:
+                        continue
+                    try:
+                        eval_request = EvalRequest(
+                            model=debug_model_name,
+                            private=False,
+                            status="",
+                            json_filepath="",
+                            precision=precision,  # Use precision from arguments
+                            inference_framework=args.inference_framework  # Use inference framework from arguments
+                        )
+                        results = process_evaluation(task, eval_request, limit=args.limit)
+                    except Exception as e:
+                        print(f"debug running error: {e}")
     else:
         while True:
             res = False

src/backend/envs.py

@@ -63,4 +63,4 @@ EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_REQUESTS_PATH_BACKEND_SYNC = os.path.join(CACHE_PATH, "eval-queue-bk-sync")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu"

src/backend/hflm_with_measurement.py

@@ -57,12 +57,12 @@ class StopWatch(TextStreamer):
             self.start_decoding = time()
         self.decoding_iterations += 1
         return
-    
+
     def end(self):
         if self.decoding_time is None and self.start_decoding is not None:
             self.decoding_time = time() - self.start_decoding
         return
-    
+
 
 class HFLMWithMeasurement(HFLM):
     def __init__(self, **kwargs):
@@ -287,7 +287,7 @@ class HFLMWithMeasurement(HFLM):
         pbar.close()
 
         return re_ord.get_original(res)
-    
+
     def _model_generate(self, context, max_length, stop, **generation_kwargs):
         # temperature = 0.0 if not set
         # if do_sample is false and temp==0.0:
@@ -318,7 +318,7 @@ class HFLMWithMeasurement(HFLM):
             **generation_kwargs,
         )
         end = time()
-        
+
         batch_size = context.shape[0]
         output_length = stop_watch.decoding_iterations
 
@@ -403,7 +403,7 @@ class HFLMWithMeasurement(HFLM):
                     f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}"
                 )
             # add EOS token to stop sequences
-            eos = self.tok_decode(self.eot_token_id, skip_special_tokens=False)
+            eos = self.tok_decode(self.eot_token_id)
             if not until:
                 until = [eos]
             else:

src/backend/manage_requests.py

@@ -27,24 +27,23 @@ class EvalRequest:
     likes: Optional[int] = 0
     params: Optional[int] = None
     license: Optional[str] = ""
+    batch_size: Optional[int] = 1
 
     def get_model_args(self) -> str:
         model_args = f"pretrained={self.model},revision={self.revision},parallelize=True"  # ,max_length=4096"
-
+        model_args += ",trust_remote_code=True,device_map=auto"
         if self.precision in ["float16", "float32", "bfloat16"]:
             model_args += f",dtype={self.precision}"
             # Quantized models need some added config, the install of bits and bytes, etc
             # elif self.precision == "8bit":
             #    model_args += ",load_in_8bit=True"
-            # elif self.precision == "4bit":
-            #    model_args += ",load_in_4bit=True"
+        elif self.precision == "4bit":
+           model_args += ",load_in_4bit=True"
             # elif self.precision == "GPTQ":
             # A GPTQ model does not need dtype to be specified,
             # it will be inferred from the config
-            pass
         elif self.precision == "8bit":
             model_args += ",load_in_8bit=True"
-            model_args += ",trust_remote_code=True"
         else:
             raise Exception(f"Unknown precision {self.precision}.")
         return model_args

src/backend/run_eval_suite.py

@@ -13,7 +13,7 @@ orig_higher_is_better = ConfigurableTask.higher_is_better
 def process_results_decorator(func):
     def wrapper(self, doc, results, *args, **kwargs):
         processed_results = [r[0] for r in results]
-        
+
         end_to_end_time = sum([r[1] for r in results]) / len(results)
         prefilling_time = sum([r[2] for r in results]) / len(results)
         decoding_throughput = sum([r[3] for r in results]) / len(results)

src/display/utils.py

@@ -13,6 +13,29 @@ TS = "T/s" #Decoding throughput (tok/s)
 InFrame = "Method" #"Inference framework"
 MULTIPLE_CHOICEs = ["mmlu"]
 
+GPU_TEMP = 'Temp(C)'
+GPU_Power = 'Power(W)'
+GPU_Mem = 'Mem(G)'
+GPU_Name = "GPU"
+GPU_Util = 'Util(%)'
+BATCH_SIZE = 'bs'
+PRECISION = "Precision"
+system_metrics_to_name_map = {
+    "end_to_end_time": f"{E2Es}",
+    "prefilling_time": f"{PREs}",
+    "decoding_throughput": f"{TS}",
+}
+
+gpu_metrics_to_name_map = {
+    GPU_Util: GPU_Util,
+    GPU_TEMP: GPU_TEMP,
+    GPU_Power: GPU_Power,
+    GPU_Mem: GPU_Mem,
+    "batch_size": BATCH_SIZE,
+    "precision": PRECISION,
+    GPU_Name: GPU_Name,
+}
+
 @dataclass
 class Task:
     benchmark: str
@@ -81,11 +104,17 @@ auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnConten
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
     # System performance metrics
-    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name}-{E2Es}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_batch_size", ColumnContent, ColumnContent(f"{task.value.col_name} {BATCH_SIZE}", "number", True)])
+    # auto_eval_column_dict.append([f"{task.name}_precision", ColumnContent, ColumnContent(f"{task.value.col_name} {PRECISION}", "str", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Name}", "str", True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True)])
     if task.value.benchmark in MULTIPLE_CHOICEs:
         continue
-    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name}-{PREs}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name}-{TS}", "number", True)])
+    # auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", False)])
+    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} {TS}", "number", True)])
+
 
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])

src/leaderboard/read_evals.py

@@ -103,6 +103,13 @@ class EvalResult:
 
                 if to_add is True:
                     multiplier = 100.0
+                    if "GPU" in metric:
+                        results[benchmark][metric] = value
+                        continue
+                    if "precision" in metric:
+                        results[benchmark][metric] = value
+                        continue
+
                     if "rouge" in metric and "truthful" not in benchmark:
                         multiplier = 1.0
                     if "squad" in benchmark:
@@ -111,6 +118,10 @@ class EvalResult:
                         multiplier = 1.0
                     if "throughput" in metric:
                         multiplier = 1.0
+                    if "batch_" in metric or "Mem" in metric or "Util" in metric:
+                        multiplier = 1
+
+
                     # print('RESULTS', data['results'])
                     # print('XXX', benchmark, metric, value, multiplier)
                     results[benchmark][metric] = value * multiplier

src/populate.py

@@ -12,7 +12,7 @@ from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, update_
 
 from src.backend.envs import Tasks as BackendTasks
 from src.display.utils import Tasks
-from src.display.utils import E2Es, PREs, TS
+from src.display.utils import system_metrics_to_name_map, gpu_metrics_to_name_map
 
 def get_leaderboard_df(
     results_path: str,
@@ -45,12 +45,7 @@ def get_leaderboard_df(
         bm = (task.benchmark, task.metric)
         name_to_bm_map[name] = bm
 
-    # bm_to_name_map = {bm: name for name, bm in name_to_bm_map.items()}
-    system_metrics_to_name_map = {
-        "end_to_end_time": f"{E2Es}",
-        "prefilling_time": f"{PREs}",
-        "decoding_throughput": f"{TS}",
-    }
+
 
     all_data_json = []
     for entry in all_data_json_:
@@ -63,6 +58,9 @@ def get_leaderboard_df(
                     if sys_metric in entry[k]:
                         new_entry[f"{k} {metric_namne}"] = entry[k][sys_metric]
 
+                for gpu_metric, metric_namne in gpu_metrics_to_name_map.items():
+                    if gpu_metric in entry[k]:
+                        new_entry[f"{k} {metric_namne}"] = entry[k][gpu_metric]
         all_data_json += [new_entry]
 
     # all_data_json.append(baseline_row)

src/utils.py

@@ -1,6 +1,14 @@
 import pandas as pd
 from huggingface_hub import snapshot_download
+import subprocess
+import re
+import os
 
+try:
+    from src.display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util, GPU_Name
+except:
+    print("local debug: from display.utils")
+    from display.utils import GPU_TEMP, GPU_Mem, GPU_Power, GPU_Util, GPU_Name
 
 def my_snapshot_download(repo_id, revision, local_dir, repo_type, max_workers):
     for i in range(10):
@@ -32,3 +40,96 @@ def get_dataset_summary_table(file_path):
     df = df[["Category", "Benchmark", "Data Split", "Data Size", "Language"]]
 
     return df
+
+def parse_nvidia_smi():
+    visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', None)
+    if visible_devices is not None:
+        gpu_indices = visible_devices.split(',')
+    else:
+        # Query all GPU indices if CUDA_VISIBLE_DEVICES is not set
+        result = subprocess.run(['nvidia-smi', '--query-gpu=index', '--format=csv,noheader'], capture_output=True, text=True)
+        if result.returncode != 0:
+            print("Failed to query GPU indices.")
+            return []
+        gpu_indices = result.stdout.strip().split('\n')
+    print(f"gpu_indices: {gpu_indices}")
+    gpu_stats = []
+
+    gpu_info_pattern = re.compile(r'(\d+)C\s+P\d+\s+(\d+)W / \d+W\s+\|\s+(\d+)MiB / \d+MiB\s+\|\s+(\d+)%')
+    gpu_name_pattern = re.compile(r'NVIDIA\s+([\w\s]+?\d+GB)')
+
+    gpu_name = ""
+    for index in gpu_indices:
+        result = subprocess.run(['nvidia-smi', '-i', index], capture_output=True, text=True)
+        output = result.stdout.strip()
+        lines = output.split("\n")
+        for line in lines:
+            match = gpu_info_pattern.search(line)
+            name_match = gpu_name_pattern.search(line)
+            gpu_info = {}
+            if name_match:
+                gpu_name = name_match.group(1).strip()
+            if match:
+                temp, power_usage, mem_usage, gpu_util = map(int, match.groups())
+                gpu_info.update({
+                    GPU_TEMP: temp,
+                    GPU_Power: power_usage,
+                    GPU_Mem: round(mem_usage / 1024, 2),
+                    GPU_Util: gpu_util
+                })
+
+            if len(gpu_info) >= 4:
+                gpu_stats.append(gpu_info)
+    print(f"gpu_stats: {gpu_stats}")
+    gpu_name = f"{len(gpu_stats)}x{gpu_name}"
+    gpu_stats_total = {
+                        GPU_TEMP: 0,
+                        GPU_Power: 0,
+                        GPU_Mem: 0,
+                        GPU_Util: 0,
+                        GPU_Name: gpu_name
+                    }
+    for gpu_stat in gpu_stats:
+        gpu_stats_total[GPU_TEMP] += gpu_stat[GPU_TEMP]
+        gpu_stats_total[GPU_Power] += gpu_stat[GPU_Power]
+        gpu_stats_total[GPU_Mem] += gpu_stat[GPU_Mem]
+        gpu_stats_total[GPU_Util] += gpu_stat[GPU_Util]
+    gpu_stats_total[GPU_Mem] = gpu_stats_total[GPU_Mem] # G
+    gpu_stats_total[GPU_TEMP] /= len(gpu_stats)
+    gpu_stats_total[GPU_Power] /= len(gpu_stats)
+    gpu_stats_total[GPU_Util] /= len(gpu_stats)
+    return [gpu_stats_total]
+
+def monitor_gpus(stop_event, interval, stats_list):
+    while not stop_event.is_set():
+        gpu_stats = parse_nvidia_smi()
+        if gpu_stats:
+            stats_list.extend(gpu_stats)
+        stop_event.wait(interval)
+
+def analyze_gpu_stats(stats_list):
+    # Check if the stats_list is empty, and return None if it is
+    if not stats_list:
+        return None
+
+    # Initialize dictionaries to store the stats
+    avg_stats = {}
+    max_stats = {}
+
+    # Calculate average stats, excluding 'GPU_Mem'
+    for key in stats_list[0].keys():
+        if key != GPU_Mem and key != GPU_Name:
+            total = sum(d[key] for d in stats_list)
+            avg_stats[key] = total / len(stats_list)
+
+    # Calculate max stats for 'GPU_Mem'
+    max_stats[GPU_Mem] = max(d[GPU_Mem] for d in stats_list)
+    if GPU_Name in stats_list[0]:
+        avg_stats[GPU_Name] = stats_list[0][GPU_Name]
+    # Update average stats with max GPU memory usage
+    avg_stats.update(max_stats)
+
+    return avg_stats
+
+if __name__ == "__main__":
+    print(analyze_gpu_stats(parse_nvidia_smi()))