added phi4

.gitignore

@@ -17,6 +17,7 @@ eval-results-bk_hhem21/
 eval-results_hhem21/
 hhem21_server/
 leaderboard_results/
+leaderboard-bk/
 
 src/assets/model_counts.html
 

app.py

@@ -21,6 +21,13 @@ try:
     )
 except Exception:
     restart_space()
+
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = populate.get_evaluation_queue_df(envs.EVAL_REQUESTS_PATH, utils.EVAL_COLS)
+
 try:
     print(envs.EVAL_RESULTS_PATH)
     snapshot_download(
@@ -32,24 +39,15 @@ except Exception:
 raw_data, original_df = populate.get_leaderboard_df(envs.EVAL_RESULTS_PATH, envs.EVAL_REQUESTS_PATH, utils.COLS, utils.BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = populate.get_evaluation_queue_df(envs.EVAL_REQUESTS_PATH, utils.EVAL_COLS)
-
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
     type_query: list,
-    precision_query: str,
-    size_query: list,
-    show_deleted: bool,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    print(f"filter: columns={columns}, type_query={type_query}, query={query}")
+    filtered_df = filter_models(hidden_df, type_query)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
     return df
@@ -83,32 +81,23 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
                     final_df.append(temp_filtered_df)
         if len(final_df) > 0:
             filtered_df = pd.concat(final_df)
-            filtered_df = filtered_df.drop_duplicates(
-                subset=[utils.AutoEvalColumn.model.name, utils.AutoEvalColumn.precision.name, utils.AutoEvalColumn.revision.name]
-            )
+            #filtered_df = filtered_df.drop_duplicates(subset=[utils.AutoEvalColumn.model.name, utils.AutoEvalColumn.precision.name, utils.AutoEvalColumn.revision.name])
+            filtered_df = filtered_df.drop_duplicates(subset=[utils.AutoEvalColumn.model.name])
 
     return filtered_df
 
 
-def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
-    # Show all models
-    # if show_deleted:
-    #   filtered_df = df
-    # else:  # Show only still on the hub models
-        # filtered_df = df[df[utils.AutoEvalColumn.still_on_hub.name]]
-
+def filter_models(df: pd.DataFrame, type_query: list) -> pd.DataFrame:
     filtered_df = df
     
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df.loc[df[utils.AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[utils.AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+    # filtered_df = filtered_df.loc[df[utils.AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
-    numeric_interval = pd.IntervalIndex(sorted([utils.NUMERIC_INTERVALS[s] for s in size_query]))
-    params_column = pd.to_numeric(df[utils.AutoEvalColumn.params.name], errors="coerce")
-    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    filtered_df = filtered_df.loc[mask]
+    # numeric_interval = pd.IntervalIndex(sorted([utils.NUMERIC_INTERVALS[s] for s in size_query]))
+    # params_column = pd.to_numeric(df[utils.AutoEvalColumn.params.name], errors="coerce")
+    # mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+    # filtered_df = filtered_df.loc[mask]
 
     return filtered_df
 
@@ -148,12 +137,8 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
-#                    with gr.Row():
-#                        deleted_models_visibility = gr.Checkbox(
-#                            value=False, label="Show gated/private/deleted models", interactive=True
-#                        )
+
                 with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
                         label="Model types",
                         choices=[t.to_str() for t in utils.ModelType],
@@ -161,20 +146,6 @@ with demo:
                         interactive=True,
                         elem_id="filter-columns-type",
                     )
-                    # filter_columns_precision = gr.CheckboxGroup(
-                    #     label="Precision",
-                    #     choices=[i.value.name for i in utils.Precision],
-                    #     value=[i.value.name for i in utils.Precision],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-precision",
-                    # )
-                    # filter_columns_size = gr.CheckboxGroup(
-                    #     label="Model sizes (in billions of parameters)",
-                    #     choices=list(utils.NUMERIC_INTERVALS.keys()),
-                    #     value=list(utils.NUMERIC_INTERVALS.keys()),
-                    #     interactive=True,
-                    #     elem_id="filter-columns-size",
-                    # )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
@@ -203,23 +174,17 @@ with demo:
                     hidden_leaderboard_table_for_search,
                     shown_columns,
                     filter_columns_type,
-#                    filter_columns_precision,
-#                    filter_columns_size,
-#                    deleted_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
             )
-            for selector in [shown_columns, filter_columns_type]: #, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
+            for selector in [shown_columns, filter_columns_type]:
                 selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
                         shown_columns,
                         filter_columns_type,
-#                        filter_columns_precision,
-#                        filter_columns_size,
-#                        deleted_models_visibility,
                         search_bar,
                     ],
                     leaderboard_table,

main_backend.py

@@ -20,10 +20,8 @@ RUNNING_STATUS = "RUNNING"
 FINISHED_STATUS = "FINISHED"
 FAILED_STATUS = "FAILED"
 
-snapshot_download(repo_id=envs.RESULTS_REPO, revision="main",
-                local_dir=envs.EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-snapshot_download(repo_id=envs.QUEUE_REPO, revision="main",
-                local_dir=envs.EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+snapshot_download(repo_id=envs.RESULTS_REPO, revision="main", local_dir=envs.EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+snapshot_download(repo_id=envs.QUEUE_REPO, revision="main", local_dir=envs.EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
 
 
 def run_auto_eval(args):
@@ -41,9 +39,9 @@ def run_auto_eval(args):
             local_dir_results=envs.EVAL_RESULTS_PATH_BACKEND
         )
         logging.info("Checked completed evals")
-        eval_requests = manage_requests.get_eval_requests(job_status=current_pending_status,
-                                                        hf_repo=envs.QUEUE_REPO,
-                                                        local_dir=envs.EVAL_REQUESTS_PATH_BACKEND)
+        eval_requests = manage_requests.get_eval_requests(
+            job_status=current_pending_status, hf_repo=envs.QUEUE_REPO, local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
+        )
         logging.info("Got eval requests")
         eval_requests = sort_queue.sort_models_by_priority(api=envs.API, models=eval_requests)
         logging.info("Sorted eval requests")
@@ -65,20 +63,11 @@ def run_auto_eval(args):
             eval_request = eval_requests[0]
             pp.pprint(eval_request)
 
-        # manage_requests.set_eval_request(
-        #     api=envs.API,
-        #     eval_request=eval_request,
-        #     new_status=RUNNING_STATUS,
-        #     hf_repo=envs.QUEUE_REPO,
-        #     local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
-        # )
-        # logging.info("Set eval request to running, now running eval")
-
         run_eval_suite.run_evaluation(
             eval_request=eval_request,
             local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
             results_repo=envs.RESULTS_REPO,
-            batch_size=1,
+            batch_size=args.batch_size,
             device=envs.DEVICE,
             no_cache=True,
             need_check=not args.publish,
@@ -88,6 +77,7 @@ def run_auto_eval(args):
     else:
         eval_request = manage_requests.EvalRequest(
             model=args.model,
+            model_path=args.model_path,
             status=PENDING_STATUS,
             precision=args.precision
         )
@@ -98,10 +88,13 @@ def run_auto_eval(args):
             eval_request=eval_request,
             local_dir=envs.EVAL_RESULTS_PATH_BACKEND,
             results_repo=envs.RESULTS_REPO,
-            batch_size=1,
+            batch_size=args.batch_size,
             device=envs.DEVICE,
             need_check=not args.publish,
-            write_results=args.update
+            write_results=args.update,
+            limit=args.limit,
+            use_vllm=args.use_vllm,
+            tensor_parallel_size=args.tensor_parallel_size,
         )
         logging.info("Reproducibility eval finished")
 
@@ -112,9 +105,14 @@ def main():
     # Optional arguments
     parser.add_argument("--reproduce", type=bool, default=False, help="Reproduce the evaluation results")
     parser.add_argument("--model", type=str, default=None, help="Your Model ID")
+    parser.add_argument("--model_path", type=str, default=None, help="Full path of model")
     parser.add_argument("--precision", type=str, default="float16", help="Precision of your model")
     parser.add_argument("--publish", type=bool, default=False, help="whether directly publish the evaluation results on HF")
     parser.add_argument("--update", type=bool, default=False, help="whether to update google drive files")
+    parser.add_argument("--limit", type=int, default=None, help="Limit on the number of items to process")
+    parser.add_argument("--use_vllm", type=bool, default=False, help="Whether to infer with vllm or not")
+    parser.add_argument("--tensor_parallel_size", type=int, default=1)
+    parser.add_argument("--batch_size", type=int, default=1)
 
     args = parser.parse_args()
 

requirements.txt

@@ -27,6 +27,7 @@ google-cloud-aiplatform>=1.38
 qwen-vl-utils
 vertexai
 # git+https://github.com/huggingface/transformers
-transformers==4.45.2
+transformers==4.51.3
 together==1.3.0
-spacy
+spacy
+vllm==0.8.5

src/backend/evaluate_model.py

@@ -29,9 +29,14 @@ class Evaluator:
         summary_generator (SummaryGenerator): Instance for generating summaries.
         eval_model (EvaluationModel): Instance for evaluating summaries.
     """
-    def __init__(self, model, revision, precision, batch_size,
-                device, no_cache, limit, write_out=True,
-                output_base_path='logs'):
+    def __init__(
+        self, model, revision, precision, batch_size,
+        device, no_cache, limit, write_out=True,
+        output_base_path='logs',
+        model_path=None,
+        use_vllm=False,
+        tensor_parallel_size=1
+    ):
         """Initializes the Evaluator with the given model and settings.
 
         Args:
@@ -47,6 +52,7 @@ class Evaluator:
             output_base_path (str): Base path for output files.
         """
         self.model = model
+        self.model_path = model_path
         self.revision = revision
         self.precision = precision
         self.batch_size = batch_size
@@ -56,7 +62,7 @@ class Evaluator:
         self.write_out = write_out
         self.output_base_path = output_base_path
         try:
-            self.summary_generator = SummaryGenerator(model, revision, self.device)
+            self.summary_generator = SummaryGenerator(model, revision, self.device, model_path=self.model_path, use_vllm=use_vllm, tensor_parallel_size=tensor_parallel_size)
             self.eval_model = EvaluationModel(envs.HEM_PATH, self.device)
         except Exception as e:
             logging.error(f"Error initializing Evaluator: {e}")
@@ -71,26 +77,35 @@ class Evaluator:
             dict: A dictionary containing evaluation results.
         """
         try:
+            # print(envs.DATA_LEADERBOARD_REPO)
+            # snapshot_download(
+            #     repo_id=envs.DATA_LEADERBOARD_REPO, local_dir=envs.DATA_LEADERBOARD_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+            # )
+
             df = pd.read_csv(envs.DATASET_PATH)
-            self.generated_summaries_df = self.summary_generator.generate_summaries(df, save_path=f"generation_results/{self.model}.csv")
+            if self.limit is not None and self.limit > 0:
+                df = df.head(self.limit)
+            self.generated_summaries_df = self.summary_generator.generate_summaries(df, save_path=f"generation_results/{self.model}.csv", batch_size=self.batch_size)
 
             avg_summary_len = self.summary_generator.avg_length
             answer_rate = self.summary_generator.answer_rate
 
-            self.hallucination_scores, self.eval_results = self.eval_model.evaluate_hallucination(
-                self.generated_summaries_df)
+            self.hallucination_scores, self.eval_results = self.eval_model.evaluate_hallucination(self.generated_summaries_df)
             factual_consistency_rate = self.eval_model.compute_factual_consistency_rate()
             hallucination_rate = self.eval_model.hallucination_rate
 
-            results = util.format_results(model_name=self.model, revision=self.revision,
-                                        precision=self.precision,
-                                        factual_consistency_rate=factual_consistency_rate,
-                                        hallucination_rate=hallucination_rate,
-                                        answer_rate=answer_rate,
-                                        avg_summary_len=avg_summary_len)
+            results = util.format_results(
+                model_name=self.model,
+                revision=self.revision,
+                precision=self.precision,
+                factual_consistency_rate=factual_consistency_rate,
+                hallucination_rate=hallucination_rate,
+                answer_rate=answer_rate,
+                avg_summary_len=avg_summary_len
+            )
             return results
         except FileNotFoundError:
-            logging.error(f"File not found: {envs.DATASET_PATH}")
+            logging.error(f"File not found: {envs.DATA_LEADERBOARD_NAME}")
             raise
         except Exception as e:
             logging.error(f"Error during evaluation: {e}")

src/backend/manage_requests.py

@@ -24,6 +24,7 @@ class EvalRequest:
     likes: Optional[int] = 0
     params: Optional[int] = None
     license: Optional[str] = ""
+    model_path: Optional[str] = None
 
     def get_model_args(self):
         model_args = f"pretrained={self.model},revision={self.revision}"
@@ -36,8 +37,7 @@ class EvalRequest:
         return model_args
 
 
-def set_eval_request(api: HfApi, eval_request: EvalRequest, new_status: str,
-                    hf_repo: str, local_dir: str):
+def set_eval_request(api: HfApi, eval_request: EvalRequest, new_status: str, hf_repo: str, local_dir: str):
     """Updates a given eval request with its new status on the hub (running, completed, failed,)"""
     json_filepath = eval_request.json_filepath
 
@@ -65,8 +65,7 @@ def get_eval_requests(job_status: list, local_dir: str, hf_repo: str) -> list[Ev
     Returns:
         list[EvalRequest]: a list of model info dicts.
     """
-    snapshot_download(repo_id=hf_repo, revision="main", local_dir=local_dir,
-                    repo_type="dataset", max_workers=60)
+    snapshot_download(repo_id=hf_repo, revision="main", local_dir=local_dir, repo_type="dataset", max_workers=60)
     json_files = glob.glob(f"{local_dir}/**/*.json", recursive=True)
 
     eval_requests = []

src/backend/model_operations.py

@@ -11,7 +11,7 @@ import pandas as pd
 import spacy
 import litellm
 from tqdm import tqdm
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, AutoModelForTokenClassification, AutoConfig, Qwen2VLForConditionalGeneration, AutoProcessor
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, AutoModelForSequenceClassification, AutoConfig, Qwen2VLForConditionalGeneration, AutoProcessor
 from peft import PeftModel
 import torch
 import cohere
@@ -19,11 +19,12 @@ from openai import OpenAI
 from together import Together
 import anthropic
 import replicate
-# import google.generativeai as genai
 import vertexai
 from vertexai.generative_models import GenerativeModel, Part, SafetySetting, FinishReason
 from mistralai import Mistral
 from qwen_vl_utils import process_vision_info
+from vllm import LLM
+from vllm import SamplingParams
 
 
 import src.backend.util as util
@@ -32,8 +33,7 @@ import src.envs as envs
 litellm.set_verbose=True
 
 # Set up basic configuration for logging
-logging.basicConfig(level=logging.INFO,
-                    format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # Load spacy model for word tokenization
 nlp = spacy.load("en_core_web_sm")
@@ -66,7 +66,7 @@ class SummaryGenerator:
         answer_rate (float): Rate of non-empty summaries.
     """
 
-    def __init__(self, model_id, revision, device):
+    def __init__(self, model_id, revision, device, model_path=None, use_vllm=False, tensor_parallel_size=1):
         """
         Initializes the SummaryGenerator with a model.
 
@@ -76,6 +76,7 @@ class SummaryGenerator:
         """
         self.model_id = model_id
         self.model = f"huggingface/{model_id}"
+        self.model_path = model_path
         self.api_base = f"https://api-inference.huggingface.co/models/{model_id}"
         self.summaries_df = pd.DataFrame()
         self.revision = revision
@@ -86,7 +87,10 @@ class SummaryGenerator:
         self.local_model = None
         self.local_pipeline = None
 
-    def generate_summaries(self, df, save_path=None):
+        self.use_vllm = use_vllm
+        self.tensor_parallel_size = tensor_parallel_size
+
+    def generate_summaries(self, df, save_path=None, batch_size=1):
         """Generate summaries for a given DataFrame of source docs.
 
         Args:
@@ -113,7 +117,6 @@ class SummaryGenerator:
                 while not _summary:
                     try:
                         _summary = self.generate_summary(system_prompt, user_prompt)
-                        # print(f"Finish index {index}")
                         break
                     except Exception as e:
                         if 'Rate limit reached' in str(e):
@@ -142,8 +145,7 @@ class SummaryGenerator:
                 # Sleep to prevent hitting rate limits too frequently
                 time.sleep(1)
 
-            self.summaries_df = pd.DataFrame(list(zip(source, summary, dataset)),
-                                            columns=["source", "summary", "dataset"])
+            self.summaries_df = pd.DataFrame(list(zip(source, summary, dataset)), columns=["source", "summary", "dataset"])
 
             if save_path is not None:
                 print(f'Save summaries to {save_path}')
@@ -164,7 +166,7 @@ class SummaryGenerator:
         using_replicate_api = False
         replicate_api_models = ['snowflake', 'llama-3.1-405b']
         using_pipeline = False
-        pipeline_models = ['llama-3.1', 'phi-3-mini','falcon-7b', 'phi-3.5', 'mistral-nemo', 'llama-3.3']
+        pipeline_models = ['llama-3.1', 'phi-3-mini','falcon-7b', 'phi-3.5', 'mistral-nemo', 'llama-3.3', 'phi-4']
 
         for replicate_api_model in replicate_api_models:
             if replicate_api_model in self.model_id.lower():
@@ -405,40 +407,32 @@ class SummaryGenerator:
                     trust_remote_code=True
                 )
             else:
-                if 'ragamuffin' in self.model_id.lower():
-                    self.tokenizer = AutoTokenizer.from_pretrained(os.path.join('/home/miaoran', self.model_id))
-
-                else:
-                    self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf" if 'openelm' in self.model_id.lower() else self.model_id, trust_remote_code=True)
-                print("Tokenizer loaded")
-                if 'jamba' in self.model_id.lower():
-                    self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id,
-                                             torch_dtype=torch.bfloat16,
-                                             attn_implementation="flash_attention_2",
-                                             device_map="auto",
-                                             use_mamba_kernels=False)
+                print(f"loading tokenizer from {self.model_path or self.model_id}")
+                self.tokenizer = AutoTokenizer.from_pretrained(self.model_path or self.model_id, trust_remote_code=True)
                     
+                print(f"loading model from {self.model_path or self.model_id}")
+                if 'jamba' in self.model_id.lower():
+                    self.local_model = AutoModelForCausalLM.from_pretrained(
+                        self.model_id,
+                        torch_dtype=torch.bfloat16,
+                        attn_implementation="flash_attention_2",
+                        device_map="auto",
+                        use_mamba_kernels=False
+                    )
                 elif 'qwen2-vl' in self.model_id.lower():
                     self.local_model = Qwen2VLForConditionalGeneration.from_pretrained(
                         self.model_id, torch_dtype="auto", device_map="auto"
                     )
                     self.processor = AutoProcessor.from_pretrained(self.model_id)
-
-                # elif 'ragamuffin' in self.model_id.lower():
-                #     print('Using ragamuffin')
-                #     self.local_model = AutoModelForCausalLM.from_pretrained(os.path.join('/home/miaoran', self.model_id),
-                #                                  torch_dtype=torch.bfloat16, # forcing bfloat16 for now
-                #                                  attn_implementation="flash_attention_2")
                 elif 'olmo' in self.model_id.lower():
-                    self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id)#torch_dtype="auto"
-                
+                    self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id)
                 elif 'qwq-' in self.model_id.lower():
                     self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, torch_dtype="auto", device_map="auto")
-
                 else:
-                    self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, trust_remote_code=True, device_map="auto")#torch_dtype="auto"
-                # print(self.local_model.device)
-                print("Local model loaded")
+                    if self.use_vllm:
+                        self.local_model = LLM(self.model_path or self.model_id, tensor_parallel_size=self.tensor_parallel_size, trust_remote_code=True)
+                    else:
+                        self.local_model = AutoModelForCausalLM.from_pretrained(self.model_path or self.model_id, trust_remote_code=True, device_map="auto")
                     
         # Using local model/pipeline
         if self.local_pipeline:
@@ -502,7 +496,10 @@ class SummaryGenerator:
             elif 'qwq' in self.model_id.lower():
                 input_ids = self.tokenizer([prompt], return_tensors="pt").to(self.device)
             else:
-                input_ids = self.tokenizer(prompt, return_tensors="pt").to(self.device)
+                if self.use_vllm:
+                    input_ids = [prompt]
+                else:
+                    input_ids = self.tokenizer(prompt, return_tensors="pt").to(self.device)
             
             # Generate outputs
             if 'granite' in self.model_id.lower():
@@ -513,14 +510,18 @@ class SummaryGenerator:
             elif 'qwq' in self.model_id.lower():
                 outputs = self.local_model.generate(**input_ids, max_new_tokens=512, do_sample=True, temperature=0.01)
             else:
-                with torch.no_grad():
-                    outputs = self.local_model.generate(**input_ids, do_sample=True, max_new_tokens=250, temperature=0.01)#, pad_token_id=self.tokenizer.eos_token_id
+                if self.use_vllm:
+                    sampling_params = SamplingParams(temperature=0.01, max_tokens=250)
+                    outputs = self.local_model.generate(input_ids, sampling_params, use_tqdm=False)
+                else:
+                    with torch.no_grad():
+                        outputs = self.local_model.generate(**input_ids, do_sample=True, max_new_tokens=250, temperature=0.01)#, pad_token_id=self.tokenizer.eos_token_id
+            
             if 'glm' in self.model_id.lower() or 'ragamuffin' in self.model_id.lower() or 'granite' in self.model_id.lower():
                 outputs = outputs[:, input_ids['input_ids'].shape[1]:]
             elif 'qwen2-vl' in self.model_id.lower() or 'qwen2.5' in self.model_id.lower() or 'qwq-' in self.model_id.lower():
-                outputs = [
-                    out_ids[len(in_ids) :] for in_ids, out_ids in zip(input_ids.input_ids, outputs)
-                ]
+                if not self.use_vllm:
+                    outputs = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(input_ids.input_ids, outputs)]
             
             # Decode outputs
             if 'qwen2-vl' in self.model_id.lower():
@@ -530,7 +531,10 @@ class SummaryGenerator:
             elif 'olmo' in self.model_id.lower() or 'qwq' in self.model_id.lower():
                 result = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
             else:
-                result = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+                if self.use_vllm:
+                    result = outputs[0].outputs[0].text
+                else:
+                    result = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
 
             if 'gemma-2' in self.model_id.lower():
                 result = result.split(user_prompt + '\nmodel')[-1].strip()
@@ -593,13 +597,14 @@ class EvaluationModel:
         Args:
             model_path (str): Path to the CrossEncoder model.
         """
-        config = AutoConfig.from_pretrained('google/flan-t5-large')
-        self.model = AutoModelForTokenClassification.from_pretrained(model_path, config=config)
+        self.model = AutoModelForSequenceClassification.from_pretrained(model_path, trust_remote_code=True)
         self.device = device
         self.model.to(self.device)
         self.scores = []
         self.factual_consistency_rate = None
         self.hallucination_rate = None
+
+        self.model.eval()
     
     def predict(self, text_pairs):
         """Load LoRA adapters of HHEM and make predictions
@@ -609,20 +614,9 @@ class EvaluationModel:
             checkpoint: model ID on Hugging Face
         """
 
-        prompt = "<pad> Determine if the hypothesis is true given the premise?\n\nPremise: {text1}\n\nHypothesis: {text2}"
-        
-        tokenizer = AutoTokenizer.from_pretrained('t5-base')
-        inputs = tokenizer(
-            [prompt.format(text1=pair[0], text2=pair[1]) for pair in text_pairs], 
-            return_tensors='pt', padding='longest').to(self.device)
-        
-        self.model.eval()
         with torch.no_grad():
-            output = self.model(**inputs)
-        logits = output.logits
-        logits = logits[:,0,:] # get the logits on the first token
-        logits = torch.softmax(logits, dim=-1)
-        scores = [round(x, 5) for x in logits[:, 1].tolist()] # list of float
+            output = self.model.predict(text_pairs)
+        scores = output.tolist()
         return scores
 
     def evaluate_hallucination(self, summaries_df):

src/backend/run_eval_suite.py

@@ -8,14 +8,15 @@ from src.backend.manage_requests import EvalRequest
 from src.backend.evaluate_model import Evaluator
 
 # Configure logging
-logging.basicConfig(level=logging.INFO,
-                    format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logging.getLogger("openai").setLevel(logging.WARNING)
 
 
-def run_evaluation(eval_request: EvalRequest, batch_size, device,
-                local_dir: str, results_repo: str, no_cache=True, limit=None, 
-                need_check=True, write_results=False):
+def run_evaluation(
+    eval_request: EvalRequest, batch_size, device,
+    local_dir: str, results_repo: str, no_cache=True, limit=None, 
+    need_check=True, write_results=False, use_vllm=False, tensor_parallel_size=1,
+):
     """
     Run the evaluation for a given model and upload the results.
 
@@ -32,21 +33,20 @@ def run_evaluation(eval_request: EvalRequest, batch_size, device,
     Returns:
         dict: A dictionary containing evaluation results.
     """
-    if limit:
+    if limit is not None and limit > 0:
         logging.warning("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
     output_folder = os.path.join(local_dir, *eval_request.model.split("/"))
-    # if os.path.exists(output_folder):
-    #     f_name = os.listdir(output_folder)[-1] 
-    #     print(f"Loading results from {os.path.join(output_folder, f_name)}")
-    #     results = json.loads(os.path.join(output_folder, f_name))
-    #     dumped = json.dumps(results, indent=2)
-    #     logging.info(dumped)
-    # else:
     try:
-        evaluator = Evaluator(eval_request.model, eval_request.revision, eval_request.precision,
-                            batch_size, device, no_cache, limit, write_out=True,
-                            output_base_path='logs')
+        evaluator = Evaluator(
+            eval_request.model, eval_request.revision, eval_request.precision,
+            batch_size, device, no_cache, limit, write_out=True,
+            output_base_path='logs',
+            model_path=eval_request.model_path,
+            use_vllm=use_vllm,
+            tensor_parallel_size=tensor_parallel_size
+        )
+
         results = evaluator.evaluate()
         if write_results:
             evaluator.write_results()
@@ -67,8 +67,7 @@ def run_evaluation(eval_request: EvalRequest, batch_size, device,
     dumped = json.dumps(results, indent=2)
     logging.info(dumped)
 
-    output_path = os.path.join(output_folder,
-                            f"results_{datetime.now()}.json") #
+    output_path = os.path.join(output_folder, f"results_{datetime.now()}.json") #
     os.makedirs(output_folder, exist_ok=True)
     with open(output_path, "w") as f:
         f.write(dumped)

src/display/about.py

@@ -10,12 +10,10 @@ class Task:
 
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    hallucination_rate = Task("hallucination_rate",
-                            "hallucination_rate", "Hallucination Rate (%)")
+    hallucination_rate = Task("hallucination_rate", "hallucination_rate", "Hallucination Rate (%)")
     factual_consistency_rate = Task("factual_consistency_rate", "factual_consistency_rate", "Factual Consistency Rate (%)")
     answer_rate = Task("answer_rate", "answer_rate", "Answer Rate (%)")
-    average_summary_length = Task("average_summary_length",
-                                "average_summary_length", "Average Summary Length")
+    average_summary_length = Task("average_summary_length", "average_summary_length", "Average Summary Length")
 
 
 # Your leaderboard name

src/display/utils.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, make_dataclass, field
 from enum import Enum
 
 import pandas as pd
@@ -21,13 +21,19 @@ class ColumnContent:
     never_hidden: bool = False
     dummy: bool = False
 
+    def __hash__(self) -> int: 
+        import time
+        import random
+        seed = hash(self.name) + hash(self.type) + hash(self.displayed_by_default) + \
+               hash(self.hidden) + hash(self.never_hidden) + hash(self.dummy) + \
+               hash(time.time()) + random.randint(0, 10000)
+        return seed
+
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent,
-                            ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent,
-                            ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 

src/envs.py

@@ -6,7 +6,7 @@ from huggingface_hub import HfApi
 # replace this with our token
 TOKEN = os.environ.get("HF_TOKEN", None)
 
-OWNER = "vectara"
+OWNER = "airlsyn"
 REPO_ID = f"{OWNER}/leaderboard"
 QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
@@ -20,13 +20,18 @@ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
+DATA_LEADERBOARD_REPO = f"{OWNER}/leaderboard_dataset"
+DATA_LEADERBOARD_PATH = os.path.join(CACHE_PATH, "leaderboard-bk")
+DATA_LEADERBOARD_NAME = os.path.join(DATA_LEADERBOARD_PATH, "leaderboard_dataset_16k.csv")
+
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') #"cpu"
 API = HfApi(token=TOKEN)
 
 LEADERBOARD_DATASET_PATH = "leaderboard_results/leaderboard_summaries.csv"
 DATASET_PATH = "src/datasets/leaderboard_dataset.csv"
 SAMPLE_DATASET_PATH = "src/datasets/sample_dataset.csv"
-HEM_PATH = 'vectara/HHEM-2.1'
+# HEM_PATH = 'vectara/HHEM-2.1'
+HEM_PATH = 'vectara/hallucination_evaluation_model'
 
 SYSTEM_PROMPT = "You are a chat bot answering questions using data. You must stick to the answers provided solely by the text in the passage provided."
 USER_PROMPT = "You are asked the question 'Provide a concise summary of the following passage, covering the core pieces of information described': "

src/leaderboard/read_evals.py

@@ -104,7 +104,7 @@ class EvalResult:
 
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-#            utils.AutoEvalColumn.precision.name: self.precision.value.name,
+#           utils.AutoEvalColumn.precision.name: self.precision.value.name,
             utils.AutoEvalColumn.model_type.name: self.model_type.value.name,
             utils.AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             utils.AutoEvalColumn.weight_type.name: self.weight_type.value.name,
@@ -114,7 +114,7 @@ class EvalResult:
 #            utils.AutoEvalColumn.revision.name: self.revision,
 #            utils.AutoEvalColumn.license.name: self.license,
 #            utils.AutoEvalColumn.likes.name: self.likes,
-#            utils.AutoEvalColumn.params.name: self.num_params,
+#           utils.AutoEvalColumn.params.name: self.num_params,
 #            utils.AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
@@ -172,8 +172,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in
-                                                    eval_result.results.items() if v is not None})
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
         else:
             eval_results[eval_name] = eval_result