Update demo

README.md

@@ -1,3 +1,16 @@
+---
+title: AgentReview
+emoji: 🎓
+colorFrom: indigo
+colorTo: pink
+sdk: gradio
+sdk_version: 5.4.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: EMNLP 2024
+---
+
 # AgentReview
 
 Official implementation for the 🔗[EMNLP 2024](https://2024.emnlp.org/) (main) paper: [AgentReview: Exploring Peer Review Dynamics with LLM Agents](https://arxiv.org/abs/2406.12708)

agentreview/agent.py

@@ -69,8 +69,10 @@ class Player(Agent):
         self.data_dir = kwargs.pop("data_dir", None)
         self.args = args
 
+
         if isinstance(backend, BackendConfig):
             backend_config = backend
+            backend_config['openai_client_type'] = args.openai_client_type
             backend = load_backend(backend_config)
         elif isinstance(backend, IntelligenceBackend):
             backend_config = backend.to_config()

arguments.py → agentreview/arguments.py

@@ -26,7 +26,8 @@ def parse_args():
 
 
     parser.add_argument(
-        "--api_version", type=str, default="2023-03-15-preview", help="API version to be used for making requests. Required for Azure OpenAI clients."
+        "--api_version", type=str, default="2023-05-15", help="API version to be used for making requests. Required "
+                                                              "for Azure OpenAI clients."
     )
 
     # Experiment configuration
@@ -54,11 +55,10 @@ def parse_args():
     )
 
     parser.add_argument(
-        "--ignore_missing_metareviews", action="store_true", help="If set, missing metareviews are ignored, allowing the experiment to continue without them."
+        "--overwrite", action="store_true", help="If set, existing results or output files will be overwritten without prompting."
     )
-
     parser.add_argument(
-        "--overwrite", action="store_true", help="If set, existing results or output files will be overwritten without prompting."
+        "--skip_logging", action="store_true", help="If set, we do not log the messages in the console."
     )
 
     parser.add_argument(

agentreview/backends/openai.py

@@ -3,17 +3,11 @@ from typing import List
 
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
-from arguments import parse_args
-from utility.authentication_utils import get_openai_client
+from agentreview.arguments import parse_args
+from agentreview.utility.authentication_utils import get_openai_client
 from .base import IntelligenceBackend
 from ..message import SYSTEM_NAME, Message
 
-args = parse_args()
-
-client = get_openai_client(client_type=args.openai_client_type)
-
-OPENAI_CLIENT_TYPE = args.openai_client_type
-
 # Default config follows the OpenAI playground
 DEFAULT_TEMPERATURE = 1.0
 DEFAULT_MAX_TOKENS = 4096
@@ -57,19 +51,22 @@ class OpenAIChat(IntelligenceBackend):
             merge_other_agents_as_one_user=merge_other_agents_as_one_user,
             **kwargs,
         )
-
+        self.client_type = kwargs.get("openai_client_type", None)
+        self.client = get_openai_client(self.client_type)
         self.temperature = temperature
         self.max_tokens = max_tokens
         self.model = model
         self.merge_other_agent_as_user = merge_other_agents_as_one_user
 
+
+
     @retry(stop=stop_after_attempt(6), wait=wait_random_exponential(min=1, max=60))
     def _get_response(self, messages):
         # Refer to https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/switching-endpoints for how to
         # make API calls
 
-        if OPENAI_CLIENT_TYPE == "openai":
-            completion = client.chat.completions.create(
+        if self.client_type == "openai":
+            completion = self.client.chat.completions.create(
                 model=self.model,
                 messages=messages,
                 temperature=self.temperature,
@@ -77,8 +74,8 @@ class OpenAIChat(IntelligenceBackend):
                 stop=STOP,
             )
 
-        elif OPENAI_CLIENT_TYPE == "azure_openai":
-            completion = client.chat.completions.create(
+        elif self.client_type == "azure_openai":
+            completion = self.client.chat.completions.create(
                 model=self.model,
                 messages=messages,
                 temperature=self.temperature,
@@ -90,6 +87,7 @@ class OpenAIChat(IntelligenceBackend):
             raise NotImplementedError
 
         response = completion.choices[0].message.content
+
         response = response.strip()
         return response
 

const.py → agentreview/const.py

@@ -10,6 +10,8 @@ PAPER_DECISIONS_ICLR2019 = ["Accept-oral", "Accept-poster", "Reject"]
 
 AREA_CHAIR_TYPES = ['inclusive', 'conformist', 'authoritarian', 'BASELINE']
 
+GLOBAL_PROMPT = "This is a realistic simulation of academic peer review."
+
 # These are papers that contain potentially sensitive content. GPT-4 refused to generate reviews for these papers.
 FILTERED_PAPER_IDS = {
     "ICLR2020": [],

agentreview/dataset/download_openreview_paper.py

@@ -15,14 +15,14 @@ import os
 import time
 import requests
 
-from arguments import parse_args
+from agentreview.arguments import parse_args
 
 try:
     import openreview
 except ImportError:
     raise ImportError("Please install openreview package using `pip install openreview-py`")
 
-def download_papers():
+def download_papers(args):
     """Downloads all papers from ICLR 2023 using OpenReview API.
 
     This function authenticates with the OpenReview API using environment
@@ -36,8 +36,6 @@ def download_papers():
         AssertionError: If the conference argument is not for ICLR.
     """
 
-    args = parse_args()
-
     openreview_username = os.environ.get("OPENREVIEW_USERNAME")
     openreview_password = os.environ.get("OPENREVIEW_PASSWORD")
 
@@ -133,4 +131,5 @@ def download_papers():
 
 
 if __name__ == "__main__":
-    download_papers()
+    args = parse_args()
+    download_papers(args)

agentreview/dataset/process_submissions.py

@@ -22,9 +22,8 @@ from tqdm import tqdm
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
-import const
-from arguments import parse_args
-from utility.utils import print_colored
+from agentreview.arguments import parse_args
+from agentreview.utility.utils import print_colored
 
 decision_map = {
     # ICLR 2023

agentreview/environments/paper_review.py

@@ -1,11 +1,10 @@
 import json
-import json
 import logging
 import os.path as osp
 from typing import List
 
 from agentreview.environments import Conversation
-from utility.utils import get_rebuttal_dir
+from agentreview.utility.utils import get_rebuttal_dir
 from .base import TimeStep
 from ..message import Message
 from ..paper_review_message import PaperReviewMessagePool

agentreview/paper_processor.py

@@ -148,7 +148,7 @@ def convert_text_into_dict(text: str) -> dict:
 
 
 if __name__ == "__main__":
-    from utility.authentication_utils import read_and_set_openai_key
+    from agentreview.utility.authentication_utils import read_and_set_openai_key
     from agentreview.review import get_lm_review
 
     read_and_set_openai_key()

agentreview/paper_review_arena.py

@@ -1,14 +1,11 @@
 import csv
-import glob
 import json
 import logging
-import os
 from typing import Union
 
 from agentreview.arena import Arena, TooManyInvalidActions
 from agentreview.role_descriptions import get_reviewer_description
-from utility.utils import get_next_review_id, get_reviewer_type_from_profile, \
-    get_paper_review_and_rebuttal_dir, format_metareviews
+from agentreview.utility.utils import format_metareviews
 from .agent import Player
 from .config import ArenaConfig
 from .environments import TimeStep, load_environment

agentreview/paper_review_player.py

@@ -56,6 +56,8 @@ class Reviewer(Player):
             global_prompt: str = None,
             **kwargs,
     ):
+        print("kwargs")
+        print(kwargs)
         super().__init__(name, role_desc, backend, global_prompt, **kwargs)
 
     def act(self, observation: List[Message]) -> str:
@@ -94,7 +96,7 @@ class PaperExtractorPlayer(Player):
         Returns:
             str: The action (response) of the player.
         """
-        print("Improve paper loading")
+
         logging.info(f"Loading {self.conference} paper {self.paper_id} ({self.paper_decision}) ...")
 
         loader = PDFReader()

agentreview/paper_review_settings.py

@@ -1,3 +1,5 @@
+from typing import Union
+
 default_reviewer_setting = {
     "is_benign": None,
     "is_knowledgeable": None,
@@ -6,7 +8,7 @@ default_reviewer_setting = {
 }
 
 
-def get_experiment_settings(setting: dict):
+def get_experiment_settings(paper_id: Union[int, None] = None, paper_decision: Union[str, None] = None, setting: dict = None):
     """
     Generate experiment settings based on provided configurations for area chairs (AC) and reviewers.
 
@@ -19,7 +21,8 @@ def get_experiment_settings(setting: dict):
     """
 
     experiment_setting = {
-        "id": None,
+        "paper_id": paper_id,
+        "paper_decision": paper_decision,
         "players": {
 
             # Paper Extractor is a special player that extracts a paper from the dataset.

agentreview/role_descriptions.py

@@ -5,7 +5,7 @@ import numpy as np
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
-import const
+from agentreview import const
 from agentreview.config import AgentConfig
 
 PLAYER_BACKEND = {

agentreview/ui/cli.py

@@ -1,5 +1,4 @@
 import logging
-import logging
 import os
 import os.path as osp
 from typing import Union
@@ -11,8 +10,8 @@ from prompt_toolkit.completion import WordCompleter
 from prompt_toolkit.styles import Style
 from rich.console import Console
 
-from utility.utils import get_rebuttal_dir, load_gpt4_generated_ac_decisions, \
-    save_gpt4_generated_ac_decisions
+from agentreview.utility.utils import get_rebuttal_dir, load_llm_ac_decisions, \
+    save_llm_ac_decisions
 from ..arena import Arena, TooManyInvalidActions
 from ..backends.human import HumanBackendError
 from ..environments import PaperReview, PaperDecision
@@ -222,7 +221,8 @@ class ArenaCLI:
             # Print the new messages
             for msg in messages:
                 message_str = f"[{msg.agent_name}->{msg.visible_to}]: {msg.content}"
-                console.print(color_dict[name_to_color[msg.agent_name]] + message_str + CRStyle.RESET_ALL)
+                if self.args.skip_logging:
+                    console.print(color_dict[name_to_color[msg.agent_name]] + message_str + CRStyle.RESET_ALL)
                 msg.logged = True
 
             step += 1
@@ -251,7 +251,7 @@ class ArenaCLI:
             self.arena.save_history(path_review_history)
 
         elif env.type_name == "paper_decision":
-            ac_decisions = load_gpt4_generated_ac_decisions(output_dir=args.output_dir,
+            ac_decisions = load_llm_ac_decisions(output_dir=args.output_dir,
                                                             conference=args.conference,
                                                             model_name=args.model_name,
                                                             ac_scoring_method=args.ac_scoring_method,
@@ -261,9 +261,9 @@ class ArenaCLI:
 
             ac_decisions += [env.ac_decisions]
 
-            save_gpt4_generated_ac_decisions(ac_decisions,
-                                             output_dir=args.output_dir,
-                                             conference=args.conference,
-                                             model_name=args.model_name,
-                                             ac_scoring_method=args.ac_scoring_method,
-                                             experiment_name=args.experiment_name)
+            save_llm_ac_decisions(ac_decisions,
+                                 output_dir=args.output_dir,
+                                 conference=args.conference,
+                                 model_name=args.model_name,
+                                 ac_scoring_method=args.ac_scoring_method,
+                                 experiment_name=args.experiment_name)

{utility → agentreview/utility}/authentication_utils.py

@@ -16,6 +16,16 @@ def get_openai_client(client_type: str):
 
     assert client_type in ["azure_openai", "openai"]
 
+    endpoint: str = os.environ['AZURE_ENDPOINT']
+
+    if not endpoint.startswith("https://"):
+        endpoint = f"https://{endpoint}.openai.azure.com"
+
+    os.environ['AZURE_ENDPOINT'] = endpoint
+
+    if not os.environ.get('OPENAI_API_VERSION'):
+        os.environ['OPENAI_API_VERSION'] = "2023-05-15"
+
     if client_type == "openai":
         client = openai.OpenAI(
             api_key=os.environ['OPENAI_API_KEY']

agentreview/utility/experiment_utils.py

@@ -0,0 +1,84 @@
+import os
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from agentreview.agent import Player
+from agentreview.paper_review_player import PaperExtractorPlayer, AreaChair, Reviewer
+from agentreview.role_descriptions import get_ac_config, get_reviewer_player_config, get_author_config, \
+    get_paper_extractor_config
+
+
+def initialize_players(experiment_setting: dict, args):
+    paper_id = experiment_setting['paper_id']
+    paper_decision = experiment_setting['paper_decision']
+
+    if args.task == "paper_decision":
+        experiment_setting["players"] = {k: v for k, v in experiment_setting["players"].items() if k.startswith("AC")}
+
+    players = []
+
+    for role, players_list in experiment_setting["players"].items():
+
+        for i, player_config in enumerate(players_list):
+            if role == "AC":
+
+                # For AC, `env_type` is either "paper_decision" or "paper_review"
+                player_config = get_ac_config(env_type=args.task,
+                                              scoring_method=args.ac_scoring_method,
+                                              num_papers_per_area_chair=args.num_papers_per_area_chair,
+                                              global_settings=experiment_setting['global_settings'],
+                                              acceptance_rate=args.acceptance_rate,
+                                              **player_config)
+
+                player_config['model'] = args.model_name
+
+                player = AreaChair(data_dir=args.data_dir,
+                                   conference=args.conference,
+                                   args=args,
+                                   **player_config)
+
+
+            elif args.task == "paper_review":
+
+
+                if role == "Paper Extractor":
+
+                    player_config = get_paper_extractor_config(global_settings=experiment_setting['global_settings'])
+
+                    player = PaperExtractorPlayer(data_dir=args.data_dir, paper_id=paper_id,
+                                                  paper_decision=paper_decision,
+                                                  args=args,
+                                                  conference=args.conference, **player_config)
+
+
+
+                elif role == "Author":
+
+                    # Author requires no behavior customization.
+                    # So we directly use the Player class
+                    player_config = get_author_config()
+                    player = Player(data_dir=args.data_dir,
+                                    conference=args.conference,
+                                    args=args,
+                                    **player_config)
+
+
+
+                elif role == "Reviewer":
+                    player_config = get_reviewer_player_config(reviewer_index=i + 1,
+                                                               global_settings=experiment_setting['global_settings'],
+                                                               **player_config)
+                    player_config['model'] = args.model_name
+                    player = Reviewer(data_dir=args.data_dir, conference=args.conference, args=args, **player_config)
+
+
+                else:
+                    raise NotImplementedError(f"Unknown role for paper review (stage 1-4): {role}")
+
+            else:
+                raise NotImplementedError(f"Unknown role for paper decision (stage 5): {role}")
+
+            players.append(player)
+
+    return players

{utility → agentreview/utility}/utils.py

@@ -9,8 +9,8 @@ from typing import Union, List, Dict, Tuple
 import numpy as np
 import pandas as pd
 
-import const
-from utility.general_utils import check_cwd, set_seed
+from agentreview import const
+from agentreview.utility.general_utils import check_cwd, set_seed
 
 
 def generate_num_papers_to_accept(n, batch_number, shuffle=True):
@@ -36,25 +36,25 @@ def generate_num_papers_to_accept(n, batch_number, shuffle=True):
     return array
 
 
-def get_papers_accepted_by_gpt4(gpt4_generated_ac_decisions) -> list:
-    papers_accepted_by_gpt4 = []
+def get_papers_accepted_by_llm(llm_ac_decisions, acceptance_rate: float) -> list:
+    papers_accepted_by_llm = []
 
-    num_papers = sum([len(batch) for batch in gpt4_generated_ac_decisions])
+    num_papers = sum([len(batch) for batch in llm_ac_decisions])
 
     if num_papers == 0:
         raise ValueError("No papers found in batch")
 
-    num_papers_to_accept = generate_num_papers_to_accept(n=paper_review_config.ACCEPTANCE_RATE * num_papers,
-                                                         batch_number=len(gpt4_generated_ac_decisions))
+    num_papers_to_accept = generate_num_papers_to_accept(n=acceptance_rate * num_papers,
+                                                         batch_number=len(llm_ac_decisions))
 
-    for idx_batch, batch in enumerate(gpt4_generated_ac_decisions):
+    for idx_batch, batch in enumerate(llm_ac_decisions):
         tups = sorted([(paper_id, rank) for paper_id, rank in batch.items()], key=lambda x: x[1], reverse=False)
 
         paper_ids = [int(paper_id) for paper_id, rank in tups]
 
-        papers_accepted_by_gpt4 += paper_ids[:num_papers_to_accept[idx_batch]]
+        papers_accepted_by_llm += paper_ids[:num_papers_to_accept[idx_batch]]
 
-    return papers_accepted_by_gpt4
+    return papers_accepted_by_llm
 
 
 def get_paper_decision_mapping(data_dir: str, conference: str, verbose: bool = False):
@@ -151,6 +151,8 @@ def get_rebuttal_dir(output_dir: str,
 
 
 def print_colored(text, color='red'):
+
+    # Dictionary of ANSI color codes for terminal
     foreground_colors = {
         'black': 30,
         'red': 31,
@@ -161,7 +163,16 @@ def print_colored(text, color='red'):
         'cyan': 36,
         'white': 37,
     }
-    print(f"\033[{foreground_colors[color]}m{text}\033[0m")
+    try:
+
+        # get_ipython is specific to Jupyter and IPython.
+        # We use this to decide whether we are running a Jupyter notebook or not.
+        get_ipython
+        print(text)  # Plain text in Jupyter
+    except:
+        # If not Jupyter, print with color codes
+        color_code = foreground_colors.get(color, 31)  # Default to red if color not found
+        print(f"\033[{color_code}m{text}\033[0m")
 
 
 def get_ac_decision_path(output_dir: str, conference: str, model_name: str, ac_scoring_method: str, experiment_name:
@@ -351,71 +362,116 @@ def get_experiment_names(conference: str = "ICLR2023"):
     return experiment_names
 
 
-def load_gpt4_generated_ac_decisions_as_array(experiment_name, **kwargs) -> Tuple[np.ndarray, np.ndarray]:
-    ac_scoring_method = kwargs.pop('ac_scoring_method')
-    acceptance_rate = kwargs.pop('acceptance_rate')
-    conference = kwargs.pop('conference')
-    model_name = kwargs.pop('model_name')
-    num_papers_per_area_chair = kwargs.pop('num_papers_per_area_chair')
+def load_llm_ac_decisions_as_array(
+    output_dir: str,
+    experiment_name: str,
+    ac_scoring_method: str,
+    acceptance_rate: float,
+    conference: str,
+    model_name: str,
+    num_papers_per_area_chair: int
+) -> Tuple[np.ndarray, np.ndarray]:
+    """Loads and processes GPT-4 generated area chair (AC) decisions for an experiment.
+
+    Args:
+        experiment_name (str): Name of the experiment.
+        ac_scoring_method (str): Method used for AC scoring ('ranking' or 'recommendation').
+        acceptance_rate (float): Acceptance rate for the conference.
+        conference (str): Name of the conference.
+        model_name (str): Model name used to generate AC decisions.
+        num_papers_per_area_chair (int): Number of papers assigned to each area chair.
+
+    Returns:
+        Tuple[np.ndarray, np.ndarray]: An array of decisions (True for accept, False for reject)
+            and an array of paper IDs in the order processed.
 
+    Raises:
+        NotImplementedError: If `ac_scoring_method` is not 'ranking' or 'recommendation'.
+    """
     print("=" * 30)
     print(f"Experiment Name: {experiment_name}")
 
-    gpt4_generated_ac_decisions = load_gpt4_generated_ac_decisions(conference=conference,
-                                                                   model_name=model_name,
-                                                                   ac_scoring_method=ac_scoring_method,
-                                                                   experiment_name=experiment_name,
-                                                                   num_papers_per_area_chair=num_papers_per_area_chair)
+    llm_ac_decisions = load_llm_ac_decisions(
+        output_dir=output_dir,
+        conference=conference,
+        model_name=model_name,
+        ac_scoring_method=ac_scoring_method,
+        experiment_name=experiment_name,
+        num_papers_per_area_chair=num_papers_per_area_chair
+    )
 
-    paper_ids = sorted([int(paper_id) for batch in gpt4_generated_ac_decisions for paper_id, rank in batch.items()])
-    # ac_decisions['paper_ids'] = paper_ids
+    paper_ids = sorted(
+        int(paper_id) for batch in llm_ac_decisions for paper_id in batch
+    )
 
     if ac_scoring_method == "ranking":
-        assert len(paper_ids) == len(set(paper_ids)), (f"Duplicate paper_ids found in the AC decisions. "
-                                                       f"{Counter(paper_ids)}")
+        if len(paper_ids) != len(set(paper_ids)):
+            raise ValueError(f"Duplicate paper_ids found in the AC decisions: {Counter(paper_ids)}")
 
-        papers_accepted_by_gpt4 = get_papers_accepted_by_gpt4(gpt4_generated_ac_decisions, acceptance_rate)
-
-        # True means accept, False means reject
-        decisions_gpt4 = np.array(
-            [True if paper_id in papers_accepted_by_gpt4 else False for paper_id in paper_ids])
+        papers_accepted_by_llm = get_papers_accepted_by_llm(llm_ac_decisions, acceptance_rate)
+        decisions_llm = np.array([paper_id in papers_accepted_by_llm for paper_id in paper_ids])
 
     elif ac_scoring_method == "recommendation":
-        gpt4_generated_ac_decisions = {int(k): v for batch in gpt4_generated_ac_decisions for k, v in batch.items()}
-        decisions_gpt4 = np.array(
-            [True if gpt4_generated_ac_decisions[paper_id].startswith("Accept") else False for paper_id in
-             paper_ids])
-
+        llm_ac_decisions_flat = {int(k): v for batch in llm_ac_decisions for k, v in batch.items()}
+        decisions_llm = np.array(
+            [llm_ac_decisions_flat[paper_id].startswith("Accept") for paper_id in paper_ids]
+        )
     else:
-        raise NotImplementedError
+        raise NotImplementedError(f"Scoring method '{ac_scoring_method}' not implemented.")
 
-    return decisions_gpt4, paper_ids
+    return decisions_llm, np.array(paper_ids)
 
 
-def load_gpt4_generated_ac_decisions(**kwargs) -> List[Dict]:
-    num_papers_per_area_chair = kwargs.pop('num_papers_per_area_chair')
-    path = get_ac_decision_path(**kwargs)
+def load_llm_ac_decisions(
+    output_dir: str,
+    conference: str,
+    model_name: str,
+    ac_scoring_method: str,
+    experiment_name: str,
+    num_papers_per_area_chair: int
+) -> List[Dict[str, str]]:
+    """Loads GPT-4 generated area chair (AC) decisions from a specified path.
+
+    Args:
+        conference (str): Name of the conference.
+        model_name (str): Model name used to generate AC decisions.
+        ac_scoring_method (str): Method used for AC scoring ('ranking' or 'recommendation').
+        experiment_name (str): Name of the experiment.
+        num_papers_per_area_chair (int): Number of papers assigned to each area chair.
+
+    Returns:
+        List[Dict[str, str]]: List of batches, where each batch contains paper ID and decision.
+
+    Raises:
+        AssertionError: If a non-final batch has a paper count different from `num_papers_per_area_chair`.
+    """
+    path = get_ac_decision_path(
+        output_dir=output_dir,
+        conference=conference,
+        model_name=model_name,
+        ac_scoring_method=ac_scoring_method,
+        experiment_name=experiment_name
+    )
 
     if osp.exists(path):
-        ac_decision = json.load(open(path, 'r', encoding='utf-8'))
+        with open(path, 'r', encoding='utf-8') as file:
+            ac_decision = json.load(file)
         print(f"Loaded {len(ac_decision)} batches of existing AC decisions from {path}")
-
     else:
         ac_decision = []
         print(f"No existing AC decisions found at {path}")
 
-    ac_decision = [batch for batch in ac_decision if len(batch) > 0]
+    ac_decision = [batch for batch in ac_decision if batch]  # Remove empty batches
 
     for i, batch in enumerate(ac_decision):
         if i != len(ac_decision) - 1:
-            assert len(batch) == num_papers_per_area_chair, (f"Batch {i} has {len(batch)} papers, "
-                                                             f"but each AC should be assigned"
-                                                             f" {num_papers_per_area_chair} "
-                                                             f"unless it is the last batch.")
+            if len(batch) != num_papers_per_area_chair:
+                raise AssertionError(
+                    f"Batch {i} has {len(batch)} papers, expected {num_papers_per_area_chair} for non-final batches."
+                )
 
     return ac_decision
 
-
 def write_to_excel(data, file_path, sheet_name):
     """
     Write data to an Excel file.
@@ -436,7 +492,7 @@ def write_to_excel(data, file_path, sheet_name):
             data.to_excel(writer, sheet_name=sheet_name, index=False)
 
 
-def save_gpt4_generated_ac_decisions(ac_decisions: List[dict], **kwargs):
+def save_llm_ac_decisions(ac_decisions: List[dict], **kwargs):
     path = get_ac_decision_path(**kwargs)
 
     json.dump(ac_decisions, open(path, 'w', encoding='utf-8'), indent=2)

data

@@ -0,0 +1 @@
+../agent4reviews/data

demo.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # AgentReview
+# 
+# 
+# 
+# In this tutorial, you will explore customizing the AgentReview experiment.
+# 
+# 📑 Venue: EMNLP 2024 (Oral)
+# 
+# 🔗 arXiv: [https://arxiv.org/abs/2406.12708](https://arxiv.org/abs/2406.12708)
+# 
+# 🌐 Website: [https://agentreview.github.io/](https://agentreview.github.io/)
+# 
+# ```bibtex
+# @inproceedings{jin2024agentreview,
+#   title={AgentReview: Exploring Peer Review Dynamics with LLM Agents},
+#   author={Jin, Yiqiao and Zhao, Qinlin and Wang, Yiyang and Chen, Hao and Zhu, Kaijie and Xiao, Yijia and Wang, Jindong},
+#   booktitle={EMNLP},
+#   year={2024}
+# }
+# ```
+# 
+
+# In[2]:
+
+
+import os
+
+import numpy as np
+
+from agentreview import const
+
+os.environ["OPENAI_API_VERSION"] = "2024-06-01-preview"
+
+
+# ## Overview
+# 
+# AgentReview features a range of customizable variables, such as characteristics of reviewers, authors, area chairs (ACs), as well as the reviewing mechanisms 
+
+# In[3]:
+
+
+
+# ## Review Pipeline
+# 
+# The simulation adopts a structured, 5-phase pipeline (Section 2 in the [paper](https://arxiv.org/abs/2406.12708)):
+# 
+# * **I. Reviewer Assessment.** Each manuscript is evaluated by three reviewers independently.
+# * **II. Author-Reviewer Discussion.** Authors submit rebuttals to address reviewers' concerns;
+# * **III. Reviewer-AC Discussion.** The AC facilitates discussions among reviewers, prompting updates to their initial assessments.
+# * **IV. Meta-Review Compilation.** The AC synthesizes the discussions into a meta-review.
+# * **V. Paper Decision.** The AC makes the final decision on whether to accept or reject the paper, based on all gathered inputs.
+
+# In[2]:
+
+
+
+# In[4]:
+
+
+import os
+
+if os.path.basename(os.getcwd()) == "notebooks":
+    os.chdir("..")
+# Change the working directory to AgentReview
+print(f"Changing the current working directory to {os.path.basename(os.getcwd())}")
+
+
+# In[5]:
+
+
+from argparse import Namespace
+
+args = Namespace(openai_key=None, 
+          deployment=None, 
+          openai_client_type='azure_openai', 
+          endpoint=None, 
+          api_version='2023-05-15',
+          ac_scoring_method='ranking', 
+          conference='ICLR2024', 
+          num_reviewers_per_paper=3,  
+          ignore_missing_metareviews=False, 
+          overwrite=False, 
+          num_papers_per_area_chair=10, 
+          model_name='gpt-4o', 
+          output_dir='outputs', 
+          max_num_words=16384, 
+          visual_dir='outputs/visual', 
+          device='cuda', 
+          data_dir='./data', # Directory to all paper PDF
+          acceptance_rate=0.32, 
+          task='paper_review')
+
+os.environ['OPENAI_API_VERSION'] = args.api_version
+
+# In[13]:
+
+
+malicious_Rx1_setting = {
+    "AC": [
+        "BASELINE"
+    ],
+
+    "reviewer": [
+        "malicious",
+        "BASELINE",
+        "BASELINE"
+    ],
+
+    "author": [
+        "BASELINE"
+    ],
+    "global_settings":{
+        "provides_numeric_rating": ['reviewer', 'ac'],
+        "persons_aware_of_authors_identities": []
+    }
+}
+
+all_settings = {"malicious_Rx1": malicious_Rx1_setting}
+args.experiment_name = "malicious_Rx1_setting"
+
+
+# 
+# `malicious_Rx1` means 1 reviewer is a malicious reviewer, and the other reviewers are default (i.e. `BASELINE`) reviewers.
+# 
+# 
+
+# ## Reviews
+# 
+# Define the review pipeline
+
+# In[10]:
+
+
+from agentreview.environments import PaperReview
+
+def review_one_paper(paper_id, setting):
+    paper_decision = paper_id2decision[paper_id]
+
+    experiment_setting = get_experiment_settings(paper_id=paper_id,
+                                                 paper_decision=paper_decision,
+                                                 setting=setting)
+    print(f"Paper ID: {paper_id} (Decision in {args.conference}: {paper_decision})")
+
+    players = initialize_players(experiment_setting=experiment_setting, args=args)
+
+    player_names = [player.name for player in players]
+
+    env = PaperReview(player_names=player_names, paper_decision=paper_decision, paper_id=paper_id,
+                          args=args, experiment_setting=experiment_setting)
+
+    arena = PaperReviewArena(players=players, environment=env, args=args)
+    arena.launch_cli(interactive=False)
+
+
+# In[11]:
+
+
+import os
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "agentreview")))
+
+from agentreview.paper_review_settings import get_experiment_settings
+from agentreview.paper_review_arena import PaperReviewArena
+from agentreview.utility.experiment_utils import initialize_players
+from agentreview.utility.utils import project_setup, get_paper_decision_mapping
+
+
+# In[14]:
+
+
+sampled_paper_ids = [39]
+
+paper_id2decision, paper_decision2ids = get_paper_decision_mapping(args.data_dir, args.conference)
+
+for paper_id in sampled_paper_ids:
+    review_one_paper(paper_id, malicious_Rx1_setting)
+
+
+
+def run_paper_decision():
+    args.task = "paper_decision"
+
+    # Make sure the same set of papers always go through the same AC no matter which setting we choose
+    NUM_PAPERS = len(const.year2paper_ids[args.conference])
+    order = np.random.choice(range(NUM_PAPERS), size=NUM_PAPERS, replace=False)
+
+
+    # Paper IDs we actually used in experiments
+    experimental_paper_ids = []
+
+    # For papers that have not been decided yet, load their metareviews
+    metareviews = []
+    print("Shuffling paper IDs")
+    sampled_paper_ids = np.array(const.year2paper_ids[args.conference])[order]
+
+    # Exclude papers that already have AC decisions
+    existing_ac_decisions = load_llm_ac_decisions(output_dir=args.output_dir,
+                                                             conference=args.conference,
+                                                             model_name=args.model_name,
+                                                             ac_scoring_method=args.ac_scoring_method,
+                                                             experiment_name=args.experiment_name,
+                                                             num_papers_per_area_chair=args.num_papers_per_area_chair)
+
+    sampled_paper_ids = [paper_id for paper_id in sampled_paper_ids if paper_id not in existing_ac_decisions]
+
+
+
+
+# In[ ]:
+
+
+
+

requirements.txt

@@ -16,4 +16,4 @@ transformers
 tenacity
 openai
 gradio
-
+jupyter

run_paper_decision_cli.py

@@ -6,17 +6,15 @@ import numpy as np
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
-import const
+from agentreview import const
+from agentreview.utility.experiment_utils import initialize_players
 from agentreview.experiment_config import all_settings
 from agentreview.paper_review_settings import get_experiment_settings
-from agentreview.config import AgentConfig
 from agentreview.environments import PaperDecision
 from agentreview.paper_review_arena import PaperReviewArena
-from agentreview.paper_review_player import AreaChair
-from arguments import parse_args
-from agentreview.role_descriptions import get_ac_config
-from utility.utils import project_setup, get_paper_decision_mapping, \
-    load_metareview, load_gpt4_generated_ac_decisions
+from agentreview.arguments import parse_args
+from agentreview.utility.utils import project_setup, get_paper_decision_mapping, \
+    load_metareview, load_llm_ac_decisions
 
 # Set up logging configuration
 logging.basicConfig(
@@ -27,6 +25,8 @@ logging.basicConfig(
     ]
 )
 
+logger = logging.getLogger(__name__)
+
 
 def main(args):
     """
@@ -46,18 +46,16 @@ def main(args):
     NUM_PAPERS = len(const.year2paper_ids[args.conference])
     order = np.random.choice(range(NUM_PAPERS), size=NUM_PAPERS, replace=False)
 
-    metareviews = []
-
     # Paper IDs we actually used in experiments
     experimental_paper_ids = []
 
     # For papers that have not been decided yet, load their metareviews
-
+    metareviews = []
     print("Shuffling paper IDs")
     sampled_paper_ids = np.array(const.year2paper_ids[args.conference])[order]
 
     # Exclude papers that already have AC decisions
-    existing_ac_decisions = load_gpt4_generated_ac_decisions(output_dir=args.output_dir,
+    existing_ac_decisions = load_llm_ac_decisions(output_dir=args.output_dir,
                                                              conference=args.conference,
                                                              model_name=args.model_name,
                                                              ac_scoring_method=args.ac_scoring_method,
@@ -68,65 +66,62 @@ def main(args):
 
     sampled_paper_ids = [paper_id for paper_id in sampled_paper_ids if paper_id not in existing_ac_decisions]
 
-    print("TODO: set paper_ids to existing values")
+    experiment_setting = get_experiment_settings(paper_id=None, paper_decision=None, setting=all_settings[
+        args.experiment_name])
 
-    sampled_paper_ids = [396, 729, 816]
+    logger.info(f"Loading metareview!")
 
     for paper_id in sampled_paper_ids:
 
-        experiment_setting = get_experiment_settings(all_settings[args.experiment_name])
-
         # Load meta-reviews
         metareview = load_metareview(output_dir=args.output_dir, paper_id=paper_id,
                                      experiment_name=args.experiment_name,
                                      model_name=args.model_name, conference=args.conference)
 
         if metareview is None:
+            print(f"Metareview for {paper_id} does not exist. This may happen because the conversation is "
+                  f"completely filtered out due to content policy. "
+                  f"Loading the BASELINE metareview...")
 
-            if args.ignore_missing_metareviews:
-
-                print(f"Metareview for {paper_id} does not exist. This may happen because the conversation is "
-                      f"completely filtered out due to content policy. "
-                      f"Loading the BASELINE metareview...")
-
-                metareview = load_metareview(paper_id=paper_id, experiment_name="BASELINE",
-                                             model_name=args.model_name, conference=args.conference)
+            metareview = load_metareview(output_dir=args.output_dir, paper_id=paper_id,
+                                         experiment_name="BASELINE",
+                                         model_name=args.model_name, conference=args.conference)
 
-            else:
-                raise ValueError(f"Metareview for {paper_id} does not exist")
+        if metareview is not None:
 
-        metareviews += [metareview]
-        experimental_paper_ids += [paper_id]
+            metareviews += [metareview]
+            experimental_paper_ids += [paper_id]
 
     num_batches = len(experimental_paper_ids) // args.num_papers_per_area_chair
 
     for batch_index in range(num_batches):
-        experiment_setting["players"] = {k: v for k, v in experiment_setting["players"].items() if k.startswith("AC")}
-
-        players = []
-
-        for role, players_li in experiment_setting["players"].items():
-
-            for i, player_config in enumerate(players_li):
-
-                # This phase should only contain the Area Chair
-                if role == "AC":
-
-                    player_config = get_ac_config(env_type="paper_decision",
-                                                  scoring_method=args.ac_scoring_method,
-                                                  num_papers_per_area_chair=args.num_papers_per_area_chair,
-                                                  global_settings=experiment_setting['global_settings'],
-                                                  acceptance_rate=args.acceptance_rate
-                                                  **player_config)
-
-                    player_config = AgentConfig(**player_config)
-                    player_config['model'] = args.model_name
-                    player = AreaChair(**player_config)
-
-                else:
-                    raise NotImplementedError(f"Unknown role: {role}")
 
-                players.append(player)
+        players = initialize_players(experiment_setting=experiment_setting, args=args)
+
+        # players = []
+        #
+        # for role, players_li in experiment_setting["players"].items():
+        #
+        #     for i, player_config in enumerate(players_li):
+        #
+        #         # This phase should only contain the Area Chair
+        #         if role == "AC":
+        #
+        #             player_config = get_ac_config(env_type="paper_decision",
+        #                                           scoring_method=args.ac_scoring_method,
+        #                                           num_papers_per_area_chair=args.num_papers_per_area_chair,
+        #                                           global_settings=experiment_setting['global_settings'],
+        #                                           acceptance_rate=args.acceptance_rate
+        #                                                           ** player_config)
+        #
+        #             # player_config = AgentConfig(**player_config)
+        #             player_config['model'] = args.model_name
+        #             player = AreaChair(**player_config)
+        #
+        #         else:
+        #             raise NotImplementedError(f"Unknown role: {role}")
+        #
+        #         players.append(player)
 
         player_names = [player.name for player in players]
 
@@ -141,7 +136,7 @@ def main(args):
                             metareviews=metareviews,
                             experiment_setting=experiment_setting, ac_scoring_method=args.ac_scoring_method)
 
-        arena = PaperReviewArena(players=players, environment=env, args=args)
+        arena = PaperReviewArena(players=players, environment=env, args=args, global_prompt=const.GLOBAL_PROMPT)
         arena.launch_cli(interactive=False)
 
 

run_paper_review_cli.py

@@ -4,18 +4,17 @@ import os
 import sys
 from argparse import Namespace
 
+
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
-from arguments import parse_args
+from agentreview import const
+from agentreview.arguments import parse_args
 from agentreview.experiment_config import all_settings
-from agentreview.agent import Player
 from agentreview.environments import PaperReview
 from agentreview.paper_review_settings import get_experiment_settings
 from agentreview.paper_review_arena import PaperReviewArena
-from agentreview.paper_review_player import PaperExtractorPlayer, AreaChair, Reviewer
-from agentreview.role_descriptions import get_ac_config, get_reviewer_player_config, get_author_config, \
-    get_paper_extractor_config
-from utility.utils import project_setup, get_paper_decision_mapping
+from agentreview.utility.experiment_utils import initialize_players
+from agentreview.utility.utils import project_setup, get_paper_decision_mapping
 
 # Set up logging configuration
 logging.basicConfig(
@@ -53,81 +52,84 @@ def main(args: Namespace):
     sampled_paper_ids = [int(os.path.basename(p).split(".pdf")[0]) for p in paper_paths if p.endswith(".pdf")]
 
     for paper_id in sampled_paper_ids:
-
-        experiment_setting = get_experiment_settings(all_settings[args.experiment_name])
-
         # Ground-truth decision in the conference.
         # We use this to partition the papers into different quality.
         paper_decision = paper_id2decision[paper_id]
 
-        logger.info(f"Experiment Started")
-        logger.info(f"Paper ID: {paper_id} ({paper_decision})")
-
-        player_names, players = [], []
-
-        for role, players_list in experiment_setting["players"].items():
-
-            for i, player_config in enumerate(players_list):
-                if role == "Paper Extractor":
-
-                    player_config = get_paper_extractor_config(global_settings=experiment_setting['global_settings'], )
-
-                    player = PaperExtractorPlayer(data_dir=args.data_dir, paper_id=paper_id,
-                                                  paper_decision=paper_decision,
-                                                  args=args,
-                                                  conference=args.conference, **player_config)
-
-                    player_names.append(player.name)
-
-
-                elif role == "AC":
-
-                    player_config = get_ac_config(env_type="paper_review",
-                                                  scoring_method=args.ac_scoring_method,
-                                                  num_papers_per_area_chair=args.num_papers_per_area_chair,
-                                                  global_settings=experiment_setting['global_settings'],
-                                                  acceptance_rate=args.acceptance_rate,
-                                                  **player_config)
-
-                    player_config['model'] = args.model_name
-
-                    player = AreaChair(data_dir=args.data_dir,
-                                       conference=args.conference,
-                                       args=args,
-                                       **player_config)
-
-                    player_names.append(player.name)
-
-
-                elif role == "Author":
-
-                    # Author requires no behavior customization.
-                    # So we directly use the Player class
-                    player_config = get_author_config()
-                    player = Player(data_dir=args.data_dir,
-                                    conference=args.conference,
-                                    args=args,
-                                    **player_config)
-
-                    player_names.append(player.name)
-
-                elif role == "Reviewer":
-                    player_config = get_reviewer_player_config(reviewer_index=i + 1,
-                                                               global_settings=experiment_setting['global_settings'],
-                                                               **player_config)
-                    player_config['model'] = args.model_name
-                    player = Reviewer(data_dir=args.data_dir, conference=args.conference, **player_config)
-                    player_names.append(player.name)
-
-                else:
-                    raise NotImplementedError(f"Unknown role: {role}")
-
-                players.append(player)
+        experiment_setting = get_experiment_settings(paper_id=paper_id,
+                                                     paper_decision=paper_decision,
+                                                     setting=all_settings[args.experiment_name])
+
+        logger.info(f"Experiment Started!")
+        logger.info(f"Paper ID: {paper_id} (Decision in {args.conference}: {paper_decision})")
+
+        players = initialize_players(experiment_setting=experiment_setting, args=args)
+
+        player_names = [player.name for player in players]
+
+        # for role, players_list in experiment_setting["players"].items():
+        #
+        #     for i, player_config in enumerate(players_list):
+        #         if role == "Paper Extractor":
+        #
+        #             player_config = get_paper_extractor_config(global_settings=experiment_setting['global_settings'], )
+        #
+        #             player = PaperExtractorPlayer(data_dir=args.data_dir, paper_id=paper_id,
+        #                                           paper_decision=paper_decision,
+        #                                           args=args,
+        #                                           conference=args.conference, **player_config)
+        #
+        #             player_names.append(player.name)
+        #
+        #
+        #         elif role == "AC":
+        #
+        #             player_config = get_ac_config(env_type="paper_review",
+        #                                           scoring_method=args.ac_scoring_method,
+        #                                           num_papers_per_area_chair=args.num_papers_per_area_chair,
+        #                                           global_settings=experiment_setting['global_settings'],
+        #                                           acceptance_rate=args.acceptance_rate,
+        #                                           **player_config)
+        #
+        #             player_config['model'] = args.model_name
+        #
+        #             player = AreaChair(data_dir=args.data_dir,
+        #                                conference=args.conference,
+        #                                args=args,
+        #                                **player_config)
+        #
+        #             player_names.append(player.name)
+        #
+        #
+        #         elif role == "Author":
+        #
+        #             # Author requires no behavior customization.
+        #             # So we directly use the Player class
+        #             player_config = get_author_config()
+        #             player = Player(data_dir=args.data_dir,
+        #                             conference=args.conference,
+        #                             args=args,
+        #                             **player_config)
+        #
+        #             player_names.append(player.name)
+        #
+        #         elif role == "Reviewer":
+        #             player_config = get_reviewer_player_config(reviewer_index=i + 1,
+        #                                                        global_settings=experiment_setting['global_settings'],
+        #                                                        **player_config)
+        #             player_config['model'] = args.model_name
+        #             player = Reviewer(data_dir=args.data_dir, conference=args.conference, **player_config)
+        #             player_names.append(player.name)
+        #
+        #         else:
+        #             raise NotImplementedError(f"Unknown role: {role}")
+        #
+        #         players.append(player)
 
         env = PaperReview(player_names=player_names, paper_decision=paper_decision, paper_id=paper_id,
                           args=args, experiment_setting=experiment_setting)
 
-        arena = PaperReviewArena(players=players, environment=env, args=args)
+        arena = PaperReviewArena(players=players, environment=env, args=args, global_prompt=const.GLOBAL_PROMPT)
         arena.launch_cli(interactive=False)
 
     logger.info("Done!")