Replaces all gpt-3.5 with gpt-4o

src/openfactcheck/solvers/factcheckgpt/factcheckgpt_cp.py

@@ -5,14 +5,20 @@ from openfactcheck import FactCheckerState, StandardTaskSolver, Solver
 
 from .factcheckgpt_utils.openai_api import gpt
 from .factcheckgpt_utils.data_util import save_to_file
-from .factcheckgpt_utils.prompt import DOC_TO_INDEPEDENT_SENTENCES_PROMPT, SENTENCES_TO_CLAIMS_PROMPT, \
-    DOC_TO_SENTENCES_PROMPT, CHECKWORTHY_PROMPT_BOOL, SPECIFY_CHECKWORTHY_CATEGORY_PROMPT
+from .factcheckgpt_utils.prompt import (
+    DOC_TO_INDEPEDENT_SENTENCES_PROMPT,
+    SENTENCES_TO_CLAIMS_PROMPT,
+    DOC_TO_SENTENCES_PROMPT,
+    CHECKWORTHY_PROMPT_BOOL,
+    SPECIFY_CHECKWORTHY_CATEGORY_PROMPT,
+)
+
 
 @Solver.register("factcheckgpt_claimprocessor", "response", "claims")
 class FactCheckGPTClaimProcessor(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("factcheckgpt_model", "gpt-3.5-turbo")
+        self.model = self.global_config.get("factcheckgpt_model", "gpt-4o")
         self.num_retries = self.global_config.get("num_retries", 3)
         self.mode = args.get("mode", "independent_sentences")
         self.decompose_system_role = "You are good at decomposing and decontextualizing text."
@@ -22,19 +28,19 @@ class FactCheckGPTClaimProcessor(StandardTaskSolver):
         self.prompt = {
             "sentences": DOC_TO_SENTENCES_PROMPT,
             "independent_sentences": DOC_TO_INDEPEDENT_SENTENCES_PROMPT,
-            "claims": SENTENCES_TO_CLAIMS_PROMPT
+            "claims": SENTENCES_TO_CLAIMS_PROMPT,
         }.get(self.mode, DOC_TO_INDEPEDENT_SENTENCES_PROMPT)
         nlp = spacy.load(self.spacy_model)
         self.rule_based_tool = {
             "nltk": lambda x: [x.strip() for x in nltk.sent_tokenize(x) if len(x.strip()) >= 3],
-            "spacy": lambda x: [x.text.strip() for x in nlp(x).sents if len(x.text.strip()) >= 3]
+            "spacy": lambda x: [x.text.strip() for x in nlp(x).sents if len(x.text.strip()) >= 3],
         }.get(self.rule_based_method, "nltk")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         # We have merged the text decomposer and worthiness filter here.
         response = state.get(self.input_name)
         claims = [response]
-        
+
         user_input = self.prompt.format(doc=response).strip()
         r = gpt(user_input, model=self.model, system_role=self.decompose_system_role, num_retries=self.num_retries)
         try:
@@ -45,13 +51,15 @@ class FactCheckGPTClaimProcessor(StandardTaskSolver):
 
         if not isinstance(claims, list):
             print(
-                f"{self.model} output {r}. It does not output a list of sentences correctly, return rule-based split results.")
+                f"{self.model} output {r}. It does not output a list of sentences correctly, return rule-based split results."
+            )
             claims = self.rule_based_tool(response)
-            
+
         worthiness = [True] * len(claims)
         user_input = CHECKWORTHY_PROMPT_BOOL.format(claims=claims)
-        response = gpt(user_input, model=self.model, system_role=self.worthines_filter_system_role,
-                       num_retries=self.num_retries)
+        response = gpt(
+            user_input, model=self.model, system_role=self.worthines_filter_system_role, num_retries=self.num_retries
+        )
         # TODO refine check worthiness prompt, value returned not reasonable.
         try:
             worthiness = eval(response)

src/openfactcheck/solvers/factcheckgpt/factcheckgpt_rtv.py

@@ -17,16 +17,16 @@ from .factcheckgpt_utils.openai_api import gpt
 from .factcheckgpt_utils.prompt import QGEN_PROMPT, QGEN_PROMPT_FMT
 from .factcheckgpt_utils.data_util import save_txt, save_json
 
+
 @Solver.register("factcheckgpt_retriever", "claims", "claims_with_evidences")
 class FactCheckGPTRetriever(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("factcheckgpt_model", "gpt-3.5-turbo")
+        self.model = self.global_config.get("factcheckgpt_model", "gpt-4o")
         self.num_retries = self.global_config.get("num_retries", 3)
         self.tokenizer = spacy.load("en_core_web_sm", disable=["ner", "tagger", "lemmatizer"])
         self.question_duplicate_model = CrossEncoder(
-            'navteca/quora-roberta-base',
-            device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+            "navteca/quora-roberta-base", device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
         )
         self.passage_ranker = CrossEncoder(
             "cross-encoder/ms-marco-MiniLM-L-6-v2",
@@ -44,7 +44,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         self.sentences_per_passage = args.get("sentences_per_passage", 5)
         self.max_passages_per_question = args.get("max_passages_per_question", 5)
         self.max_aggregated_evidences = args.get("max_aggregated_evidences", 5)
-        self.question_persist_path = args.get("question_persist_path", 'questions.txt')
+        self.question_persist_path = args.get("question_persist_path", "questions.txt")
         self.snippets_persist_path = args.get("snippets_persist_path", "passage.json")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
@@ -52,7 +52,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         claims_with_evidences = {}
         for i, claim in enumerate(claims):
             evidences = self.get_web_evidences_for_claim(claim)
-            claims_with_evidences[claim] = [(q, e['text']) for q, e in evidences['aggregated']]
+            claims_with_evidences[claim] = [(q, e["text"]) for q, e in evidences["aggregated"]]
         state.set(self.output_name, claims_with_evidences)
         return True, state
 
@@ -69,11 +69,9 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         snippets = {}
         for question in questions:
             retrieved_passages = self.get_relevant_snippets(question)
-            snippets[question] = sorted(
-                retrieved_passages,
-                key=lambda x: x['retrieval_score'],
-                reverse=True
-            )[:self.max_passages_per_question]
+            snippets[question] = sorted(retrieved_passages, key=lambda x: x["retrieval_score"], reverse=True)[
+                : self.max_passages_per_question
+            ]
         save_json(snippets, self.snippets_persist_path)
         return snippets
 
@@ -110,7 +108,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
                 model=self.model,
                 system_role=self.qgen_system_role,
                 num_retries=self.num_retries,
-                temperature=self.qgen_temp
+                temperature=self.qgen_temp,
             )
             try:
                 cur_round_questions = set(eval(response))
@@ -182,8 +180,8 @@ class FactCheckGPTRetriever(StandardTaskSolver):
             return False
         return True
 
-    def search_google(self, query: str, num_web_pages: int = 10, timeout: int = 6, save_url: str = '') -> list[str]:
-        """Searches the query using Google. 
+    def search_google(self, query: str, num_web_pages: int = 10, timeout: int = 6, save_url: str = "") -> list[str]:
+        """Searches the query using Google.
         Args:
             query: Search query.
             num_web_pages: the number of web pages to request.
@@ -198,7 +196,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0"
         # mobile user-agent
         MOBILE_USER_AGENT = "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36"
-        headers = {'User-Agent': USER_AGENT}
+        headers = {"User-Agent": USER_AGENT}
 
         # set language
         # set the Google interface language, use &hl=XX
@@ -222,18 +220,18 @@ class FactCheckGPTRetriever(StandardTaskSolver):
 
         # save all url into a txt file
         if not save_url == "":
-            with open(save_url, 'w') as file:
+            with open(save_url, "w") as file:
                 for url in urls:
-                    file.write(url + '\n')
+                    file.write(url + "\n")
         return urls
 
     def chunk_text(
-            self,
-            text: str,
-            tokenizer,
-            sentences_per_passage: int = 5,
-            filter_sentence_len: int = 250,
-            sliding_distance: int = 2,
+        self,
+        text: str,
+        tokenizer,
+        sentences_per_passage: int = 5,
+        filter_sentence_len: int = 250,
+        sliding_distance: int = 2,
     ) -> list[str]:
         """Chunks text into passages using a sliding window.
 
@@ -260,15 +258,16 @@ class FactCheckGPTRetriever(StandardTaskSolver):
             ]
             for idx in range(0, len(sents), sliding_distance):
                 passages.append(
-                    (" ".join(sents[idx: idx + sentences_per_passage]), idx, idx + sentences_per_passage - 1))
+                    (" ".join(sents[idx : idx + sentences_per_passage]), idx, idx + sentences_per_passage - 1)
+                )
         except UnicodeEncodeError as _:  # Sometimes run into Unicode error when tokenizing.
             print("Unicode error when using Spacy. Skipping text.")
 
         return passages
 
     def get_relevant_snippets(
-            self,
-            query,
+        self,
+        query,
     ):
         search_results = self.search_google(query, timeout=self.search_timeout)
 
@@ -278,11 +277,9 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         scraped_results = [r for r in scraped_results if r[0] and ".pdf" not in r[1]]
         # print("Num Bing Search Results: ", len(scraped_results))
         retrieved_passages = list()
-        for webtext, url in scraped_results[:self.max_search_results_per_query]:
+        for webtext, url in scraped_results[: self.max_search_results_per_query]:
             passages = self.chunk_text(
-                text=webtext,
-                tokenizer=self.tokenizer,
-                sentences_per_passage=self.sentences_per_passage
+                text=webtext, tokenizer=self.tokenizer, sentences_per_passage=self.sentences_per_passage
             )
             if not passages:
                 continue
@@ -304,7 +301,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
                             overlap = True
                             break
 
-                # Only consider top non-overlapping relevant passages to maximise for information 
+                # Only consider top non-overlapping relevant passages to maximise for information
                 if not overlap:
                     relevant_items.append(deepcopy(passage_item))
                     retrieved_passages.append(

src/openfactcheck/solvers/factcheckgpt/factcheckgpt_vfr.py

@@ -9,24 +9,22 @@ from .factcheckgpt_utils.data_util import save_to_file
 from .factcheckgpt_utils.prompt import IDENTIFY_STANCE_PROMPT, IDENTIFY_STANCE_PROMPT_FUNC
 from .factcheckgpt_utils.nli import nli_infer
 
+
 @Solver.register("factcheckgpt_verifier", "claims_with_evidences", "label")
 class FactCheckGPTVerifier(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.stance_model = args.get("stance_model", "gpt-3.5-turbo")
+        self.stance_model = args.get("stance_model", "gpt-4o")
         self.num_retries = self.global_config.get("num_retries", 3)
         # self.system_role = args.get("system_role", "You are a helpful factchecker assistant.")
         self.system_role = "You are a helpful factchecker assistant."
         self.verify_retries = args.get("verify_retries", 3)
-        self.stance_map = {
-            1: "support",
-            -1: "refute",
-            0: "irrelevant"
-        }
+        self.stance_map = {1: "support", -1: "refute", 0: "irrelevant"}
 
     def verify_by_stance(
-            self, claim: str,
-            evidences: list[str],
+        self,
+        claim: str,
+        evidences: list[str],
     ) -> Any:
         labels = []
         for evidence in evidences:
@@ -45,12 +43,7 @@ class FactCheckGPTVerifier(StandardTaskSolver):
 
     def identify_stance_gpt(self, evidence, claim):
         user_input = IDENTIFY_STANCE_PROMPT_FUNC.format(claim=claim, evidence=evidence)
-        r = gpt(
-            user_input,
-            model=self.stance_model,
-            system_role=self.system_role,
-            num_retries=self.num_retries
-        )
+        r = gpt(user_input, model=self.stance_model, system_role=self.system_role, num_retries=self.num_retries)
         label = 0
         try:
             label = eval(r)
@@ -58,9 +51,9 @@ class FactCheckGPTVerifier(StandardTaskSolver):
             print(f"An unexpected error occurred: {e}.")
         return label
 
-    def stance(self, evidence, claim, model="gpt-3.5-turbo"):
+    def stance(self, evidence, claim, model="gpt-4o"):
         """input: a claim and an evidence
-           output: label in [support, refute, irrelevant]"""
+        output: label in [support, refute, irrelevant]"""
         label = 0
         if self.stance_model == "nli":
             label = nli_infer(premise=evidence, hypothesis=claim)
@@ -73,7 +66,7 @@ class FactCheckGPTVerifier(StandardTaskSolver):
     def verify_claim(self, claim: str, evidences: list[str]) -> dict[str, Any]:
         results = None
         user_input = VERIFY_PROMPT.format(claim=claim, evidence=evidences)
-        r = ''
+        r = ""
         for _ in range(self.verify_retries):
             r = gpt(
                 user_input,
@@ -97,12 +90,7 @@ class FactCheckGPTVerifier(StandardTaskSolver):
         else:
             print(f"Error output {r}. It does not output a dict, return factual label by stance aggregation.")
             factual_label = self.verify_by_stance(claim, evidences)
-            results = {
-                "reasoning": "",
-                "error": "",
-                "correction": "",
-                "factuality": factual_label
-            }
+            results = {"reasoning": "", "error": "", "correction": "", "factuality": factual_label}
             return results
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
@@ -113,6 +101,6 @@ class FactCheckGPTVerifier(StandardTaskSolver):
             result["claim"] = claim
             result["evidences"] = evidences
             results.append(result)
-        state.set(self.output_name, all([x['factuality'] > 0 for x in results]))
+        state.set(self.output_name, all([x["factuality"] > 0 for x in results]))
         state.set("detail", results)
         return True, state

src/openfactcheck/solvers/factool/factool_utils/chat_api.py

@@ -15,56 +15,57 @@ import openai
 from openai import OpenAI, AsyncOpenAI
 import re
 
-class OpenAIChat():
+
+class OpenAIChat:
     def __init__(
-            self,
-            model_name='gpt-3.5-turbo',
-            max_tokens=2500,
-            temperature=0,
-            top_p=1,
-            request_timeout=120,
+        self,
+        model_name="gpt-4o",
+        max_tokens=2500,
+        temperature=0,
+        top_p=1,
+        request_timeout=120,
     ):
-        if 'gpt' not in model_name:
+        if "gpt" not in model_name:
             openai.api_base = "http://localhost:8000/v1"
         else:
-            #openai.api_base = "https://api.openai.com/v1"
+            # openai.api_base = "https://api.openai.com/v1"
             openai.api_key = os.environ.get("OPENAI_API_KEY", None)
             assert openai.api_key is not None, "Please set the OPENAI_API_KEY environment variable."
-            assert openai.api_key !='', "Please set the OPENAI_API_KEY environment variable."
+            assert openai.api_key != "", "Please set the OPENAI_API_KEY environment variable."
         self.client = AsyncOpenAI()
 
         self.config = {
-            'model_name': model_name,
-            'max_tokens': max_tokens,
-            'temperature': temperature,
-            'top_p': top_p,
-            'request_timeout': request_timeout,
+            "model_name": model_name,
+            "max_tokens": max_tokens,
+            "temperature": temperature,
+            "top_p": top_p,
+            "request_timeout": request_timeout,
         }
 
     def extract_list_from_string(self, input_string):
-        # pattern = r'\[.*\]'  
+        # pattern = r'\[.*\]'
         # result = re.search(pattern, input_string)
         # if result:
         #     return result.group()
         # else:
         #     return None
-        start_index = input_string.find('[')  
-        end_index = input_string.rfind(']') 
+        start_index = input_string.find("[")
+        end_index = input_string.rfind("]")
 
         if start_index != -1 and end_index != -1 and start_index < end_index:
-            return input_string[start_index:end_index + 1]
+            return input_string[start_index : end_index + 1]
         else:
             return None
-        
+
     def extract_dict_from_string(self, input_string):
-        start_index = input_string.find('{')
-        end_index = input_string.rfind('}')
+        start_index = input_string.find("{")
+        end_index = input_string.rfind("}")
 
         if start_index != -1 and end_index != -1 and start_index < end_index:
-            return input_string[start_index:end_index + 1]
+            return input_string[start_index : end_index + 1]
         else:
             return None
-    
+
     def _boolean_fix(self, output):
         return output.replace("true", "True").replace("false", "False")
 
@@ -75,7 +76,7 @@ class OpenAIChat():
                 return None
             return output_eval
         except:
-            '''
+            """
             if(expected_type == List):
                 valid_output = self.extract_list_from_string(output)
                 output_eval = ast.literal_eval(valid_output)
@@ -88,46 +89,47 @@ class OpenAIChat():
                 if not isinstance(output_eval, expected_type):
                     return None
                 return output_eval
-            '''
+            """
             return None
 
-    async def dispatch_openai_requests(self, messages_list,) -> list[str]:
+    async def dispatch_openai_requests(
+        self,
+        messages_list,
+    ) -> list[str]:
         """
         Dispatches requests to OpenAI API asynchronously.
-        
+
         Args:
             messages_list: List of messages to be sent to OpenAI ChatCompletion API.
         Returns:
             List of responses from OpenAI API.
         """
+
         async def _request_with_retry(messages, retry=3):
             for attempt in range(retry):
                 try:
                     response = await self.client.chat.completions.create(
-                        model=self.config['model_name'],
+                        model=self.config["model_name"],
                         messages=messages,
-                        max_tokens=self.config['max_tokens'],
-                        temperature=self.config['temperature'],
-                        top_p=self.config['top_p']
+                        max_tokens=self.config["max_tokens"],
+                        temperature=self.config["temperature"],
+                        top_p=self.config["top_p"],
                     )
                     return response
                 except openai.RateLimitError as e:
-                    await asyncio.sleep((2 ** attempt) * 0.5)  # exponential backoff
+                    await asyncio.sleep((2**attempt) * 0.5)  # exponential backoff
                 except (openai.Timeout, openai.APIError) as e:
-                    await asyncio.sleep((2 ** attempt) * 0.5)  # exponential backoff
+                    await asyncio.sleep((2**attempt) * 0.5)  # exponential backoff
                 except Exception as e:
                     # Log unexpected exception for further investigation
-                    await asyncio.sleep((2 ** attempt) * 0.5)  # fallback in case of unknown errors
-                    
+                    await asyncio.sleep((2**attempt) * 0.5)  # fallback in case of unknown errors
+
             raise RuntimeError("All retries failed for OpenAI API request")
 
-        async_responses = [
-            _request_with_retry(messages)
-            for messages in messages_list
-        ]
+        async_responses = [_request_with_retry(messages) for messages in messages_list]
 
         return await asyncio.gather(*async_responses, return_exceptions=True)
-    
+
     def run(self, messages_list, expected_type):
         retry = 1
         responses = [None for _ in range(len(messages_list))]
@@ -135,24 +137,32 @@ class OpenAIChat():
 
         while retry > 0 and len(messages_list_cur_index) > 0:
             messages_list_cur = [messages_list[i] for i in messages_list_cur_index]
-            
-            predictions = asyncio.run(self.dispatch_openai_requests(
-                messages_list=messages_list_cur,
-            ))
 
-            preds = [self._type_check(self._boolean_fix(prediction.choices[0].message.content), expected_type) if prediction is not None else None for prediction in predictions]
+            predictions = asyncio.run(
+                self.dispatch_openai_requests(
+                    messages_list=messages_list_cur,
+                )
+            )
+
+            preds = [
+                self._type_check(self._boolean_fix(prediction.choices[0].message.content), expected_type)
+                if prediction is not None
+                else None
+                for prediction in predictions
+            ]
             finised_index = []
             for i, pred in enumerate(preds):
                 if pred is not None:
                     responses[messages_list_cur_index[i]] = pred
                     finised_index.append(messages_list_cur_index[i])
-            
+
             messages_list_cur_index = [i for i in messages_list_cur_index if i not in finised_index]
-            
+
             retry -= 1
-        
+
         return responses
 
+
 # class OpenAIEmbed():
 #     def __init__():
 #         openai.api_key = os.environ.get("OPENAI_API_KEY", None)
@@ -190,9 +200,9 @@ class OpenAIChat():
 #     ))
 
 #     print(predictions)
-    # Usage
-    # embed = OpenAIEmbed()
-    # batch = ["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10"]  # Your batch of strings
-    # embeddings = asyncio.run(embed.process_batch(batch, retry=3))
-    # for embedding in embeddings:
-    #     print(embedding["data"][0]["embedding"])
+# Usage
+# embed = OpenAIEmbed()
+# batch = ["string1", "string2", "string3", "string4", "string5", "string6", "string7", "string8", "string9", "string10"]  # Your batch of strings
+# embeddings = asyncio.run(embed.process_batch(batch, retry=3))
+# for embedding in embeddings:
+#     print(embedding["data"][0]["embedding"])

src/openfactcheck/solvers/rarr/rarr_agreement_gate.py

@@ -3,32 +3,34 @@ from .prompts import rarr_prompts
 
 from openfactcheck import FactCheckerState, StandardTaskSolver, Solver
 
+
 @Solver.register("rarr_agreement_gate", "claims_with_evidences", "claims_with_gates")
 class RARRAgreementGate(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
         self.max_evidences_per_question = args.get("max_evidences_per_question", 1)
-        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
+        self.model = self.global_config.get("model", "gpt-4o-instruct")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         claims = state.get(self.input_name)
 
         for claim, contents in claims.items():
             context = contents.get("context", None)
-            evidences = contents.get("evidences", [])[:self.max_evidences_per_question]
+            evidences = contents.get("evidences", [])[: self.max_evidences_per_question]
             gates = []
             for evidence in evidences:
                 gate = agreement_gate.run_agreement_gate(
                     claim=claim,
                     context=context,
-                    query=evidence['query'],
-                    evidence=evidence['text'],
+                    query=evidence["query"],
+                    evidence=evidence["text"],
                     model=self.model,
                     prompt=rarr_prompts.CONTEXTUAL_AGREEMENT_GATE_PROMPT
-                    if context else rarr_prompts.AGREEMENT_GATE_PROMPT
+                    if context
+                    else rarr_prompts.AGREEMENT_GATE_PROMPT,
                 )
                 gates.append(gate)
-            contents['gates'] = gates
+            contents["gates"] = gates
 
         state.set(self.output_name, claims)
         return True, state

src/openfactcheck/solvers/rarr/rarr_editor.py

@@ -5,12 +5,13 @@ from .prompts import rarr_prompts
 
 from openfactcheck import FactCheckerState, StandardTaskSolver, Solver
 
+
 @Solver.register("rarr_editor", "claims_with_evidences", "revised_claims")
 class RARREditor(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
-        # self.model = args.get("model", "gpt-3.5-turbo-instruct")
+        self.model = self.global_config.get("model", "gpt-4o-instruct")
+        # self.model = args.get("model", "gpt-4o-instruct")
         self.max_evidences_per_question = args.get("max_evidences_per_question", 1)
         self.max_edit_ratio = args.get("max_edit_ratio", 100)
         self.output_claim_only = args.get("output_claim_only", False)
@@ -20,7 +21,7 @@ class RARREditor(StandardTaskSolver):
         final_result = {}
         for claim, contents in claims.items():
             context = contents.get("context", None)
-            evidences = contents.get("evidences", [])[:self.max_evidences_per_question]
+            evidences = contents.get("evidences", [])[: self.max_evidences_per_question]
             agreement_gates = []
             revision_steps = []
             claim_for_iterative_revision = claim
@@ -28,32 +29,31 @@ class RARREditor(StandardTaskSolver):
                 gate = agreement_gate.run_agreement_gate(
                     claim=claim_for_iterative_revision,
                     context=context,
-                    query=evidence['query'],
-                    evidence=evidence['text'],
+                    query=evidence["query"],
+                    evidence=evidence["text"],
                     model=self.model,
                     prompt=rarr_prompts.CONTEXTUAL_AGREEMENT_GATE_PROMPT
-                    if context else rarr_prompts.AGREEMENT_GATE_PROMPT
+                    if context
+                    else rarr_prompts.AGREEMENT_GATE_PROMPT,
                 )
                 agreement_gates.append(gate)
 
-                if gate['is_open']:
+                if gate["is_open"]:
                     edited_claim = editor.run_rarr_editor(
                         claim=claim_for_iterative_revision,
                         context=context,
-                        query=evidence['query'],
-                        evidence=evidence['text'],
+                        query=evidence["query"],
+                        evidence=evidence["text"],
                         model=self.model,
-                        prompt=rarr_prompts.CONTEXTUAL_EDITOR_PROMPT
-                        if context
-                        else rarr_prompts.EDITOR_PROMPT,
-                    )['text']
+                        prompt=rarr_prompts.CONTEXTUAL_EDITOR_PROMPT if context else rarr_prompts.EDITOR_PROMPT,
+                    )["text"]
                     if Levenshtein.distance(claim, edited_claim) / len(claim) <= self.max_edit_ratio:
                         claim_for_iterative_revision = edited_claim
                 revision_steps.append({"text": claim_for_iterative_revision})
             result = {
                 "context": context,
                 "text": claim,
-                "questions": contents['questions'],
+                "questions": contents["questions"],
                 "evidences_for_questions": evidences,
                 "revisions": [
                     {
@@ -66,7 +66,7 @@ class RARREditor(StandardTaskSolver):
                 ],
             }
             selected_evidences = evidence_selection.select_evidences(result)
-            result['selected_evidences'] = selected_evidences
-            final_result[claim] = result['revisions'][0]['revised_text'] if self.output_claim_only else result
+            result["selected_evidences"] = selected_evidences
+            final_result[claim] = result["revisions"][0]["revised_text"] if self.output_claim_only else result
         state.set(self.output_name, final_result)
         return True, state

src/openfactcheck/solvers/rarr/rarr_llm_retriever.py

@@ -3,11 +3,12 @@ from .prompts.hallucination_prompts import EVIDENCE_HALLUCINATION
 
 from openfactcheck import FactCheckerState, StandardTaskSolver, Solver
 
+
 @Solver.register("llm_retriever", "claims_with_questions", "claims_with_evidences")
 class RARRLLMRetriever(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
+        self.model = self.global_config.get("model", "gpt-4o-instruct")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         claims = state.get(self.input_name)
@@ -16,14 +17,8 @@ class RARRLLMRetriever(StandardTaskSolver):
             questions = contents.get("questions", [])
             evidences = []
             for question in questions:
-                evidences.append(
-                    run_evidence_hallucination(
-                        question,
-                        model=self.model,
-                        prompt=EVIDENCE_HALLUCINATION
-                    )
-                )
-            claims[claim]['evidences'] = evidences
+                evidences.append(run_evidence_hallucination(question, model=self.model, prompt=EVIDENCE_HALLUCINATION))
+            claims[claim]["evidences"] = evidences
 
         state.set(self.output_name, claims)
         return True, state

src/openfactcheck/solvers/rarr/rarr_question_generator.py

@@ -3,11 +3,12 @@ from .prompts import rarr_prompts
 
 from openfactcheck import FactCheckerState, StandardTaskSolver, Solver
 
+
 @Solver.register("rarr_question_generator", "claims_with_context", "claims_with_questions")
 class RARRQuestionGenerator(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("model", "gpt-3.5-turbo-instruct")
+        self.model = self.global_config.get("model", "gpt-4o-instruct")
         self.temperature_qgen = args.get("temperature_qgen", 0.7)
         self.num_rounds_qgen = args.get("num_rounds_qgen", 3)
 
@@ -18,13 +19,11 @@ class RARRQuestionGenerator(StandardTaskSolver):
             claims = {c: dict() for c in claims}
         for claim, contents in claims.items():
             context = contents.get("context", None)
-            claims[claim]['questions'] = run_rarr_question_generation(
+            claims[claim]["questions"] = run_rarr_question_generation(
                 claim=claim,
                 context=context,
                 model=self.model,
-                prompt=rarr_prompts.CONTEXTUAL_QGEN_PROMPT
-                if context
-                else rarr_prompts.QGEN_PROMPT,
+                prompt=rarr_prompts.CONTEXTUAL_QGEN_PROMPT if context else rarr_prompts.QGEN_PROMPT,
                 temperature=self.temperature_qgen,
                 num_rounds=self.num_rounds_qgen,
             )

src/openfactcheck/solvers/tutorial/utils/api.py

@@ -9,32 +9,36 @@ from typing import Any, Dict, List, Tuple
 # OpenAI ChatGPT and davicci-text
 # ----------------------------------------------------------
 client = None
+
+
 def init_client():
     global client
     if client is None:
-        if openai.api_key is None and 'OPENAI_API_KEY' not in os.environ:
+        if openai.api_key is None and "OPENAI_API_KEY" not in os.environ:
             print("openai_key not presented, delay to initialize.")
             return
         client = OpenAI()
 
+
 def chatgpt(user_input):
     response = client.chat.completions.create(
-        model="gpt-3.5-turbo",
+        model="gpt-4o",
         messages=[
-                {"role": "system", "content": "You are a NLP expert that is good at fact checking"},
-                {"role": "user", "content": user_input},
-        ]
+            {"role": "system", "content": "You are a NLP expert that is good at fact checking"},
+            {"role": "user", "content": user_input},
+        ],
     )
 
-    result = ''
+    result = ""
     for choice in response.choices:
         result += choice.message.content
 
     return result
 
+
 def davinci(prompt):
     # Set up the model and prompt
-    model_engine = "gpt-3.5-turbo-instruct"
+    model_engine = "gpt-4o-instruct"
 
     # Generate a response
     completion = client.completions.create(
@@ -49,11 +53,13 @@ def davinci(prompt):
     response = completion.choices[0].text
     return response
 
+
 # ----------------------------------------------------------
 # Bing Search
 # ----------------------------------------------------------
 BING_SEARCH_URL = "https://api.bing.microsoft.com/v7.0/search/"
-SUBSCRIPTION_KEY = "" # fill your bing api key
+SUBSCRIPTION_KEY = ""  # fill your bing api key
+
 
 def search_bing(query: str, timeout: float = 3) -> List[str]:
     """Searches the query using Bing.
@@ -63,7 +69,7 @@ def search_bing(query: str, timeout: float = 3) -> List[str]:
     Returns:
         search_results: A list of the top URLs relevant to the query.
     """
-    
+
     headers = {"Ocp-Apim-Subscription-Key": SUBSCRIPTION_KEY}
     params = {"q": query, "textDecorations": True, "textFormat": "HTML"}
     response = requests.get(BING_SEARCH_URL, headers=headers, params=params, timeout=timeout)
@@ -73,7 +79,8 @@ def search_bing(query: str, timeout: float = 3) -> List[str]:
     search_results = [r["url"] for r in response["webPages"]["value"]]
     return search_results
 
-# Test Bing search 
+
+# Test Bing search
 # search_results = search_bing("What are the different awards that Preslav Nakov has received")
 # print(search_results)
 
@@ -81,7 +88,7 @@ def search_bing(query: str, timeout: float = 3) -> List[str]:
 # ----------------------------------------------------------
 # Google Search
 # ----------------------------------------------------------
-def search_google(query: str, num_web_pages: int = 10, save_url: str = '') -> List[str]:
+def search_google(query: str, num_web_pages: int = 10, save_url: str = "") -> List[str]:
     """Searches the query using Google.
     Args:
         query: Search query.
@@ -97,13 +104,13 @@ def search_google(query: str, num_web_pages: int = 10, save_url: str = '') -> Li
     USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0"
     # mobile user-agent
     MOBILE_USER_AGENT = "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36"
-    headers = {'User-Agent': USER_AGENT}
-    
+    headers = {"User-Agent": USER_AGENT}
+
     # set language
     # set the Google interface language, use &hl=XX
     # set the preferred language of the search results, use &lr=lang_XX
     # set language as en, otherwise it will return many translation web pages to Arabic that can't be opened correctly.
-    lang = "en" 
+    lang = "en"
 
     # scrape google results
     urls = []
@@ -121,11 +128,12 @@ def search_google(query: str, num_web_pages: int = 10, save_url: str = '') -> Li
 
     # save all url into a txt file
     if not save_url == "":
-        with open(save_url, 'w') as file:
+        with open(save_url, "w") as file:
             for url in urls:
-                file.write(url + '\n')
+                file.write(url + "\n")
     return urls
 
+
 # Test google search
 # query = "Google Company Introduction"
 # urls = search_google(query)

src/openfactcheck/solvers/webservice/factcheckgpt_cp.py

@@ -6,14 +6,20 @@ from openfactcheck.solver import StandardTaskSolver, Solver
 
 from .factcheckgpt_utils.openai_api import gpt
 from .factcheckgpt_utils.data_util import save_to_file
-from .factcheckgpt_utils.prompt import DOC_TO_INDEPEDENT_SENTENCES_PROMPT, SENTENCES_TO_CLAIMS_PROMPT, \
-    DOC_TO_SENTENCES_PROMPT, CHECKWORTHY_PROMPT_BOOL, SPECIFY_CHECKWORTHY_CATEGORY_PROMPT
+from .factcheckgpt_utils.prompt import (
+    DOC_TO_INDEPEDENT_SENTENCES_PROMPT,
+    SENTENCES_TO_CLAIMS_PROMPT,
+    DOC_TO_SENTENCES_PROMPT,
+    CHECKWORTHY_PROMPT_BOOL,
+    SPECIFY_CHECKWORTHY_CATEGORY_PROMPT,
+)
+
 
 @Solver.register("factcheckgpt_claimprocessor", "response", "claims")
 class FactCheckGPTClaimProcessor(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("factcheckgpt_model", "gpt-3.5-turbo")
+        self.model = self.global_config.get("factcheckgpt_model", "gpt-4o")
         self.num_retries = self.global_config.get("num_retries", 3)
         self.mode = args.get("mode", "independent_sentences")
         self.decompose_system_role = "You are good at decomposing and decontextualizing text."
@@ -23,19 +29,19 @@ class FactCheckGPTClaimProcessor(StandardTaskSolver):
         self.prompt = {
             "sentences": DOC_TO_SENTENCES_PROMPT,
             "independent_sentences": DOC_TO_INDEPEDENT_SENTENCES_PROMPT,
-            "claims": SENTENCES_TO_CLAIMS_PROMPT
+            "claims": SENTENCES_TO_CLAIMS_PROMPT,
         }.get(self.mode, DOC_TO_INDEPEDENT_SENTENCES_PROMPT)
         nlp = spacy.load(self.spacy_model)
         self.rule_based_tool = {
             "nltk": lambda x: [x.strip() for x in nltk.sent_tokenize(x) if len(x.strip()) >= 3],
-            "spacy": lambda x: [x.text.strip() for x in nlp(x).sents if len(x.text.strip()) >= 3]
+            "spacy": lambda x: [x.text.strip() for x in nlp(x).sents if len(x.text.strip()) >= 3],
         }.get(self.rule_based_method, "nltk")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         # We have merged the text decomposer and worthiness filter here.
         response = state.get(self.input_name)
         claims = [response]
-        
+
         user_input = self.prompt.format(doc=response).strip()
         r = gpt(user_input, model=self.model, system_role=self.decompose_system_role, num_retries=self.num_retries)
         try:
@@ -46,13 +52,15 @@ class FactCheckGPTClaimProcessor(StandardTaskSolver):
 
         if not isinstance(claims, list):
             print(
-                f"{self.model} output {r}. It does not output a list of sentences correctly, return rule-based split results.")
+                f"{self.model} output {r}. It does not output a list of sentences correctly, return rule-based split results."
+            )
             claims = self.rule_based_tool(response)
-            
+
         worthiness = [True] * len(claims)
         user_input = CHECKWORTHY_PROMPT_BOOL.format(claims=claims)
-        response = gpt(user_input, model=self.model, system_role=self.worthines_filter_system_role,
-                       num_retries=self.num_retries)
+        response = gpt(
+            user_input, model=self.model, system_role=self.worthines_filter_system_role, num_retries=self.num_retries
+        )
         # TODO refine check worthiness prompt, value returned not reasonable.
         try:
             worthiness = eval(response)

src/openfactcheck/solvers/webservice/factcheckgpt_rtv.py

@@ -18,16 +18,16 @@ from .factcheckgpt_utils.openai_api import gpt
 from .factcheckgpt_utils.prompt import QGEN_PROMPT, QGEN_PROMPT_FMT
 from .factcheckgpt_utils.data_util import save_txt, save_json
 
+
 @Solver.register("factcheckgpt_retriever", "claims", "claims_with_evidences")
 class FactCheckGPTRetriever(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("factcheckgpt_model", "gpt-3.5-turbo")
+        self.model = self.global_config.get("factcheckgpt_model", "gpt-4o")
         self.num_retries = self.global_config.get("num_retries", 3)
         self.tokenizer = spacy.load("en_core_web_sm", disable=["ner", "tagger", "lemmatizer"])
         self.question_duplicate_model = CrossEncoder(
-            'navteca/quora-roberta-base',
-            device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+            "navteca/quora-roberta-base", device=torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
         )
         self.passage_ranker = CrossEncoder(
             "cross-encoder/ms-marco-MiniLM-L-6-v2",
@@ -45,7 +45,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         self.sentences_per_passage = args.get("sentences_per_passage", 5)
         self.max_passages_per_question = args.get("max_passages_per_question", 5)
         self.max_aggregated_evidences = args.get("max_aggregated_evidences", 5)
-        self.question_persist_path = args.get("question_persist_path", 'questions.txt')
+        self.question_persist_path = args.get("question_persist_path", "questions.txt")
         self.snippets_persist_path = args.get("snippets_persist_path", "passage.json")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
@@ -53,7 +53,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         claims_with_evidences = {}
         for i, claim in enumerate(claims):
             evidences = self.get_web_evidences_for_claim(claim)
-            claims_with_evidences[claim] = [(q, e['text']) for q, e in evidences['aggregated']]
+            claims_with_evidences[claim] = [(q, e["text"]) for q, e in evidences["aggregated"]]
         state.set(self.output_name, claims_with_evidences)
         return True, state
 
@@ -70,11 +70,9 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         snippets = {}
         for question in questions:
             retrieved_passages = self.get_relevant_snippets(question)
-            snippets[question] = sorted(
-                retrieved_passages,
-                key=lambda x: x['retrieval_score'],
-                reverse=True
-            )[:self.max_passages_per_question]
+            snippets[question] = sorted(retrieved_passages, key=lambda x: x["retrieval_score"], reverse=True)[
+                : self.max_passages_per_question
+            ]
         save_json(snippets, self.snippets_persist_path)
         return snippets
 
@@ -111,7 +109,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
                 model=self.model,
                 system_role=self.qgen_system_role,
                 num_retries=self.num_retries,
-                temperature=self.qgen_temp
+                temperature=self.qgen_temp,
             )
             try:
                 cur_round_questions = set(eval(response))
@@ -183,8 +181,8 @@ class FactCheckGPTRetriever(StandardTaskSolver):
             return False
         return True
 
-    def search_google(self, query: str, num_web_pages: int = 10, timeout: int = 6, save_url: str = '') -> list[str]:
-        """Searches the query using Google. 
+    def search_google(self, query: str, num_web_pages: int = 10, timeout: int = 6, save_url: str = "") -> list[str]:
+        """Searches the query using Google.
         Args:
             query: Search query.
             num_web_pages: the number of web pages to request.
@@ -199,7 +197,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0"
         # mobile user-agent
         MOBILE_USER_AGENT = "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36"
-        headers = {'User-Agent': USER_AGENT}
+        headers = {"User-Agent": USER_AGENT}
 
         # set language
         # set the Google interface language, use &hl=XX
@@ -223,18 +221,18 @@ class FactCheckGPTRetriever(StandardTaskSolver):
 
         # save all url into a txt file
         if not save_url == "":
-            with open(save_url, 'w') as file:
+            with open(save_url, "w") as file:
                 for url in urls:
-                    file.write(url + '\n')
+                    file.write(url + "\n")
         return urls
 
     def chunk_text(
-            self,
-            text: str,
-            tokenizer,
-            sentences_per_passage: int = 5,
-            filter_sentence_len: int = 250,
-            sliding_distance: int = 2,
+        self,
+        text: str,
+        tokenizer,
+        sentences_per_passage: int = 5,
+        filter_sentence_len: int = 250,
+        sliding_distance: int = 2,
     ) -> list[str]:
         """Chunks text into passages using a sliding window.
 
@@ -261,15 +259,16 @@ class FactCheckGPTRetriever(StandardTaskSolver):
             ]
             for idx in range(0, len(sents), sliding_distance):
                 passages.append(
-                    (" ".join(sents[idx: idx + sentences_per_passage]), idx, idx + sentences_per_passage - 1))
+                    (" ".join(sents[idx : idx + sentences_per_passage]), idx, idx + sentences_per_passage - 1)
+                )
         except UnicodeEncodeError as _:  # Sometimes run into Unicode error when tokenizing.
             print("Unicode error when using Spacy. Skipping text.")
 
         return passages
 
     def get_relevant_snippets(
-            self,
-            query,
+        self,
+        query,
     ):
         search_results = self.search_google(query, timeout=self.search_timeout)
 
@@ -279,11 +278,9 @@ class FactCheckGPTRetriever(StandardTaskSolver):
         scraped_results = [r for r in scraped_results if r[0] and ".pdf" not in r[1]]
         # print("Num Bing Search Results: ", len(scraped_results))
         retrieved_passages = list()
-        for webtext, url in scraped_results[:self.max_search_results_per_query]:
+        for webtext, url in scraped_results[: self.max_search_results_per_query]:
             passages = self.chunk_text(
-                text=webtext,
-                tokenizer=self.tokenizer,
-                sentences_per_passage=self.sentences_per_passage
+                text=webtext, tokenizer=self.tokenizer, sentences_per_passage=self.sentences_per_passage
             )
             if not passages:
                 continue
@@ -305,7 +302,7 @@ class FactCheckGPTRetriever(StandardTaskSolver):
                             overlap = True
                             break
 
-                # Only consider top non-overlapping relevant passages to maximise for information 
+                # Only consider top non-overlapping relevant passages to maximise for information
                 if not overlap:
                     relevant_items.append(deepcopy(passage_item))
                     retrieved_passages.append(

src/openfactcheck/solvers/webservice/factcheckgpt_vfr.py

@@ -10,24 +10,22 @@ from .factcheckgpt_utils.data_util import save_to_file
 from .factcheckgpt_utils.prompt import IDENTIFY_STANCE_PROMPT, IDENTIFY_STANCE_PROMPT_FUNC
 from .factcheckgpt_utils.nli import nli_infer
 
+
 @Solver.register("factcheckgpt_verifier", "claims_with_evidences", "label")
 class FactCheckGPTVerifier(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.stance_model = args.get("stance_model", "gpt-3.5-turbo")
+        self.stance_model = args.get("stance_model", "gpt-4o")
         self.num_retries = self.global_config.get("num_retries", 3)
         # self.system_role = args.get("system_role", "You are a helpful factchecker assistant.")
         self.system_role = "You are a helpful factchecker assistant."
         self.verify_retries = args.get("verify_retries", 3)
-        self.stance_map = {
-            1: "support",
-            -1: "refute",
-            0: "irrelevant"
-        }
+        self.stance_map = {1: "support", -1: "refute", 0: "irrelevant"}
 
     def verify_by_stance(
-            self, claim: str,
-            evidences: list[str],
+        self,
+        claim: str,
+        evidences: list[str],
     ) -> Any:
         labels = []
         for evidence in evidences:
@@ -46,12 +44,7 @@ class FactCheckGPTVerifier(StandardTaskSolver):
 
     def identify_stance_gpt(self, evidence, claim):
         user_input = IDENTIFY_STANCE_PROMPT_FUNC.format(claim=claim, evidence=evidence)
-        r = gpt(
-            user_input,
-            model=self.stance_model,
-            system_role=self.system_role,
-            num_retries=self.num_retries
-        )
+        r = gpt(user_input, model=self.stance_model, system_role=self.system_role, num_retries=self.num_retries)
         label = 0
         try:
             label = eval(r)
@@ -59,9 +52,9 @@ class FactCheckGPTVerifier(StandardTaskSolver):
             print(f"An unexpected error occurred: {e}.")
         return label
 
-    def stance(self, evidence, claim, model="gpt-3.5-turbo"):
+    def stance(self, evidence, claim, model="gpt-4o"):
         """input: a claim and an evidence
-           output: label in [support, refute, irrelevant]"""
+        output: label in [support, refute, irrelevant]"""
         label = 0
         if self.stance_model == "nli":
             label = nli_infer(premise=evidence, hypothesis=claim)
@@ -74,7 +67,7 @@ class FactCheckGPTVerifier(StandardTaskSolver):
     def verify_claim(self, claim: str, evidences: list[str]) -> dict[str, Any]:
         results = None
         user_input = VERIFY_PROMPT.format(claim=claim, evidence=evidences)
-        r = ''
+        r = ""
         for _ in range(self.verify_retries):
             r = gpt(
                 user_input,
@@ -98,12 +91,7 @@ class FactCheckGPTVerifier(StandardTaskSolver):
         else:
             print(f"Error output {r}. It does not output a dict, return factual label by stance aggregation.")
             factual_label = self.verify_by_stance(claim, evidences)
-            results = {
-                "reasoning": "",
-                "error": "",
-                "correction": "",
-                "factuality": factual_label
-            }
+            results = {"reasoning": "", "error": "", "correction": "", "factuality": factual_label}
             return results
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
@@ -114,6 +102,6 @@ class FactCheckGPTVerifier(StandardTaskSolver):
             result["claim"] = claim
             result["evidences"] = evidences
             results.append(result)
-        state.set(self.output_name, all([x['factuality'] > 0 for x in results]))
+        state.set(self.output_name, all([x["factuality"] > 0 for x in results]))
         state.set("detail", results)
         return True, state

src/openfactcheck/solvers/webservice/factool_utils/chat_api.py

@@ -21,53 +21,54 @@ import re
 # env
 # openai.api_key = factool_env_config.openai_api_key
 
-class OpenAIChat():
+
+class OpenAIChat:
     def __init__(
-            self,
-            model_name='gpt-3.5-turbo',
-            max_tokens=2500,
-            temperature=0,
-            top_p=1,
-            request_timeout=120,
+        self,
+        model_name="gpt-4o",
+        max_tokens=2500,
+        temperature=0,
+        top_p=1,
+        request_timeout=120,
     ):
-        if 'gpt' not in model_name:
+        if "gpt" not in model_name:
             openai.api_base = "http://localhost:8000/v1"
         else:
             # openai.api_base = "https://api.openai.com/v1"
             openai.api_key = os.environ.get("OPENAI_API_KEY", None)
             assert openai.api_key is not None, "Please set the OPENAI_API_KEY environment variable."
-            assert openai.api_key != '', "Please set the OPENAI_API_KEY environment variable."
+            assert openai.api_key != "", "Please set the OPENAI_API_KEY environment variable."
         self.client = AsyncOpenAI()
 
         self.config = {
-            'model_name': model_name,
-            'max_tokens': max_tokens,
-            'temperature': temperature,
-            'top_p': top_p,
-            'request_timeout': request_timeout,
+            "model_name": model_name,
+            "max_tokens": max_tokens,
+            "temperature": temperature,
+            "top_p": top_p,
+            "request_timeout": request_timeout,
         }
 
     def extract_list_from_string(self, input_string):
-        # pattern = r'\[.*\]'  
+        # pattern = r'\[.*\]'
         # result = re.search(pattern, input_string)
         # if result:
         #     return result.group()
         # else:
         #     return None
-        start_index = input_string.find('[')
-        end_index = input_string.rfind(']')
+        start_index = input_string.find("[")
+        end_index = input_string.rfind("]")
 
         if start_index != -1 and end_index != -1 and start_index < end_index:
-            return input_string[start_index:end_index + 1]
+            return input_string[start_index : end_index + 1]
         else:
             return None
 
     def extract_dict_from_string(self, input_string):
-        start_index = input_string.find('{')
-        end_index = input_string.rfind('}')
+        start_index = input_string.find("{")
+        end_index = input_string.rfind("}")
 
         if start_index != -1 and end_index != -1 and start_index < end_index:
-            return input_string[start_index:end_index + 1]
+            return input_string[start_index : end_index + 1]
         else:
             return None
 
@@ -81,7 +82,7 @@ class OpenAIChat():
                 return None
             return output_eval
         except:
-            '''
+            """
             if(expected_type == List):
                 valid_output = self.extract_list_from_string(output)
                 output_eval = ast.literal_eval(valid_output)
@@ -94,15 +95,15 @@ class OpenAIChat():
                 if not isinstance(output_eval, expected_type):
                     return None
                 return output_eval
-            '''
+            """
             return None
 
     async def dispatch_openai_requests(
-            self,
-            messages_list,
+        self,
+        messages_list,
     ) -> list[str]:
         """Dispatches requests to OpenAI API asynchronously.
-        
+
         Args:
             messages_list: List of messages to be sent to OpenAI ChatCompletion API.
         Returns:
@@ -113,11 +114,11 @@ class OpenAIChat():
             for _ in range(retry):
                 try:
                     response = await self.client.chat.completions.create(
-                        model=self.config['model_name'],
+                        model=self.config["model_name"],
                         messages=messages,
-                        max_tokens=self.config['max_tokens'],
-                        temperature=self.config['temperature'],
-                        top_p=self.config['top_p']
+                        max_tokens=self.config["max_tokens"],
+                        temperature=self.config["temperature"],
+                        top_p=self.config["top_p"],
                     )
                     return response
                 except openai.RateLimitError:
@@ -146,10 +147,7 @@ class OpenAIChat():
 
             return None
 
-        async_responses = [
-            _request_with_retry(messages)
-            for messages in messages_list
-        ]
+        async_responses = [_request_with_retry(messages) for messages in messages_list]
 
         return await asyncio.gather(*async_responses, return_exceptions=True)
 
@@ -161,12 +159,18 @@ class OpenAIChat():
         while retry > 0 and len(messages_list_cur_index) > 0:
             messages_list_cur = [messages_list[i] for i in messages_list_cur_index]
 
-            predictions = asyncio.run(self.dispatch_openai_requests(
-                messages_list=messages_list_cur,
-            ))
-
-            preds = [self._type_check(self._boolean_fix(prediction.choices[0].message.content),
-                                      expected_type) if prediction is not None else None for prediction in predictions]
+            predictions = asyncio.run(
+                self.dispatch_openai_requests(
+                    messages_list=messages_list_cur,
+                )
+            )
+
+            preds = [
+                self._type_check(self._boolean_fix(prediction.choices[0].message.content), expected_type)
+                if prediction is not None
+                else None
+                for prediction in predictions
+            ]
             finised_index = []
             for i, pred in enumerate(preds):
                 if pred is not None:
@@ -179,6 +183,7 @@ class OpenAIChat():
 
         return responses
 
+
 # class OpenAIEmbed():
 #     def __init__():
 #         openai.api_key = os.environ.get("OPENAI_API_KEY", None)

src/openfactcheck/solvers/webservice/ftool_cp.py

@@ -4,11 +4,12 @@ from openfactcheck.solver import StandardTaskSolver, Solver
 from .factool_utils.chat_api import OpenAIChat
 from .factool_utils.prompt import CLAIM_EXTRACTION_PROMPT
 
+
 @Solver.register("factool_claimprocessor", "response", "claims")
 class FactoolClaimProcessor(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.gpt_model = self.global_config.get("factool_gpt_model", "gpt-3.5-turbo")
+        self.gpt_model = self.global_config.get("factool_gpt_model", "gpt-4o")
         self.gpt = OpenAIChat(self.gpt_model)
         self.claim_prompt = CLAIM_EXTRACTION_PROMPT
 
@@ -16,7 +17,7 @@ class FactoolClaimProcessor(StandardTaskSolver):
         response = state.get(self.input_name)
 
         claims = self._claim_extraction(responses=[response])[0]
-        
+
         extracted_claims = [claim["claim"] for claim in claims]
 
         state.set(self.output_name, extracted_claims)

src/openfactcheck/solvers/webservice/ftool_rtv.py

@@ -5,11 +5,12 @@ from .factool_utils.chat_api import OpenAIChat
 from .factool_utils.search_api import GoogleSerperAPIWrapper
 from .factool_utils.prompt import QUERY_GENERATION_PROMPT
 
+
 @Solver.register("factool_retriever", "claims", "claims_with_evidences")
 class FactoolRetriever(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.gpt_model = self.global_config.get("factool_gpt_model", "gpt-3.5-turbo")
+        self.gpt_model = self.global_config.get("factool_gpt_model", "gpt-4o")
         self.snippet_cnt = args.get("snippet_cnt", 10)
         self.gpt = OpenAIChat(self.gpt_model)
         self.query_prompt = QUERY_GENERATION_PROMPT
@@ -22,8 +23,8 @@ class FactoolRetriever(StandardTaskSolver):
         evidences = self.search_engine.run(queries)
         results = {}
         for query, claim, evidence in zip(queries, claims, evidences):
-            merged_query = ' '.join(query) if len(query) > 1 else str(query)
-            results[claim] = [(merged_query, x['content']) for x in evidence]
+            merged_query = " ".join(query) if query and len(query) > 1 else str(query) if query else ""
+            results[claim] = [(merged_query, x["content"]) for x in evidence]
         state.set(self.output_name, results)
         return True, state
 

src/openfactcheck/solvers/webservice/ftool_vfr.py

@@ -4,11 +4,12 @@ from openfactcheck.solver import StandardTaskSolver, Solver
 from .factool_utils.chat_api import OpenAIChat
 from .factool_utils.prompt import VERIFICATION_PROMPT
 
+
 @Solver.register("factool_verifier", "claims_with_evidences", "label")
 class FactoolVerifier(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.gpt_model = self.global_config.get("factool_gpt_model", "gpt-3.5-turbo")
+        self.gpt_model = self.global_config.get("factool_gpt_model", "gpt-4o")
         self.gpt = OpenAIChat(self.gpt_model)
         self.verification_prompt = VERIFICATION_PROMPT
 
@@ -16,19 +17,23 @@ class FactoolVerifier(StandardTaskSolver):
         claims_with_evidences = state.get(self.input_name)
         results = self._verification(claims_with_evidences)
         for i, k in enumerate(list(claims_with_evidences.keys())):
-            results[i]['claim'] = k
-            results[i]['evidences'] = claims_with_evidences[k]
+            results[i]["claim"] = k
+            results[i]["evidences"] = claims_with_evidences[k]
         state.set("detail", results)
-        label = all(v['factuality'] for v in results)
+        label = all(v["factuality"] for v in results)
         state.set(self.output_name, label)
         return True, state
 
     def _verification(self, claims_with_evidences):
         messages_list = [
             [
-                {"role": "system", "content": self.verification_prompt['system']},
-                {"role": "user", "content": self.verification_prompt['user'].format(claim=claim, evidence=str(
-                    [e[1] for e in evidence]))},
+                {"role": "system", "content": self.verification_prompt["system"]},
+                {
+                    "role": "user",
+                    "content": self.verification_prompt["user"].format(
+                        claim=claim, evidence=str([e[1] for e in evidence])
+                    ),
+                },
             ]
             for claim, evidence in claims_with_evidences.items()
         ]

src/openfactcheck/solvers/webservice/rarr_rtv.py

@@ -5,11 +5,12 @@ from .rarr_utils.question_generation import run_rarr_question_generation
 from .rarr_utils.functional_prompt import QGEN_PROMPT
 from .rarr_utils import search
 
+
 @Solver.register("rarr_retriever", "claims", "claims_with_evidences")
 class RARRRetriever(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
-        self.model = self.global_config.get("rarr_model", "gpt-3.5-turbo-instruct")
+        self.model = self.global_config.get("rarr_model", "gpt-4o-instruct")
         self.temperature_qgen = args.get("temperature_qgen", 0.7)
         self.num_rounds_qgen = args.get("num_rounds_qgen", 3)
         self.max_search_results_per_query = args.get("max_search_results_per_query", 5)
@@ -19,7 +20,7 @@ class RARRRetriever(StandardTaskSolver):
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         claims = state.get(self.input_name)
-        
+
         results = dict()
         for claim in claims:
             questions = run_rarr_question_generation(
@@ -39,8 +40,8 @@ class RARRRetriever(StandardTaskSolver):
                     sliding_distance=self.sliding_distance,
                     max_passages_per_search_result_to_return=self.max_passages_per_search_result,
                 )
-                evidences.extend([(question, x['text']) for x in q_evidences])
-               
+                evidences.extend([(question, x["text"]) for x in q_evidences])
+
             results[claim] = evidences
 
         state.set(self.output_name, results)

src/openfactcheck/solvers/webservice/rarr_vfr.py

@@ -4,19 +4,20 @@ from openfactcheck.solver import StandardTaskSolver, Solver
 from .rarr_utils.agreement_gate import run_agreement_gate
 from .rarr_utils.functional_prompt import AGREEMENT_GATE_PROMPT
 
+
 @Solver.register("rarr_verifier", "claims_with_evidences", "label")
 class RARRAgreementGate(StandardTaskSolver):
     def __init__(self, args):
         super().__init__(args)
         self.max_evidences_per_question = args.get("max_evidences_per_question", 1)
-        self.model = self.global_config.get("rarr_model", "gpt-3.5-turbo-instruct")
+        self.model = self.global_config.get("rarr_model", "gpt-4o-instruct")
 
     def __call__(self, state: FactCheckerState, *args, **kwargs):
         claims_with_evidences = state.get(self.input_name)
         results = []
         for claim, evidences in claims_with_evidences.items():
             result = {}
-            evidences = evidences[:self.max_evidences_per_question]
+            evidences = evidences[: self.max_evidences_per_question]
             labels = []
             for query, evidence in evidences:
                 gate = run_agreement_gate(
@@ -25,14 +26,14 @@ class RARRAgreementGate(StandardTaskSolver):
                     query=query,
                     evidence=evidence,
                     model=self.model,
-                    prompt=AGREEMENT_GATE_PROMPT
+                    prompt=AGREEMENT_GATE_PROMPT,
                 )
-                labels.append(gate['is_open'])
-            result['claim'] = claim
-            result['evidences'] = evidences
-            result['labels'] = labels
-            result['factuality'] = all(labels)
+                labels.append(gate["is_open"])
+            result["claim"] = claim
+            result["evidences"] = evidences
+            result["labels"] = labels
+            result["factuality"] = all(labels)
             results.append(result)
-        state.set(self.output_name, all([x['factuality'] for x in results]))
+        state.set(self.output_name, all([x["factuality"] for x in results]))
         state.set("detail", results)
         return True, state

src/openfactcheck/state.py

@@ -1,52 +1,89 @@
+from typing import Any, Optional
+
 from openfactcheck.utils.logging import get_logger
 
 # Get the logger
 logger = get_logger(__name__)
 
+
 class FactCheckerState:
     """
-    A class to manage the state of a fact checking system. It holds a question 
-    and its corresponding response, and provides methods to set and get these 
-    attributes dynamically.
-
-    Parameters
-    ----------
-    question : str
-        The question to be fact-checked.
-    response : str
-        The response to the question.
+    A class to manage the state of a fact-checking system.
+
+    It holds a question and its corresponding response, and provides methods
+    to set and get these attributes dynamically.
     """
-    def __init__(self, question: str = None, response: str = None):
+
+    def __init__(self, question: Optional[str] = None, response: Optional[str] = None) -> None:
         """
         Initialize the FactCheckerState object.
+
+        Parameters
+        ----------
+        question : Optional[str]
+            The question to be fact-checked.
+        response : Optional[str]
+            The response to the question.
         """
-        self.question: str = question
-        self.response: str = response
+        self.question: Optional[str] = question
+        self.response: Optional[str] = response
 
-    def set(self, name, value):
+    def set(self, name: str, value: Any) -> None:
         """
         Set an attribute of the state object.
+
+        Parameters
+        ----------
+        name : str
+            The name of the attribute to set.
+        value : Any
+            The value to set for the attribute.
         """
         if hasattr(self, name):
-            logging.warning(f"FactCheckerState.set: Modifying existing attribute {name}")
+            logger.warning(f"Modifying existing attribute '{name}'")
         setattr(self, name, value)
 
-    def get(self, name):
+    def get(self, name: str) -> Any:
         """
         Get an attribute of the state object.
+
+        Parameters
+        ----------
+        name : str
+            The name of the attribute to retrieve.
+
+        Returns
+        -------
+        Any
+            The value of the requested attribute.
+
+        Raises
+        ------
+        ValueError
+            If the attribute does not exist.
         """
         if not hasattr(self, name):
-            raise ValueError(f"FactCheckerState.get: Attribute {name} does not exist")
-        return getattr(self, name, None)
+            raise ValueError(f"Attribute '{name}' does not exist")
+        return getattr(self, name)
 
-    def __str__(self):
+    def __str__(self) -> str:
         """
         Return a string representation of the state object.
+
+        Returns
+        -------
+        str
+            A string representation of the object's dictionary.
         """
         return str(self.__dict__)
 
-    def to_dict(self):
+    def to_dict(self) -> dict[str, Any]:
         """
         Return a dictionary representation of the state object.
+
+        Returns
+        -------
+        Dict[str, Any]
+            A dictionary containing the object's attributes.
         """
         return self.__dict__

src/openfactcheck/templates/solver_configs/webservice.yaml

@@ -9,7 +9,7 @@ factool_retriever:
 factool_verifier:
   input_name: claims_with_evidences
   output_name: label
-factcheckgpt_model: gpt-3.5-turbo
+factcheckgpt_model: gpt-4o
 factcheckgpt_claimprocessor:
   input_name: response
   output_name: claims
@@ -31,9 +31,9 @@ factcheckgpt_retriever:
 factcheckgpt_verifier:
   input_name: claims_with_evidences
   output_name: label
-  stance_model: gpt-3.5-turbo
+  stance_model: gpt-4o
   verify_retries: 3
-rarr_model: gpt-3.5-turbo-instruct
+rarr_model: gpt-4o-instruct
 rarr_retriever:
   input_name: claims
   output_name: claims_with_evidences