Delete tools/planner

tools/planner/apis.py

@@ -1,388 +0,0 @@
-import sys
-import os
-sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "..")))
-from langchain.prompts import PromptTemplate
-from agents.prompts import planner_agent_prompt, cot_planner_agent_prompt, react_planner_agent_prompt,reflect_prompt,react_reflect_planner_agent_prompt, REFLECTION_HEADER
-from langchain.chat_models import ChatOpenAI
-from langchain.llms.base import BaseLLM
-from langchain.schema import (
-    AIMessage,
-    HumanMessage,
-    SystemMessage
-)
-from env import ReactEnv,ReactReflectEnv
-import tiktoken
-import re
-import openai
-import time
-from enum import Enum
-from typing import List, Union, Literal
-from langchain_google_genai import ChatGoogleGenerativeAI
-
-
-def catch_openai_api_error():
-    error = sys.exc_info()[0]
-    if error == openai.error.APIConnectionError:
-        print("APIConnectionError")
-    elif error == openai.error.RateLimitError:
-        print("RateLimitError")
-        time.sleep(60)
-    elif error == openai.error.APIError:
-        print("APIError")
-    elif error == openai.error.AuthenticationError:
-        print("AuthenticationError")
-    else:
-        print("API error:", error)
-
-
-class ReflexionStrategy(Enum):
-    """
-    REFLEXION: Apply reflexion to the next reasoning trace 
-    """
-    REFLEXION = 'reflexion'
-
-
-class Planner:
-    def __init__(self,
-                 # args,
-                 agent_prompt: PromptTemplate = planner_agent_prompt,
-                 model_name: str = 'gpt-3.5-turbo-1106',
-                 ) -> None:
-
-        self.agent_prompt = agent_prompt
-        self.scratchpad: str = ''
-        self.model_name = model_name
-        self.enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
-
-        if model_name in  ['mistral-7B-32K']:
-            self.llm = ChatOpenAI(temperature=0,
-                     max_tokens=4096,
-                     openai_api_key="EMPTY", 
-                     openai_api_base="http://localhost:8301/v1", 
-                     model_name="gpt-3.5-turbo")
-        
-        if model_name in  ['ChatGLM3-6B-32K']:
-            self.llm = ChatOpenAI(temperature=0,
-                     max_tokens=4096,
-                     openai_api_key="EMPTY", 
-                     openai_api_base="http://localhost:8501/v1", 
-                     model_name="gpt-3.5-turbo")
-            
-        elif model_name in ['mixtral']:
-            self.max_token_length = 30000
-            self.llm = ChatOpenAI(temperature=0,
-                     max_tokens=4096,
-                     openai_api_key="EMPTY", 
-                     openai_api_base="http://10.176.40.135:8000/v1", 
-                     model_name="/home/huggingface_models/models--mistralai--Mixtral-8x7B-Instruct-v0.1/snapshots/e0bbb53cee412aba95f3b3fa4fc0265b1a0788b2")
-            
-        elif model_name in ['gemini']:
-            self.llm = ChatGoogleGenerativeAI(temperature=0,model="gemini-pro",google_api_key='AIzaSyDarE2hG-cCeE6-GzNcEHflQa4kjY0QCK0')
-        else:
-            self.llm = ChatOpenAI(model_name=model_name, temperature=0, max_tokens=4096, openai_api_key='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')
-
-
-        print(f"PlannerAgent {model_name} loaded.")
-
-    def run(self, text, query, log_file=None) -> str:
-        if log_file:
-            log_file.write('\n---------------Planner\n'+self._build_agent_prompt(text, query))
-        # print(self._build_agent_prompt(text, query))
-        if self.model_name in ['gemini']:
-            return str(self.llm.invoke(self._build_agent_prompt(text, query)).content)
-        else:
-            if len(self.enc.encode(self._build_agent_prompt(text, query))) > 12000:
-                return 'Max Token Length Exceeded.'
-            else:
-                return self.llm([HumanMessage(content=self._build_agent_prompt(text, query))]).content
-
-    def _build_agent_prompt(self, text, query) -> str:
-        return self.agent_prompt.format(
-            text=text,
-            query=query)
-
-
-class ReactPlanner:
-    """
-    A question answering ReAct Agent.
-    """
-    def __init__(self,
-                 agent_prompt: PromptTemplate = react_planner_agent_prompt,
-                 model_name: str = 'gpt-3.5-turbo-1106',
-                 ) -> None:
-        
-        self.agent_prompt = agent_prompt
-        self.react_llm = ChatOpenAI(model_name=model_name, temperature=0, max_tokens=1024, openai_api_key='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',model_kwargs={"stop": ["Action","Thought","Observation"]})
-        self.env = ReactEnv()
-        self.query = None
-        self.max_steps = 30
-        self.reset()
-        self.finished = False
-        self.answer = ''
-        self.enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
-
-    def run(self, text, query, reset = True) -> None:
-
-        self.query = query
-        self.text = text
-
-        if reset:
-            self.reset()
-        
-
-        while not (self.is_halted() or self.is_finished()):
-            self.step()
-        
-        return self.answer, self.scratchpad
-
-    
-    def step(self) -> None:
-        # Think
-        self.scratchpad += f'\nThought {self.curr_step}:'
-        self.scratchpad += ' ' + self.prompt_agent()
-        print(self.scratchpad.split('\n')[-1])
-
-        # Act
-        self.scratchpad += f'\nAction {self.curr_step}:'
-        action = self.prompt_agent()
-        self.scratchpad += ' ' + action
-        print(self.scratchpad.split('\n')[-1])
-
-        # Observe
-        self.scratchpad += f'\nObservation {self.curr_step}: '
-
-        action_type, action_arg = parse_action(action)
-
-        if action_type == 'CostEnquiry':
-            try:
-                input_arg = eval(action_arg)
-                if type(input_arg) != dict:
-                    raise ValueError('The sub plan can not be parsed into json format, please check. Only one day plan is supported.')
-                observation = f'Cost: {self.env.run(input_arg)}'
-            except SyntaxError:
-                observation = f'The sub plan can not be parsed into json format, please check.'
-            except ValueError as e:
-                observation = str(e)
-        
-        elif action_type == 'Finish':
-            self.finished = True
-            observation = f'The plan is finished.'
-            self.answer = action_arg
-        
-        else:
-            observation = f'Action {action_type} is not supported.'
-        
-        self.curr_step += 1
-
-        self.scratchpad += observation
-        print(self.scratchpad.split('\n')[-1])
-
-    def prompt_agent(self) -> str:
-        while True:
-            try:
-                return format_step(self.react_llm([HumanMessage(content=self._build_agent_prompt())]).content)
-            except:
-                catch_openai_api_error()
-                print(self._build_agent_prompt())
-                print(len(self.enc.encode(self._build_agent_prompt())))
-                time.sleep(5)
-    
-    def _build_agent_prompt(self) -> str:
-        return self.agent_prompt.format(
-                            query = self.query,
-                            text = self.text,
-                            scratchpad = self.scratchpad)
-    
-    def is_finished(self) -> bool:
-        return self.finished
-
-    def is_halted(self) -> bool:
-        return ((self.curr_step > self.max_steps) or (
-                    len(self.enc.encode(self._build_agent_prompt())) > 14000)) and not self.finished
-
-    def reset(self) -> None:
-        self.scratchpad = ''
-        self.answer = ''
-        self.curr_step = 1
-        self.finished = False
-
-
-class ReactReflectPlanner:
-    """
-    A question answering Self-Reflecting React Agent.
-    """
-    def __init__(self,
-                 agent_prompt: PromptTemplate = react_reflect_planner_agent_prompt,
-                reflect_prompt: PromptTemplate = reflect_prompt,
-                 model_name: str = 'gpt-3.5-turbo-1106',
-                 ) -> None:
-        
-        self.agent_prompt = agent_prompt
-        self.reflect_prompt = reflect_prompt
-        if model_name in ['gemini']:
-            self.react_llm = ChatGoogleGenerativeAI(temperature=0,model="gemini-pro",google_api_key='AIzaSyDarE2hG-cCeE6-GzNcEHflQa4kjY0QCK0')
-            self.reflect_llm = ChatGoogleGenerativeAI(temperature=0,model="gemini-pro",google_api_key='AIzaSyDarE2hG-cCeE6-GzNcEHflQa4kjY0QCK0')
-        else:
-            self.react_llm = ChatOpenAI(model_name=model_name, temperature=0, max_tokens=1024, openai_api_key='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',model_kwargs={"stop": ["Action","Thought","Observation"]})
-            self.reflect_llm = ChatOpenAI(model_name=model_name, temperature=0, max_tokens=1024, openai_api_key='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',model_kwargs={"stop": ["Action","Thought","Observation"]})
-        self.model_name = model_name
-        self.env = ReactReflectEnv()
-        self.query = None
-        self.max_steps = 30
-        self.reset()
-        self.finished = False
-        self.answer = ''
-        self.reflections: List[str] = []
-        self.reflections_str: str = ''
-        self.enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
-
-    def run(self, text, query, reset = True) -> None:
-
-        self.query = query
-        self.text = text
-
-        if reset:
-            self.reset()
-        
-
-        while not (self.is_halted() or self.is_finished()):
-            self.step()
-            if self.env.is_terminated and not self.finished:
-                self.reflect(ReflexionStrategy.REFLEXION)
-
-        
-        return self.answer, self.scratchpad
-
-    
-    def step(self) -> None:
-        # Think
-        self.scratchpad += f'\nThought {self.curr_step}:'
-        self.scratchpad += ' ' + self.prompt_agent()
-        print(self.scratchpad.split('\n')[-1])
-
-        # Act
-        self.scratchpad += f'\nAction {self.curr_step}:'
-        action = self.prompt_agent()
-        self.scratchpad += ' ' + action
-        print(self.scratchpad.split('\n')[-1])
-
-        # Observe
-        self.scratchpad += f'\nObservation {self.curr_step}: '
-
-        action_type, action_arg = parse_action(action)
-
-        if action_type == 'CostEnquiry':
-            try:
-                input_arg = eval(action_arg)
-                if type(input_arg) != dict:
-                    raise ValueError('The sub plan can not be parsed into json format, please check. Only one day plan is supported.')
-                observation = f'Cost: {self.env.run(input_arg)}'
-            except SyntaxError:
-                observation = f'The sub plan can not be parsed into json format, please check.'
-            except ValueError as e:
-                observation = str(e)
-        
-        elif action_type == 'Finish':
-            self.finished = True
-            observation = f'The plan is finished.'
-            self.answer = action_arg
-        
-        else:
-            observation = f'Action {action_type} is not supported.'
-        
-        self.curr_step += 1
-
-        self.scratchpad += observation
-        print(self.scratchpad.split('\n')[-1])
-
-    def reflect(self, strategy: ReflexionStrategy) -> None:
-        print('Reflecting...')
-        if strategy == ReflexionStrategy.REFLEXION: 
-            self.reflections += [self.prompt_reflection()]
-            self.reflections_str = format_reflections(self.reflections)
-        else:
-            raise NotImplementedError(f'Unknown reflection strategy: {strategy}')
-        print(self.reflections_str)
-
-    def prompt_agent(self) -> str:
-        while True:
-            try:
-                if self.model_name in ['gemini']:
-                    return format_step(self.react_llm.invoke(self._build_agent_prompt()).content)
-                else:
-                    return format_step(self.react_llm([HumanMessage(content=self._build_agent_prompt())]).content)
-            except:
-                catch_openai_api_error()
-                print(self._build_agent_prompt())
-                print(len(self.enc.encode(self._build_agent_prompt())))
-                time.sleep(5)
-    
-    def prompt_reflection(self) -> str:
-        while True:
-            try:
-                if self.model_name in ['gemini']:
-                    return format_step(self.reflect_llm.invoke(self._build_reflection_prompt()).content)
-                else:
-                    return format_step(self.reflect_llm([HumanMessage(content=self._build_reflection_prompt())]).content)
-            except:
-                catch_openai_api_error()
-                print(self._build_reflection_prompt())
-                print(len(self.enc.encode(self._build_reflection_prompt())))
-                time.sleep(5)
-    
-    def _build_agent_prompt(self) -> str:
-        return self.agent_prompt.format(
-                            query = self.query,
-                            text = self.text,
-                            scratchpad = self.scratchpad,
-                            reflections = self.reflections_str)
-    
-    def _build_reflection_prompt(self) -> str:
-        return self.reflect_prompt.format(
-                            query = self.query,
-                            text = self.text,
-                            scratchpad = self.scratchpad)
-    
-    def is_finished(self) -> bool:
-        return self.finished
-
-    def is_halted(self) -> bool:
-        return ((self.curr_step > self.max_steps) or (
-                    len(self.enc.encode(self._build_agent_prompt())) > 14000)) and not self.finished
-
-    def reset(self) -> None:
-        self.scratchpad = ''
-        self.answer = ''
-        self.curr_step = 1
-        self.finished = False
-        self.reflections = []
-        self.reflections_str = ''
-        self.env.reset()
-
-def format_step(step: str) -> str:
-    return step.strip('\n').strip().replace('\n', '')
-
-def parse_action(string):
-    pattern = r'^(\w+)\[(.+)\]$'
-    match = re.match(pattern, string)
-
-    try:
-        if match:
-            action_type = match.group(1)
-            action_arg = match.group(2)
-            return action_type, action_arg
-        else:
-            return None, None
-        
-    except:
-        return None, None
-
-def format_reflections(reflections: List[str],
-                        header: str = REFLECTION_HEADER) -> str:
-    if reflections == []:
-        return ''
-    else:
-        return header + 'Reflections:\n- ' + '\n- '.join([r.strip() for r in reflections])
-
-# if __name__ == '__main__':
-    

tools/planner/env.py

@@ -1,208 +0,0 @@
-from tools.flights.apis import Flights
-from tools.accommodations.apis import Accommodations
-from tools.restaurants.apis import Restaurants
-from tools.googleDistanceMatrix.apis import GoogleDistanceMatrix
-from tools.googlePlaces.apis import GooglePlaces
-from tools.attractions.apis import Attractions
-from evaluation.hardConstriant import extract_from_to,get_valid_name_city
-import math
-
-
-
-    
-
-class ReactEnv:
-    def __init__(self):
-        
-        self.flight = Flights()
-        self.accommodation = Accommodations()
-        self.restaurants = Restaurants()
-        self.googleDistanceMatrix = GoogleDistanceMatrix()
-        self.googlePlaces = GooglePlaces()
-        self.attractions = Attractions()
-    
-    def run(self, tested_data):
-
-        total_cost = 0
-        unit = tested_data
-        people_number = tested_data['people_number']
-        returned_info = []
-
-        if 'transportation' in unit and unit['transportation'] and  unit['transportation'] != '-':
-            value = unit['transportation']
-            org_city, dest_city = extract_from_to(value)
-            if org_city == None or dest_city == None:
-                org_city, dest_city = extract_from_to(unit['current_city'])
-            if 'flight number' in value.lower():
-                    try:
-                        res = self.flight.data[self.flight.data['Flight Number'] == value.split('Flight Number: ')[1].split(',')[0]]
-                        if len(res) > 0:
-                            total_cost += res['Price'].values[0] * people_number
-                        else:
-                            returned_info.append('The filght information is not valid')
-                    except:
-                        returned_info.append('The filght information is not valid')
-
-            elif 'self-driving' in value.lower() or 'taxi' in value.lower():
-                try:
-                    if 'self-driving' in value.lower():
-                        # print(org_city,dest_city)
-                        cost = self.googleDistanceMatrix.run_for_evaluation(org_city,dest_city,'self-driving')['cost']
-                        if cost == None:
-                            returned_info.append('The transporation information is not valid, please check.')
-                        else:
-                            total_cost += cost * math.ceil(people_number * 1.0 / 5)
-                    else:
-                        cost = self.googleDistanceMatrix.run_for_evaluation(org_city,dest_city,'taxi')['cost']
-                        if cost == None:
-                            returned_info.append('The transporation information is not valid, please check.')
-                        else:
-                            total_cost += cost * math.ceil(people_number * 1.0 / 4)
-                except:
-                    returned_info.append('The transporation information is not valid, please check. You have to make sure there are two cities (from A to B) in your transportation plan.')
-
-        if 'breakfast' in unit and unit['breakfast'] and unit['breakfast'] != '-':
-            name, city = get_valid_name_city(unit['breakfast'])
-            if name != '-' and city != '-':
-                res = self.restaurants.data[(self.restaurants.data['Name'] == name) & (self.restaurants.data['City'] == city)]
-                if len(res) > 0:
-                    total_cost += res['Average Cost'].values[0] * people_number
-                else:
-                    returned_info.append('The breakfase information is not valid, please check.')
-
-        if 'lunch' in unit and  unit['lunch'] and unit['lunch'] != '-':
-            name, city = get_valid_name_city(unit['lunch'])
-            if name != '-' and city != '-':
-                res = self.restaurants.data[(self.restaurants.data['Name'] == name) & (self.restaurants.data['City'] == city)]
-                if len(res) > 0:
-                    total_cost += res['Average Cost'].values[0] * people_number
-                else:
-                    returned_info.append('The lunch information is not valid, please check.')
-
-        if 'dinner' in unit and unit['dinner'] and unit['dinner'] != '-':
-            name, city = get_valid_name_city(unit['dinner'])
-            if name != '-' and city != '-':
-                res = self.restaurants.data[(self.restaurants.data['Name'] == name) & (self.restaurants.data['City'] == city)]
-                if len(res) > 0:
-                    total_cost += res['Average Cost'].values[0] * people_number
-                else:
-                    returned_info.append('The dinner information is not valid, please check.')
-
-        if 'accommodation' in unit and unit['accommodation'] and unit['accommodation'] != '-':
-            name, city = get_valid_name_city(unit['accommodation'])
-            if name != '-' and city != '-':
-                res = self.accommodation.data[(self.accommodation.data['NAME'] == name) & (self.accommodation.data['city'] == city)]
-                if len(res) > 0:
-                    total_cost += res['price'].values[0] * math.ceil(people_number * 1.0 / res['maximum occupancy'].values[0])
-                else:
-                    returned_info.append('The accommodation information is not valid, please check.')
-        
-        if len(returned_info) == 0:
-            return "The cost of your plan is " + str(total_cost) + " dollars."
-        else:
-            message = "Sorry, the cost of your plan is not available because of the following reasons:"
-            for idx, info in enumerate(returned_info):
-                message += str(idx + 1) + ". " + info + " " + '\t'
-            return message
-        
-class ReactReflectEnv(ReactEnv):
-    def __init__(self):
-        super().__init__()
-        self.is_terminated = False
-        self.max_retry_step = 3
-        self.retry_step = 0
-
-    def reset(self):
-        self.is_terminated = False
-        self.retry_step = 0
-
-    def run(self, tested_data):
-        total_cost = 0
-        unit = tested_data
-        people_number = tested_data['people_number']
-        returned_info = []
-
-        if 'transportation' in unit and unit['transportation'] and  unit['transportation'] != '-':
-            value = unit['transportation']
-            org_city, dest_city = extract_from_to(value)
-            if org_city == None or dest_city == None:
-                org_city, dest_city = extract_from_to(unit['current_city'])
-                
-            
-            if org_city == None or dest_city == None:
-                returned_info.append('The transporation information is not valid, please check.')
-
-            else:    
-                if 'flight number' in value.lower():
-                        try:
-                            res = self.flight.data[self.flight.data['Flight Number'] == value.split('Flight Number: ')[1].split(',')[0]]
-                            if len(res) > 0:
-                                total_cost += res['Price'].values[0] * people_number
-                            else:
-                                returned_info.append('The filght information is not valid')
-                        except:
-                            returned_info.append('The filght information is not valid')
-
-                elif 'self-driving' in value.lower() or 'taxi' in value.lower():
-                        if 'self-driving' in value.lower():
-                            cost = self.googleDistanceMatrix.run_for_evaluation(org_city,dest_city,'self-driving')['cost']
-                            if cost == None:
-                                returned_info.append('The transporation information is not valid, please check.')
-                            else:
-                                total_cost += cost * math.ceil(people_number * 1.0 / 5)
-                        else:
-                            cost = self.googleDistanceMatrix.run_for_evaluation(org_city,dest_city,'taxi')['cost']
-                            if cost == None:
-                                returned_info.append('The transporation information is not valid, please check.')
-                            else:
-                                total_cost += cost * math.ceil(people_number * 1.0 / 4)
-
-        if 'breakfast' in unit and unit['breakfast'] and unit['breakfast'] != '-':
-            name, city = get_valid_name_city(unit['breakfast'])
-            if name != '-' and city != '-':
-                res = self.restaurants.data[(self.restaurants.data['Name'] == name) & (self.restaurants.data['City'] == city)]
-                if len(res) > 0:
-                    total_cost += res['Average Cost'].values[0] * people_number
-                else:
-                    returned_info.append('The breakfase information is not valid, please check.')
-
-        if 'lunch' in unit and  unit['lunch'] and unit['lunch'] != '-':
-            name, city = get_valid_name_city(unit['lunch'])
-            if name != '-' and city != '-':
-                res = self.restaurants.data[(self.restaurants.data['Name'] == name) & (self.restaurants.data['City'] == city)]
-                if len(res) > 0:
-                    total_cost += res['Average Cost'].values[0] * people_number
-                else:
-                    returned_info.append('The lunch information is not valid, please check.')
-
-        if 'dinner' in unit and unit['dinner'] and unit['dinner'] != '-':
-            name, city = get_valid_name_city(unit['dinner'])
-            if name != '-' and city != '-':
-                res = self.restaurants.data[(self.restaurants.data['Name'] == name) & (self.restaurants.data['City'] == city)]
-                if len(res) > 0:
-                    total_cost += res['Average Cost'].values[0] * people_number
-                else:
-                    returned_info.append('The dinner information is not valid, please check.')
-
-        if 'accommodation' in unit and unit['accommodation'] and unit['accommodation'] != '-':
-            name, city = get_valid_name_city(unit['accommodation'])
-            if name != '-' and city != '-':
-                res = self.accommodation.data[(self.accommodation.data['NAME'] == name) & (self.accommodation.data['city'] == city)]
-                if len(res) > 0:
-                    total_cost += res['price'].values[0] * math.ceil(people_number * 1.0 / res['maximum occupancy'].values[0])
-                else:
-                    returned_info.append('The accommodation information is not valid, please check.')
-        
-        if len(returned_info) == 0:
-            self.retry_step = 0
-            self.is_terminated = False
-            return "The cost of your plan is " + str(total_cost) + " dollars."
-        else:
-            message = "Sorry, the cost of your plan is not available because of the following reasons:"
-            for idx, info in enumerate(returned_info):
-                message += str(idx + 1) + ". " + info + " " + '\t'
-            self.retry_step += 1
-            if self.retry_step >= self.max_retry_step:
-                self.is_terminated = True
-            return message
-        

tools/planner/planner_with_human_annotated_info.py

@@ -1,124 +0,0 @@
-import os
-import re
-import sys
-sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "..")))
-sys.path.append('/home/xj/toolAugEnv/code/toolConstraint')
-# print(sys.path)
-os.chdir(os.path.dirname(os.path.abspath(__file__)))
-from agents.prompts import planner_agent_prompt, cot_planner_agent_prompt, react_planner_agent_prompt,react_reflect_planner_agent_prompt,reflect_prompt
-# from annotation.src.utils import get_valid_name_city,extract_before_parenthesis, extract_numbers_from_filenames
-import json
-import time
-from langchain.callbacks import get_openai_callback
-
-from tqdm import tqdm
-from tools.planner.apis import Planner, ReactPlanner, ReactReflectPlanner
-import openai
-
-os.environ["http_proxy"] = "http://127.0.0.1:7890"
-os.environ["https_proxy"] = "http://127.0.0.1:7890"
-
-
-
-def load_line_json_data(filename):
-    data = []
-    with open(filename, 'r', encoding='utf-8') as f:
-        for line in f.read().strip().split('\n'):
-            unit = json.loads(line)
-            data.append(unit)
-    return data
-
-def extract_numbers_from_filenames(directory):
-    # Define the pattern to match files
-    pattern = r'annotation_(\d+).json'
-
-    # List all files in the directory
-    files = os.listdir(directory)
-
-    # Extract numbers from filenames that match the pattern
-    numbers = [int(re.search(pattern, file).group(1)) for file in files if re.match(pattern, file)]
-
-    return numbers
-
-
-def catch_openai_api_error():
-    error = sys.exc_info()[0]
-    if error == openai.error.APIConnectionError:
-        print("APIConnectionError")
-    elif error == openai.error.RateLimitError:
-        print("RateLimitError")
-        time.sleep(60)
-    elif error == openai.error.APIError:
-        print("APIError")
-    elif error == openai.error.AuthenticationError:
-        print("AuthenticationError")
-    else:
-        print("API error:", error)
-
-# if __name__ == "__main__":
-#     user_name = 'zk'
-#     directory = '../../data/annotation/{}'.format(user_name)
-#     query_data_list = load_line_json_data('../../data/query/{}.jsonl'.format(user_name))
-#     numbers = extract_numbers_from_filenames(directory)
-#     with get_openai_callback() as cb:
-#         for number in tqdm(numbers[:10]):
-#             print(number)
-#             json_data = json.load(open(os.path.join(directory, 'annotation_{}.json'.format(number))))
-#             human_collected_info_data = json.load(open(os.path.join(directory, 'human_collected_info_{}.json'.format(number))))
-#             query_data = query_data_list[number-1]
-#             planner_results = planner.run(human_collected_info_data, query_data['query'])
-#             org_result = json.load(open(os.path.join('../../results/turbo16k-turbo16k/{}/plan_{}.json'.format(user_name,number))))
-#             # org_result.append({})
-#             org_result[-1]['chatgpt_human_collected_info_results'] = planner_results
-#             # write to json file
-#             # with open(os.path.join('../../results/turbo16k-turbo16k/{}/plan_{}.json'.format(user_name,number)), 'w') as f:
-#             #     json.dump(org_result, f, indent=4)
-#         print(cb)
-
-if __name__ == "__main__":
-    model_name=['gpt-3.5-turbo-1106','gpt-4-1106-preview','gemini','mixtral'][1]
-    set_type = ['dev','test'][0]
-    method = ['direct','cot','react','reflexion'][0]
-    directory = f'/home/xj/toolAugEnv/code/toolConstraint/data/final_data/{set_type}'
-    query_data_list = load_line_json_data(os.path.join(directory, 'query/query.jsonl'))
-    numbers = [i for i in range(1,len(query_data_list)+1)]
-
-    if method == 'direct':
-        planner = Planner(model_name=model_name, agent_prompt=planner_agent_prompt)
-    elif method == 'cot':
-        planner = Planner(model_name=model_name, agent_prompt=cot_planner_agent_prompt)
-    elif method == 'react':
-        planner = ReactPlanner(model_name=model_name, agent_prompt=react_planner_agent_prompt)
-    elif method == 'reflexion':
-        planner = ReactReflectPlanner(model_name=model_name, agent_prompt=react_reflect_planner_agent_prompt,reflect_prompt=reflect_prompt)
-
-
-    with get_openai_callback() as cb:
-        for number in tqdm(numbers[:]):
-            # print(number)
-            # json_data = json.load(open(os.path.join(directory, 'plan/annotation_{}.json'.format(number))))
-            human_collected_info_data = json.load(open(os.path.join(directory, 'plan/human_collected_info_{}.json'.format(number))))
-            query_data = query_data_list[number-1]
-
-            while True:
-                    if method in ['react','reflexion']:
-                        planner_results, scratchpad  = planner.run(human_collected_info_data, query_data['query'])
-                    else:
-                        planner_results  = planner.run(human_collected_info_data, query_data['query'])
-                    if planner_results != None:
-                        break
-            print(planner_results)
-            # check if the directory exists
-            if not os.path.exists(os.path.join(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}')):
-                os.makedirs(os.path.join(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}'))
-            if not os.path.exists(os.path.join(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}/plan_{number}.json')):
-                result =  [{}]
-            else:
-                result = json.load(open(os.path.join(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}/plan_{number}.json')))
-            if method in ['react','reflexion']:
-                result[-1][f'{model_name}_{method}_collected_info_results_logs'] = scratchpad 
-            result[-1][f'{model_name}_{method}_collected_info_results'] = planner_results
-            # write to json file
-            with open(os.path.join(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}/plan_{number}.json'), 'w') as f:
-                json.dump(result, f, indent=4)
-        print(cb)

tools/planner/test.py

@@ -1 +0,0 @@
-print(eval("[ddd"))