Spaces:
Build error
Build error
| from bs4 import BeautifulSoup | |
| from urllib import request | |
| from bot.web_scrapping.searchable_index import SearchableIndex | |
| from bot.utils.show_log import logger | |
| from bot.utils.constanst import set_api_key | |
| import pandas as pd | |
| import requests | |
| import os | |
| set_api_key(api_key='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') | |
| def save_content_to_file(url=None, text=None, output_folder=None, file_format=None): | |
| file_path = os.path.join(output_folder, f"combined_content.{file_format}") | |
| if file_format == 'txt': | |
| with open(f"{file_path}", "a", encoding="utf-8") as file: | |
| for t in text: | |
| file.write(f'{t.text}\n') | |
| logger.info(f"Content appended to {file_path}") | |
| elif file_format == 'pdf': | |
| request.urlretrieve(url, file_path) | |
| logger.info(f"Content appended to {file_path}") | |
| elif file_format == 'csv': | |
| df = pd.DataFrame({'Content': [t.text for t in text]}) | |
| df.to_csv(f"{file_path}", mode='a', index=False, header=False) | |
| logger.info(f"Content appended to {file_path}") | |
| elif file_format == 'xml': | |
| xml_content = ''.join([f'<item>{t.text}</item>' for t in text]) | |
| with open(f"{file_path}", "a", encoding="utf-8") as file: | |
| file.write(xml_content) | |
| logger.info(f"Content appended to {file_path}") | |
| else: | |
| logger.warning("Invalid file format. Supported formats: txt, pdf, csv, xml") | |
| return file_path | |
| def content_crawler_and_index(url, file_format='txt', output_folder='learning_documents'): | |
| if url != 'NO_URL': | |
| # Send an HTTP GET request to the URL | |
| responses = requests.get(url) | |
| # Check if the request was successful | |
| if responses.status_code == 200: | |
| # Create output folder if it doesn't exist | |
| if not os.path.exists(output_folder): | |
| os.makedirs(output_folder) | |
| # Parse the HTML content using BeautifulSoup | |
| soup = BeautifulSoup(responses.text, "html.parser") | |
| text = soup.find_all(['h2', 'p', 'i', 'ul']) | |
| if text: | |
| # Save content based on the specified file format | |
| file_path = save_content_to_file(text=text, output_folder=output_folder, file_format=file_format) | |
| # Create or update the index | |
| index = SearchableIndex.embed_index(url, file_path) | |
| if os.path.isfile(file_path): | |
| os.remove(file_path) | |
| return index | |
| else: | |
| file_path = save_content_to_file(url=url, output_folder=output_folder, file_format=file_format) | |
| index = SearchableIndex.embed_index(url, file_path) | |
| if os.path.isfile(file_path): | |
| os.remove(file_path) | |
| return index | |
| else: | |
| logger.warning("Failed to retrieve content from the URL.") | |
| else: | |
| index = SearchableIndex.embed_index(url=url, path=output_folder) | |
| return index | |
| if __name__ == '__main__': | |
| pass | |
| # Example usage: | |
| # First URL | |
| # idx = content_crawler_and_index("https://www.presight.io/terms-of-use.html", file_format='txt') | |
| # | |
| # Second URL (appends content to existing files) | |
| # idx = content_crawler_and_index(url='https://arxiv.org/pdf/2309.11235v1.pdf', file_format='pdf') | |
| # # example get response chatbot | |
| # prompt = 'explain the paper' | |
| # llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0) | |
| # response = SearchableIndex.query(prompt, llm, idx) | |
| # print(response) | |
| # logger.info(response) | |