import streamlit as st import requests from bs4 import BeautifulSoup from PIL import Image import io import base64 from urllib.parse import urljoin, urlparse import pandas as pd import plotly.graph_objects as go import numpy as np from difflib import SequenceMatcher from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity import time import asyncio from playwright.sync_api import sync_playwright import sys import subprocess def install_playwright_deps(): try: from playwright.sync_api import sync_playwright # Install browsers if not already installed subprocess.run(['playwright', 'install'], check=True) except Exception as e: st.error(f"Error installing Playwright dependencies: {str(e)}") st.info("Try running 'pip install playwright' and 'playwright install' manually") def initialize_session_state(): if 'visited_urls' not in st.session_state: st.session_state.visited_urls = [] if 'load_times' not in st.session_state: st.session_state.load_times = [] if 'screenshots' not in st.session_state: st.session_state.screenshots = [] if 'crawl_results' not in st.session_state: st.session_state.crawl_results = [] def setup_browser(): """Initialize Playwright browser""" try: playwright = sync_playwright().start() browser = playwright.chromium.launch(headless=True) return playwright, browser except Exception as e: st.error(f"Error setting up browser: {str(e)}") return None, None def capture_screenshot(page): """Capture screenshot using Playwright""" screenshot_bytes = page.screenshot() return Image.open(io.BytesIO(screenshot_bytes)) def calculate_similarity(text1, text2): # Basic similarity basic_ratio = SequenceMatcher(None, text1, text2).ratio() # Semantic similarity vectorizer = TfidfVectorizer() try: tfidf = vectorizer.fit_transform([text1, text2]) semantic_ratio = cosine_similarity(tfidf[0:1], tfidf[1:2])[0][0] except: semantic_ratio = 0 return basic_ratio, semantic_ratio async def crawl_website(url, max_pages=10, search_term=None): visited = set() to_visit = {url} results = [] try: with sync_playwright() as p: browser = p.chromium.launch(headless=True) page = browser.new_page() while to_visit and len(visited) < max_pages: current_url = to_visit.pop() if current_url in visited: continue try: page.goto(current_url, wait_until="networkidle") visited.add(current_url) # Extract text content text_content = page.content() # If search term provided, check for matches match_found = search_term.lower() in text_content.lower() if search_term else True if match_found: results.append({ 'url': current_url, 'title': page.title(), 'content_preview': text_content[:200], 'matches_search': match_found }) # Find new links links = page.eval_on_selector_all('a[href]', 'elements => elements.map(el => el.href)') for href in links: absolute_url = urljoin(current_url, href) if urlparse(absolute_url).netloc == urlparse(url).netloc: to_visit.add(absolute_url) except Exception as e: st.error(f"Error crawling {current_url}: {str(e)}") browser.close() except Exception as e: st.error(f"Error in crawl process: {str(e)}") return results def main(): st.title("Web Testing and Crawling Suite") initialize_session_state() # Install dependencies if needed with st.spinner("Checking dependencies..."): install_playwright_deps() # Sidebar for tool selection tool = st.sidebar.radio( "Select Tool", ["WebTest", "Crawler", "AI Content Comparison"] ) if tool == "WebTest": st.header("WebTest - Web Performance Testing") url = st.text_input("Enter URL to test") interval = st.slider("Time interval between requests (seconds)", 1, 30, 5) max_cycles = st.number_input("Number of test cycles", 1, 100, 1) if st.button("Start Testing"): playwright, browser = setup_browser() if playwright and browser: try: page = browser.new_page() for cycle in range(max_cycles): start_time = time.time() try: page.goto(url, wait_until="networkidle") load_time = time.time() - start_time st.session_state.load_times.append(load_time) # Capture screenshot screenshot = capture_screenshot(page) st.session_state.screenshots.append(screenshot) # Show results st.success(f"Cycle {cycle + 1} completed - Load time: {load_time:.2f}s") st.image(screenshot, caption=f"Screenshot - Cycle {cycle + 1}") # Plot load times fig = go.Figure(data=go.Scatter( x=list(range(1, len(st.session_state.load_times) + 1)), y=st.session_state.load_times, mode='lines+markers' )) fig.update_layout(title="Page Load Times", xaxis_title="Cycle", yaxis_title="Load Time (s)") st.plotly_chart(fig) time.sleep(interval) except Exception as e: st.error(f"Error in cycle {cycle + 1}: {str(e)}") finally: browser.close() playwright.stop() elif tool == "Crawler": st.header("Web Crawler") base_url = st.text_input("Enter base URL to crawl") max_pages = st.number_input("Maximum pages to crawl", 1, 100, 10) search_term = st.text_input("Search term (optional)") if st.button("Start Crawling"): results = asyncio.run(crawl_website(base_url, max_pages, search_term)) st.session_state.crawl_results = results # Display results df = pd.DataFrame(results) st.dataframe(df) # Export options if st.button("Export Results"): csv = df.to_csv(index=False) b64 = base64.b64encode(csv.encode()).decode() href = f'Download CSV' st.markdown(href, unsafe_allow_html=True) else: # AI Content Comparison st.header("AI Content Comparison") url1 = st.text_input("Enter first URL (AI-generated content)") url2 = st.text_input("Enter second URL (Comparison content)") if st.button("Compare Content"): playwright, browser = setup_browser() if playwright and browser: try: page = browser.new_page() # Get content from first URL page.goto(url1, wait_until="networkidle") content1 = page.content() # Get content from second URL page.goto(url2, wait_until="networkidle") content2 = page.content() # Calculate similarities basic_ratio, semantic_ratio = calculate_similarity(content1, content2) # Display results st.subheader("Similarity Results") col1, col2 = st.columns(2) with col1: st.metric("Basic Similarity", f"{basic_ratio:.2%}") with col2: st.metric("Semantic Similarity", f"{semantic_ratio:.2%}") # Show content previews st.subheader("Content Previews") st.text_area("Content 1 (First 500 chars)", content1[:500]) st.text_area("Content 2 (First 500 chars)", content2[:500]) finally: browser.close() playwright.stop() if __name__ == "__main__": main() Goals=""" Computer Use 1. Browser based testing app 2. similar to apps I wrote years ago which would operate a browser then run tests against my web apps including being able to compare any image or text content together to search results from one of my ai programs to determine content overlap which is then used to evaluate the results and update my ai model context data to store anything that was found that adds to the original idea. When I looked at this problem before I found chrome driver for automatic testing, saucelabs which can kind of do it, and then some python testing libraries which could do it. Can you enlighten me on which python libraries and potenitally dev tools which would help me with this to automate my testing and evaluation of my ai generated content which resides at many different URLs on huggingface as running apps 3. Past apps per wayback from 2004: - https://web.archive.org/web/20040520102150/http://www.evolvable.com/EStore/ WebTest 8.0 WebTest is a stress and load testing browser. You can use WebTest to identify defects that occur when web sites incurr a large amount of traffic. To use WebTest, simply visit the pages that you want to test and WebTest remembers your navigation history. You can save the history to a text file that you can open from other machines or other copies of WebTest running on your computer. To perform stress testing, set the time interval you would like WebTest to visit each Url, then set the cycle option to cycle through your site list. Features Include: Screen Captures Graph of Page Load Times Memory of Visited Pages Tunable Delay and Maximum Wait Time Hotkeys for Typical Browser Resolution Sizing XML/XSL based Usability Reporting Crawler 1.0 Have you ever wanted to analyze a web site to see what it contains yet didn't want to spend the next few days traversing all the links on the site? With Crawler, you can enter a web address and Crawler visits every page, extracting a reusable list of links from the pages it reviews. You can save the HTML source code from each page that Crawler encounters so you can analyze the site further, or automatically surf the site and take snapshots using Evolvable WebTest. Crawler is an indespensable tool because it provides results in an easy to use format and also allows you to search for terms across different sites and save the list of links that match your search criteria. Let me help you explore the Python libraries and tools that would be well-suited for automated browser testing and content comparison for AI-generated content. The most popular and robust options for this type of testing are: Selenium with Python pythonCopyfrom selenium import webdriver from selenium.webdriver.common.by import By driver = webdriver.Chrome() driver.get("https://huggingface.co/your-app") content = driver.find_element(By.CLASS_NAME, "content").text Playwright (more modern alternative to Selenium) pythonCopyfrom playwright.sync_api import sync_playwright with sync_playwright() as p: browser = p.chromium.launch() page = browser.new_page() page.goto("https://huggingface.co/your-app") content = page.inner_text(".content") For content comparison and similarity analysis: pythonCopyfrom difflib import SequenceMatcher from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity # Basic text similarity def similarity_ratio(text1, text2): return SequenceMatcher(None, text1, text2).ratio() # More advanced semantic similarity def semantic_similarity(text1, text2): vectorizer = TfidfVectorizer() tfidf = vectorizer.fit_transform([text1, text2]) return cosine_similarity(tfidf[0:1], tfidf[1:2])[0][0] For image comparison: pythonCopyfrom PIL import Image import imagehash import numpy as np def compare_images(image1_path, image2_path): hash1 = imagehash.average_hash(Image.open(image1_path)) hash2 = imagehash.average_hash(Image.open(image2_path)) return hash1 - hash2 # Lower number means more similar Would you like me to create a more comprehensive example that combines these elements into a complete testing framework? I could show you how to: Automate browsing multiple HuggingFace URLs Extract both text and image content Compare content using different similarity metrics Store results in a structured format for model context updates """ st.markdown(Goals)