import gradio as gr import os import tempfile import cv2 import numpy as np import urllib.parse from screencoder.main import generate_html_for_demo from PIL import Image import shutil import html import base64 from bs4 import BeautifulSoup from pathlib import Path # Predefined examples examples_data = [ [ "screencoder/data/input/test1.png", "", "", "", "", "screencoder/data/input/test1.png" ], [ "screencoder/data/input/test3.png", "", "", "", "", "screencoder/data/input/test3.png" ], [ "screencoder/data/input/draft.png", "Add more text about 'Trump-Musk Fued' in the whole area.", "Beautify the logo 'Google'.", "", "Add text content about 'Trump and Musk' in 'Top Stories' and 'Wikipedia'. Add boundary box for each part.", "screencoder/data/input/draft.png" ], ] example_rows = [row[:5] for row in examples_data] # TAILWIND_SCRIPT = "" def image_to_data_url(image_path): """Convert an image file to a data URL for embedding in HTML.""" try: with open(image_path, 'rb') as img_file: img_data = img_file.read() # Detect image type from file extension ext = os.path.splitext(image_path)[1].lower() mime_type = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', '.webp': 'image/webp' }.get(ext, 'image/png') encoded = base64.b64encode(img_data).decode('utf-8') return f'data:{mime_type};base64,{encoded}' except Exception as e: print(f"Error converting image to data URL: {e}") return None def patch_css_js_paths(soup: BeautifulSoup, output_dir: Path): """ Fix CSS and JS paths in the HTML to work with Gradio's file serving. Converts relative paths to /file= paths or removes them if files don't exist. """ try: # CSS for link in soup.find_all("link", rel=lambda x: x and "stylesheet" in x): href = link.get("href", "") if href.startswith(("http", "data:")): continue f = output_dir / href.lstrip("./") if f.exists(): link["href"] = f"/file={f}" print(f"Fixed CSS path: {href} -> /file={f}") else: print(f"Removing non-existent CSS: {href}") link.decompose() # JS for script in soup.find_all("script", src=True): src = script["src"] if src.startswith(("http", "data:")): continue f = output_dir / src.lstrip("./") if f.exists(): script["src"] = f"/file={f}" print(f"Fixed JS path: {src} -> /file={f}") else: print(f"Removing non-existent JS: {src}") script.decompose() except Exception as e: print(f"Error in patch_css_js_paths: {e}") return soup def render_preview(code: str, width: int, height: int, scale: float) -> str: """ Preview renderer with both width and height control for the inner canvas. """ try: soup = BeautifulSoup(code, 'html.parser') for script in soup.find_all('script'): src = script.get('src', '') if src and any(pattern in src for pattern in ['assets/', 'index-', 'iframeResizer']): script.decompose() for link in soup.find_all('link'): href = link.get('href', '') if href and any(pattern in href for pattern in ['assets/', 'index-']): link.decompose() cleaned_code = str(soup) except Exception as e: print(f"Error cleaning HTML in render_preview: {e}") # Fallback to original code if cleaning fails cleaned_code = code safe_code = html.escape(cleaned_code).replace("'", "'") iframe_html = f"""
""" return iframe_html def process_and_generate(image_input, image_path_from_state, sidebar_prompt, header_prompt, navigation_prompt, main_content_prompt): """ Main processing pipeline: takes an image (path or numpy), generates code, creates a downloadable package, and returns the initial preview and code outputs for both layout and final versions. """ final_image_path = "" is_temp_file = False # Handle image_input which can be a numpy array (from upload) or a string (from example) if isinstance(image_input, str) and os.path.exists(image_input): final_image_path = image_input elif image_input is not None: # Assumes numpy array is_temp_file = True with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp: # Gradio Image component provides RGB numpy array cv2.imwrite(tmp.name, cv2.cvtColor(image_input, cv2.COLOR_RGB2BGR)) final_image_path = tmp.name elif image_path_from_state: final_image_path = image_path_from_state else: # Return empty values for all outputs return "No image provided.", "", "", "Please upload or select an image.", gr.update(visible=False), None instructions = { "sidebar": sidebar_prompt, "header": header_prompt, "navigation": navigation_prompt, "main content": main_content_prompt } layout_html, final_html, run_id = generate_html_for_demo(final_image_path, instructions) if not run_id: # Handle potential errors from the generator error_message = f"Generation failed. Error: {layout_html}" return error_message, "", "", error_message, gr.update(visible=False), None # --- Helper function to process HTML content --- def process_html(html_content, run_id): if not html_content: return "", "" # Return empty strings if content is missing base_dir = Path(__file__).parent.resolve() soup = BeautifulSoup(html_content, 'html.parser') # Fix CSS and JS paths try: output_dir = base_dir / 'screencoder' / 'data' / 'output' / run_id soup = patch_css_js_paths(soup, output_dir) except Exception as e: print(f"Error fixing CSS/JS paths: {e}") # Convert image paths to data URLs for img in soup.find_all('img'): if img.get('src') and not img['src'].startswith(('http', 'data:')): original_src = img['src'] img_path = base_dir / 'screencoder' / 'data' / 'output' / run_id / original_src if img_path.exists(): data_url = image_to_data_url(str(img_path)) if data_url: img['src'] = data_url else: img['src'] = f'/file={str(img_path)}' else: img['src'] = original_src # Keep original if not found processed_html = str(soup) preview = render_preview(processed_html, 1920, 1080, 0.55) return preview, processed_html # --- Process both HTML versions --- layout_preview, layout_code = process_html(layout_html, run_id) final_preview, final_code = process_html(final_html, run_id) # --- Package the output --- base_dir = Path(__file__).parent.resolve() output_dir = base_dir / 'screencoder' / 'data' / 'output' / run_id packages_dir = base_dir / 'screencoder' / 'data' / 'packages' packages_dir.mkdir(exist_ok=True) package_path = packages_dir / f'{run_id}.zip' shutil.make_archive(str(packages_dir / run_id), 'zip', str(output_dir)) package_url = f'/file={str(package_path)}' if is_temp_file: os.unlink(final_image_path) # Return all the outputs, including for the state objects return layout_preview, final_preview, final_code, layout_code, final_code, gr.update(value=package_url, visible=True) with gr.Blocks(theme=gr.themes.Soft()) as demo: gr.Markdown("# ScreenCoder: Screenshot to Code") with gr.Row(): with gr.Column(scale=1): gr.Markdown("## Step 1: Provide an Image") active_image = gr.Image(type="filepath", height=400) upload_button = gr.UploadButton("Click to Upload", file_types=["image"], variant="primary") gr.Markdown("## Step 2: Write Prompts (Optional)") with gr.Accordion("Component-specific Prompts", open=False): sidebar_prompt = gr.Textbox(label="Sidebar", placeholder="Instructions for the sidebar...") header_prompt = gr.Textbox(label="Header", placeholder="Instructions for the header...") navigation_prompt = gr.Textbox(label="Navigation", placeholder="Instructions for the navigation...") main_content_prompt = gr.Textbox(label="Main Content", placeholder="Instructions for the main content...") generate_btn = gr.Button("Generate HTML", variant="primary") with gr.Column(scale=2): gr.Markdown("## Preview Area") gr.Markdown("**Tips**: \n- Use the sliders below to adjust the preview size and zoom level for better viewing experience. \n- Adjust the page's viewing angle by swiping up, down, left, or right. \n- Click the download button on the right to download the generated package.") with gr.Row(): with gr.Column(scale=4): with gr.Tabs(): with gr.TabItem("Preview With Placeholder"): with gr.Row(): scale_slider = gr.Slider(0.2, 1.5, value=0.55, step=0.05, label="Zoom") width_slider = gr.Slider(400, 2000, value=1920, step=50, label="Canvas Width (px)") height_slider = gr.Slider(300, 1200, value=1080, step=50, label="Canvas Height (px)") html_preview = gr.HTML(label="Rendered HTML", show_label=False) with gr.TabItem("Preview"): with gr.Row(): scale_slider_with_placeholder = gr.Slider(0.2, 1.5, value=0.55, step=0.05, label="Zoom") width_slider_with_placeholder = gr.Slider(400, 2000, value=1920, step=100, label="Canvas Width (px)") height_slider_with_placeholder = gr.Slider(300, 1200, value=1080, step=50, label="Canvas Height (px)") html_preview_with_placeholder = gr.HTML(label="Rendered HTML", show_label=False) with gr.TabItem("Code"): html_code_output = gr.Code(label="Generated HTML", language="html") with gr.Column(scale=1): download_button = gr.Button("Download Package", visible=False, variant="secondary", size="sm") gr.Examples( examples=example_rows, inputs=[active_image, sidebar_prompt, header_prompt, navigation_prompt, main_content_prompt], cache_examples=False, label="Examples" ) # State to hold the HTML content for each preview tab layout_code_state = gr.State("") final_code_state = gr.State("") active_image_path_state = gr.State() active_image.change( lambda p: p if isinstance(p, str) else None, inputs=active_image, outputs=active_image_path_state, show_progress=False ) demo.load( lambda: (examples_data[0][0], examples_data[0][0]), None, [active_image, active_image_path_state] ) def handle_upload(uploaded_image_np): # When a new image is uploaded, it's numpy. Clear the path state. return uploaded_image_np, None, gr.update(visible=False) upload_button.upload(handle_upload, upload_button, [active_image, active_image_path_state, download_button]) generate_btn.click( process_and_generate, [active_image, active_image_path_state, sidebar_prompt, header_prompt, navigation_prompt, main_content_prompt], [html_preview, html_preview_with_placeholder, html_code_output, layout_code_state, final_code_state, download_button], show_progress="full" ) preview_controls = [scale_slider, width_slider, height_slider] for control in preview_controls: control.change( render_preview, [layout_code_state, width_slider, height_slider, scale_slider], html_preview, show_progress=True ) preview_controls_with_placeholder = [scale_slider_with_placeholder, width_slider_with_placeholder, height_slider_with_placeholder] for control in preview_controls_with_placeholder: control.change( render_preview, [final_code_state, width_slider_with_placeholder, height_slider_with_placeholder, scale_slider_with_placeholder], html_preview_with_placeholder, show_progress=True ) download_button.click(None, download_button, None, js= \ "(url) => { const link = document.createElement('a'); link.href = url; link.download = ''; document.body.appendChild(link); link.click(); document.body.removeChild(link); }") base_dir = Path(__file__).parent.resolve() allowed_paths = [ str(base_dir), str(base_dir / 'screencoder' / 'data' / 'output'), str(base_dir / 'screencoder' / 'data' / 'packages') ] for example in examples_data: example_abs_path = (base_dir / example[0]).resolve() example_dir = example_abs_path.parent if str(example_dir) not in allowed_paths: allowed_paths.append(str(example_dir)) print("Allowed paths for file serving:") for path in allowed_paths: print(f" - {path}") if __name__ == "__main__": demo.launch( allowed_paths=allowed_paths, server_name="0.0.0.0", server_port=7860, share=False )