| import gradio as gr | |
| import json | |
| import asyncio | |
| import os | |
| from main import StructuredExtractionSystem | |
| import time | |
| api_key = os.getenv("OPENAI_API_KEY") | |
| if not api_key: | |
| raise ValueError("OPENAI_API_KEY environment variable is required") | |
| system = StructuredExtractionSystem(api_key) | |
| async def extract_data(content, schema_text, progress=gr.Progress()): | |
| try: | |
| progress(0.1, desc="Parsing schema...") | |
| schema = json.loads(schema_text) | |
| progress(0.5, desc="Processing...") | |
| result = await system.extract_structured_data(content, schema) | |
| extracted_data = json.dumps(result["data"], indent=2) | |
| metadata = result["extraction_metadata"] | |
| total_expected = len(schema.get('properties', {})) | |
| extracted_count = len([k for k, v in result["data"].items() if v is not None and v != ""]) | |
| completeness = extracted_count / total_expected if total_expected > 0 else 0 | |
| analysis = f"""Fields Extracted: {extracted_count}/{total_expected} ({completeness:.1%}) | |
| Complexity Tier: {metadata['complexity_tier']} | |
| Processing Stages: {metadata['stages_executed']} | |
| Processing Time: {metadata['actual_processing_time']:.2f}s | |
| Schema Compliance: {metadata['schema_compliance']:.1%}""" | |
| status_flags = result.get("review_flags", []) | |
| if completeness >= 0.8 and not any(flag in ["incomplete_extraction", "low_quality", "schema_violations"] for flag in status_flags): | |
| status = "Success" | |
| elif completeness >= 0.5: | |
| status = "Partial Success" | |
| else: | |
| status = "Incomplete" | |
| if status_flags: | |
| analysis += f"\nIssues: {', '.join(status_flags)}" | |
| progress(1.0, desc="Complete") | |
| return extracted_data, analysis, status | |
| except json.JSONDecodeError as e: | |
| return "", f"Invalid JSON Schema: {str(e)}", "Schema Error" | |
| except Exception as e: | |
| return "", f"Extraction Error: {str(e)}", "Error" | |
| def extract_wrapper(content, schema_text): | |
| return asyncio.run(extract_data(content, schema_text)) | |
| github_schema = """{ | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"}, | |
| "description": {"type": "string"}, | |
| "author": {"type": "string"}, | |
| "inputs": { | |
| "type": "object", | |
| "patternProperties": { | |
| "^[_a-zA-Z][a-zA-Z0-9_-]*$": { | |
| "type": "object", | |
| "properties": { | |
| "description": {"type": "string"}, | |
| "required": {"type": "boolean"}, | |
| "default": {"type": "string"} | |
| } | |
| } | |
| } | |
| }, | |
| "outputs": { | |
| "type": "object", | |
| "patternProperties": { | |
| "^[_a-zA-Z][a-zA-Z0-9_-]*$": { | |
| "type": "object", | |
| "properties": { | |
| "description": {"type": "string"}, | |
| "value": {"type": "string"} | |
| } | |
| } | |
| } | |
| }, | |
| "runs": { | |
| "type": "object", | |
| "properties": { | |
| "using": {"type": "string"}, | |
| "steps": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"}, | |
| "uses": {"type": "string"}, | |
| "run": {"type": "string"}, | |
| "shell": {"type": "string"} | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "branding": { | |
| "type": "object", | |
| "properties": { | |
| "color": {"type": "string"}, | |
| "icon": {"type": "string"} | |
| } | |
| } | |
| }, | |
| "required": ["name", "description", "runs"] | |
| }""" | |
| github_content = """MkDocs Publisher Action | |
| I keep repeating the steps to build and deploy our MkDocs documentation sites to GitHub Pages across different repos. Let's create a reusable composite action to handle this. | |
| Action Name: MkDocs Publisher | |
| Purpose: A simple action to build an MkDocs site and push it to the gh-pages branch. Should be easy to use. Author should be listed as 'DevRel Team'. | |
| Inputs Needed: | |
| python-version: We need to specify the Python version for setting up the environment. Users should be able to choose. Let's make this optional and default it to 3.11. Description: 'The version of Python to set up for building.' | |
| requirements-file: Users might have different requirements files (e.g., requirements.txt, docs/requirements.txt). This input should specify the path. It's required. Description: 'Path to the Python requirements file'. | |
| gh-token: The GitHub token for pushing to gh-pages. This is absolutely required. Description: 'GitHub token for deployment.' | |
| Outputs: | |
| The action needs to output the URL where the site was deployed. Let's call this output page-url. Its description should be 'The URL of the deployed GitHub Pages site.' | |
| How it Runs: | |
| This will be a composite action (using: composite). Here are the steps involved: | |
| Checkout Code: First, we need the repository code. Use the standard actions/checkout@v4. | |
| Setup Python: Next, set up the Python environment. Use actions/setup-python@v5. | |
| Install Dependencies: Run a command to install the Python packages. The command is pip install -r requirements.txt. Execute this using the bash shell. | |
| Build Site: Run the command mkdocs build. Use bash for this too. | |
| Deploy to Pages: Use peaceiris/actions-gh-pages@v3 for deployment. | |
| Branding: For the marketplace look, let's use the color blue and the book-open icon.""" | |
| resume_schema = """{ | |
| "type": "object", | |
| "properties": { | |
| "basics": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"}, | |
| "label": {"type": "string"}, | |
| "email": {"type": "string"}, | |
| "phone": {"type": "string"}, | |
| "url": {"type": "string"}, | |
| "summary": {"type": "string"}, | |
| "location": { | |
| "type": "object", | |
| "properties": { | |
| "address": {"type": "string"}, | |
| "postalCode": {"type": "string"}, | |
| "city": {"type": "string"}, | |
| "countryCode": {"type": "string"}, | |
| "region": {"type": "string"} | |
| } | |
| }, | |
| "profiles": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "network": {"type": "string"}, | |
| "username": {"type": "string"}, | |
| "url": {"type": "string"} | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "work": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"}, | |
| "location": {"type": "string"}, | |
| "position": {"type": "string"}, | |
| "startDate": {"type": "string"}, | |
| "endDate": {"type": "string"}, | |
| "highlights": { | |
| "type": "array", | |
| "items": {"type": "string"} | |
| } | |
| } | |
| } | |
| }, | |
| "education": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "institution": {"type": "string"}, | |
| "area": {"type": "string"}, | |
| "studyType": {"type": "string"}, | |
| "startDate": {"type": "string"}, | |
| "endDate": {"type": "string"} | |
| } | |
| } | |
| }, | |
| "skills": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"}, | |
| "keywords": { | |
| "type": "array", | |
| "items": {"type": "string"} | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }""" | |
| resume_content = """John Doe | |
| Software Engineer | |
| Email: [email protected] | |
| Phone: +1-555-0123 | |
| Address: 123 Main St, San Francisco, CA 94105, US | |
| Website: https://johndoe.dev | |
| PROFESSIONAL SUMMARY | |
| Experienced software engineer with 5+ years developing web applications and distributed systems. Skilled in full-stack development with expertise in Python, JavaScript, and cloud technologies. | |
| WORK EXPERIENCE | |
| Senior Software Engineer | TechCorp Inc | San Francisco, CA | 2022 - Present | |
| - Designed and implemented microservices architecture serving 1M+ users | |
| - Led development of real-time data processing pipeline using Apache Kafka | |
| - Reduced system latency by 40% through performance optimization | |
| Software Engineer | StartupXYZ | Palo Alto, CA | 2020 - 2022 | |
| - Built responsive web applications using React and Node.js | |
| - Implemented CI/CD pipelines resulting in 50% faster deployment cycles | |
| - Collaborated with cross-functional teams on product development | |
| EDUCATION | |
| Bachelor of Science in Computer Science | Stanford University | 2016 - 2020 | |
| - Relevant Coursework: Data Structures, Algorithms, Database Systems | |
| - Senior Project: Machine Learning Platform for Predictive Analytics | |
| TECHNICAL SKILLS | |
| Programming Languages: Python, JavaScript, Java, Go, SQL | |
| Web Technologies: React, Node.js, HTML/CSS, REST APIs, GraphQL | |
| Cloud & DevOps: AWS, Docker, Kubernetes, Jenkins, Git""" | |
| citation_schema = """{ | |
| "type": "object", | |
| "properties": { | |
| "cff-version": {"type": "string"}, | |
| "message": {"type": "string"}, | |
| "title": {"type": "string"}, | |
| "authors": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "given-names": {"type": "string"}, | |
| "family-names": {"type": "string"}, | |
| "affiliation": {"type": "string"}, | |
| "orcid": {"type": "string"} | |
| } | |
| } | |
| }, | |
| "type": {"type": "string"}, | |
| "date-published": {"type": "string"}, | |
| "url": {"type": "string"}, | |
| "abstract": {"type": "string"}, | |
| "keywords": { | |
| "type": "array", | |
| "items": {"type": "string"} | |
| }, | |
| "preferred-citation": { | |
| "type": "object", | |
| "properties": { | |
| "type": {"type": "string"}, | |
| "title": {"type": "string"}, | |
| "authors": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "given-names": {"type": "string"}, | |
| "family-names": {"type": "string"} | |
| } | |
| } | |
| }, | |
| "collection-title": {"type": "string"}, | |
| "volume": {"type": "integer"}, | |
| "year": {"type": "integer"}, | |
| "publisher": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"} | |
| } | |
| } | |
| } | |
| } | |
| } | |
| }""" | |
| citation_content = """Title: Attention Is All You Need | |
| Authors: Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Łukasz Kaiser, Illia Polosukhin | |
| This paper introduces the Transformer, a novel neural network architecture based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. | |
| Published in: Advances in Neural Information Processing Systems, Volume 30, 2017 | |
| Publisher: Curran Associates, Inc. | |
| URL: https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf | |
| The Transformer uses multi-head self-attention to draw global dependencies between input and output. The model achieves state-of-the-art results on machine translation tasks while being more parallelizable and requiring significantly less time to train. | |
| Keywords: attention mechanism, transformer, neural networks, machine translation, self-attention | |
| This work has become foundational for modern NLP models including BERT, GPT, and T5.""" | |
| contract_schema = """{ | |
| "type": "object", | |
| "properties": { | |
| "contract_type": {"type": "string"}, | |
| "parties": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"}, | |
| "type": {"type": "string"}, | |
| "address": {"type": "string"} | |
| } | |
| } | |
| }, | |
| "contract_value": {"type": "string"}, | |
| "payment_terms": {"type": "string"}, | |
| "duration": {"type": "string"}, | |
| "start_date": {"type": "string"}, | |
| "deliverables": { | |
| "type": "array", | |
| "items": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string"}, | |
| "deadline": {"type": "string"}, | |
| "description": {"type": "string"} | |
| } | |
| } | |
| }, | |
| "key_terms": { | |
| "type": "object", | |
| "properties": { | |
| "liability_cap": {"type": "string"}, | |
| "termination_notice": {"type": "string"}, | |
| "intellectual_property": {"type": "string"} | |
| } | |
| } | |
| } | |
| }""" | |
| contract_content = """SOFTWARE DEVELOPMENT AGREEMENT | |
| This Agreement is made between: | |
| Client: TechCorp Inc., 123 Business Ave, San Francisco, CA 94105 | |
| Contractor: DevStudio LLC, 456 Developer St, Austin, TX 78701 | |
| Contract Value: $150,000 | |
| Payment Terms: 50% upfront, 50% upon completion | |
| Duration: 6 months | |
| Start Date: January 1, 2024 | |
| DELIVERABLES: | |
| 1. Web Application Development | |
| - Complete e-commerce platform with user authentication | |
| - Deadline: March 15, 2024 | |
| 2. Mobile App Development | |
| - iOS and Android applications | |
| - Deadline: May 1, 2024 | |
| 3. API Integration | |
| - Third-party payment processing integration | |
| - Deadline: April 15, 2024 | |
| KEY TERMS: | |
| - Liability is capped at the total contract value | |
| - Either party may terminate with 30 days written notice | |
| - All intellectual property developed under this agreement belongs to the Client | |
| - Contractor agrees to maintain confidentiality of all proprietary information""" | |
| with gr.Blocks(title="Unstructured to Structured Converter", theme=gr.themes.Default()) as app: | |
| gr.Markdown("# Unstructured to Structured JSON Converter") | |
| gr.Markdown("Convert any unstructured text into structured JSON following complex schemas") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("### Input Content") | |
| content_input = gr.Textbox( | |
| label="Document Content", | |
| placeholder="Enter your unstructured text here...", | |
| lines=12, | |
| max_lines=20 | |
| ) | |
| with gr.Column(): | |
| gr.Markdown("### JSON Schema") | |
| schema_input = gr.Textbox( | |
| label="Target Schema", | |
| placeholder="Enter your JSON schema here...", | |
| lines=12, | |
| max_lines=20, | |
| value=github_schema | |
| ) | |
| with gr.Row(): | |
| extract_btn = gr.Button("Extract Data", variant="primary") | |
| clear_btn = gr.Button("Clear") | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| gr.Markdown("### Extracted Data") | |
| output_json = gr.Textbox( | |
| label="JSON Output", | |
| lines=15, | |
| show_copy_button=True | |
| ) | |
| with gr.Column(scale=1): | |
| gr.Markdown("### Results") | |
| metadata_output = gr.Textbox(label="Analysis", lines=8) | |
| status_output = gr.Textbox(label="Status") | |
| gr.Markdown("### Test Cases") | |
| gr.Examples( | |
| examples=[ | |
| [github_content, github_schema], | |
| [resume_content, resume_schema], | |
| [citation_content, citation_schema], | |
| [contract_content, contract_schema] | |
| ], | |
| inputs=[content_input, schema_input], | |
| label="Select a test case:" | |
| ) | |
| gr.Markdown(""" | |
| ### System Features | |
| - **Schema Complexity**: Supports 6+ levels nesting, 250+ fields, unlimited enums | |
| - **Document Size**: Handles 50+ page documents and 10MB+ files | |
| - **Dynamic Scaling**: Cost ranges from $0.01 to $5.00 based on complexity | |
| - **Quality Assurance**: Confidence scoring with human review routing | |
| """) | |
| extract_btn.click( | |
| fn=extract_wrapper, | |
| inputs=[content_input, schema_input], | |
| outputs=[output_json, metadata_output, status_output] | |
| ) | |
| clear_btn.click( | |
| lambda: ("", "", "", "", ""), | |
| outputs=[content_input, schema_input, output_json, metadata_output, status_output] | |
| ) | |
| if __name__ == "__main__": | |
| app.launch() |