app.py

import gradio as gr
import json
import asyncio
import os
from main import StructuredExtractionSystem
import time

api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
    raise ValueError("OPENAI_API_KEY environment variable is required")

system = StructuredExtractionSystem(api_key)

async def extract_data(content, schema_text, progress=gr.Progress()):
    try:
        progress(0.1, desc="Parsing schema...")
        schema = json.loads(schema_text)
        
        progress(0.5, desc="Processing...")
        result = await system.extract_structured_data(content, schema)
        
        extracted_data = json.dumps(result["data"], indent=2)
        metadata = result["extraction_metadata"]
        
        total_expected = len(schema.get('properties', {}))
        extracted_count = len([k for k, v in result["data"].items() if v is not None and v != ""])
        completeness = extracted_count / total_expected if total_expected > 0 else 0
        
        analysis = f"""Fields Extracted: {extracted_count}/{total_expected} ({completeness:.1%})
Complexity Tier: {metadata['complexity_tier']}
Processing Stages: {metadata['stages_executed']}
Processing Time: {metadata['actual_processing_time']:.2f}s
Schema Compliance: {metadata['schema_compliance']:.1%}"""
        
        status_flags = result.get("review_flags", [])
        
        if completeness >= 0.8 and not any(flag in ["incomplete_extraction", "low_quality", "schema_violations"] for flag in status_flags):
            status = "Success"
        elif completeness >= 0.5:
            status = "Partial Success"
        else:
            status = "Incomplete"
        
        if status_flags:
            analysis += f"\nIssues: {', '.join(status_flags)}"
        
        progress(1.0, desc="Complete")
        return extracted_data, analysis, status
        
    except json.JSONDecodeError as e:
        return "", f"Invalid JSON Schema: {str(e)}", "Schema Error"
    except Exception as e:
        return "", f"Extraction Error: {str(e)}", "Error"

def extract_wrapper(content, schema_text):
    return asyncio.run(extract_data(content, schema_text))

github_schema = """{
  "type": "object",
  "properties": {
    "name": {"type": "string"},
    "description": {"type": "string"},
    "author": {"type": "string"},
    "inputs": {
      "type": "object",
      "patternProperties": {
        "^[_a-zA-Z][a-zA-Z0-9_-]*$": {
          "type": "object",
          "properties": {
            "description": {"type": "string"},
            "required": {"type": "boolean"},
            "default": {"type": "string"}
          }
        }
      }
    },
    "outputs": {
      "type": "object",
      "patternProperties": {
        "^[_a-zA-Z][a-zA-Z0-9_-]*$": {
          "type": "object",
          "properties": {
            "description": {"type": "string"},
            "value": {"type": "string"}
          }
        }
      }
    },
    "runs": {
      "type": "object",
      "properties": {
        "using": {"type": "string"},
        "steps": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "name": {"type": "string"},
              "uses": {"type": "string"},
              "run": {"type": "string"},
              "shell": {"type": "string"}
            }
          }
        }
      }
    },
    "branding": {
      "type": "object",
      "properties": {
        "color": {"type": "string"},
        "icon": {"type": "string"}
      }
    }
  },
  "required": ["name", "description", "runs"]
}"""

github_content = """MkDocs Publisher Action

I keep repeating the steps to build and deploy our MkDocs documentation sites to GitHub Pages across different repos. Let's create a reusable composite action to handle this.

Action Name: MkDocs Publisher
Purpose: A simple action to build an MkDocs site and push it to the gh-pages branch. Should be easy to use. Author should be listed as 'DevRel Team'.

Inputs Needed:
python-version: We need to specify the Python version for setting up the environment. Users should be able to choose. Let's make this optional and default it to 3.11. Description: 'The version of Python to set up for building.'
requirements-file: Users might have different requirements files (e.g., requirements.txt, docs/requirements.txt). This input should specify the path. It's required. Description: 'Path to the Python requirements file'.
gh-token: The GitHub token for pushing to gh-pages. This is absolutely required. Description: 'GitHub token for deployment.'

Outputs:
The action needs to output the URL where the site was deployed. Let's call this output page-url. Its description should be 'The URL of the deployed GitHub Pages site.'

How it Runs:
This will be a composite action (using: composite). Here are the steps involved:
Checkout Code: First, we need the repository code. Use the standard actions/checkout@v4.
Setup Python: Next, set up the Python environment. Use actions/setup-python@v5.
Install Dependencies: Run a command to install the Python packages. The command is pip install -r requirements.txt. Execute this using the bash shell.
Build Site: Run the command mkdocs build. Use bash for this too.
Deploy to Pages: Use peaceiris/actions-gh-pages@v3 for deployment.

Branding: For the marketplace look, let's use the color blue and the book-open icon."""

resume_schema = """{
  "type": "object",
  "properties": {
    "basics": {
      "type": "object",
      "properties": {
        "name": {"type": "string"},
        "label": {"type": "string"},
        "email": {"type": "string"},
        "phone": {"type": "string"},
        "url": {"type": "string"},
        "summary": {"type": "string"},
        "location": {
          "type": "object",
          "properties": {
            "address": {"type": "string"},
            "postalCode": {"type": "string"},
            "city": {"type": "string"},
            "countryCode": {"type": "string"},
            "region": {"type": "string"}
          }
        },
        "profiles": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "network": {"type": "string"},
              "username": {"type": "string"},
              "url": {"type": "string"}
            }
          }
        }
      }
    },
    "work": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "name": {"type": "string"},
          "location": {"type": "string"},
          "position": {"type": "string"},
          "startDate": {"type": "string"},
          "endDate": {"type": "string"},
          "highlights": {
            "type": "array",
            "items": {"type": "string"}
          }
        }
      }
    },
    "education": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "institution": {"type": "string"},
          "area": {"type": "string"},
          "studyType": {"type": "string"},
          "startDate": {"type": "string"},
          "endDate": {"type": "string"}
        }
      }
    },
    "skills": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "name": {"type": "string"},
          "keywords": {
            "type": "array",
            "items": {"type": "string"}
          }
        }
      }
    }
  }
}"""

resume_content = """John Doe
Software Engineer
Email: [email protected]
Phone: +1-555-0123
Address: 123 Main St, San Francisco, CA 94105, US
Website: https://johndoe.dev

PROFESSIONAL SUMMARY
Experienced software engineer with 5+ years developing web applications and distributed systems. Skilled in full-stack development with expertise in Python, JavaScript, and cloud technologies.

WORK EXPERIENCE

Senior Software Engineer | TechCorp Inc | San Francisco, CA | 2022 - Present
- Designed and implemented microservices architecture serving 1M+ users
- Led development of real-time data processing pipeline using Apache Kafka
- Reduced system latency by 40% through performance optimization

Software Engineer | StartupXYZ | Palo Alto, CA | 2020 - 2022  
- Built responsive web applications using React and Node.js
- Implemented CI/CD pipelines resulting in 50% faster deployment cycles
- Collaborated with cross-functional teams on product development

EDUCATION

Bachelor of Science in Computer Science | Stanford University | 2016 - 2020
- Relevant Coursework: Data Structures, Algorithms, Database Systems
- Senior Project: Machine Learning Platform for Predictive Analytics

TECHNICAL SKILLS

Programming Languages: Python, JavaScript, Java, Go, SQL
Web Technologies: React, Node.js, HTML/CSS, REST APIs, GraphQL
Cloud & DevOps: AWS, Docker, Kubernetes, Jenkins, Git"""

citation_schema = """{
  "type": "object",
  "properties": {
    "cff-version": {"type": "string"},
    "message": {"type": "string"},
    "title": {"type": "string"},
    "authors": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "given-names": {"type": "string"},
          "family-names": {"type": "string"},
          "affiliation": {"type": "string"},
          "orcid": {"type": "string"}
        }
      }
    },
    "type": {"type": "string"},
    "date-published": {"type": "string"},
    "url": {"type": "string"},
    "abstract": {"type": "string"},
    "keywords": {
      "type": "array",
      "items": {"type": "string"}
    },
    "preferred-citation": {
      "type": "object",
      "properties": {
        "type": {"type": "string"},
        "title": {"type": "string"},
        "authors": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "given-names": {"type": "string"},
              "family-names": {"type": "string"}
            }
          }
        },
        "collection-title": {"type": "string"},
        "volume": {"type": "integer"},
        "year": {"type": "integer"},
        "publisher": {
          "type": "object",
          "properties": {
            "name": {"type": "string"}
          }
        }
      }
    }
  }
}"""

citation_content = """Title: Attention Is All You Need
Authors: Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Łukasz Kaiser, Illia Polosukhin

This paper introduces the Transformer, a novel neural network architecture based solely on attention mechanisms, dispensing with recurrence and convolutions entirely.

Published in: Advances in Neural Information Processing Systems, Volume 30, 2017
Publisher: Curran Associates, Inc.
URL: https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf

The Transformer uses multi-head self-attention to draw global dependencies between input and output. The model achieves state-of-the-art results on machine translation tasks while being more parallelizable and requiring significantly less time to train.

Keywords: attention mechanism, transformer, neural networks, machine translation, self-attention

This work has become foundational for modern NLP models including BERT, GPT, and T5."""

contract_schema = """{
  "type": "object",
  "properties": {
    "contract_type": {"type": "string"},
    "parties": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "name": {"type": "string"},
          "type": {"type": "string"},
          "address": {"type": "string"}
        }
      }
    },
    "contract_value": {"type": "string"},
    "payment_terms": {"type": "string"},
    "duration": {"type": "string"},
    "start_date": {"type": "string"},
    "deliverables": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "name": {"type": "string"},
          "deadline": {"type": "string"},
          "description": {"type": "string"}
        }
      }
    },
    "key_terms": {
      "type": "object",
      "properties": {
        "liability_cap": {"type": "string"},
        "termination_notice": {"type": "string"},
        "intellectual_property": {"type": "string"}
      }
    }
  }
}"""

contract_content = """SOFTWARE DEVELOPMENT AGREEMENT

This Agreement is made between:
Client: TechCorp Inc., 123 Business Ave, San Francisco, CA 94105
Contractor: DevStudio LLC, 456 Developer St, Austin, TX 78701

Contract Value: $150,000
Payment Terms: 50% upfront, 50% upon completion
Duration: 6 months
Start Date: January 1, 2024

DELIVERABLES:
1. Web Application Development
   - Complete e-commerce platform with user authentication
   - Deadline: March 15, 2024

2. Mobile App Development  
   - iOS and Android applications
   - Deadline: May 1, 2024

3. API Integration
   - Third-party payment processing integration
   - Deadline: April 15, 2024

KEY TERMS:
- Liability is capped at the total contract value
- Either party may terminate with 30 days written notice
- All intellectual property developed under this agreement belongs to the Client
- Contractor agrees to maintain confidentiality of all proprietary information"""

with gr.Blocks(title="Unstructured to Structured Converter", theme=gr.themes.Default()) as app:
    gr.Markdown("# Unstructured to Structured JSON Converter")
    gr.Markdown("Convert any unstructured text into structured JSON following complex schemas")
    
    with gr.Row():
        with gr.Column():
            gr.Markdown("### Input Content")
            content_input = gr.Textbox(
                label="Document Content",
                placeholder="Enter your unstructured text here...",
                lines=12,
                max_lines=20
            )
            
        with gr.Column():
            gr.Markdown("### JSON Schema")
            schema_input = gr.Textbox(
                label="Target Schema",
                placeholder="Enter your JSON schema here...",
                lines=12,
                max_lines=20,
                value=github_schema
            )
    
    with gr.Row():
        extract_btn = gr.Button("Extract Data", variant="primary")
        clear_btn = gr.Button("Clear")
    
    with gr.Row():
        with gr.Column(scale=2):
            gr.Markdown("### Extracted Data")
            output_json = gr.Textbox(
                label="JSON Output",
                lines=15,
                show_copy_button=True
            )
            
        with gr.Column(scale=1):
            gr.Markdown("### Results")
            metadata_output = gr.Textbox(label="Analysis", lines=8)
            status_output = gr.Textbox(label="Status")
    
    gr.Markdown("### Test Cases")
    gr.Examples(
        examples=[
            [github_content, github_schema],
            [resume_content, resume_schema],
            [citation_content, citation_schema],
            [contract_content, contract_schema]
        ],
        inputs=[content_input, schema_input],
        label="Select a test case:"
    )
    
    gr.Markdown("""
    ### System Features
    - **Schema Complexity**: Supports 6+ levels nesting, 250+ fields, unlimited enums
    - **Document Size**: Handles 50+ page documents and 10MB+ files
    - **Dynamic Scaling**: Cost ranges from $0.01 to $5.00 based on complexity
    - **Quality Assurance**: Confidence scoring with human review routing
    """)
    
    extract_btn.click(
        fn=extract_wrapper,
        inputs=[content_input, schema_input],
        outputs=[output_json, metadata_output, status_output]
    )
    
    clear_btn.click(
        lambda: ("", "", "", "", ""),
        outputs=[content_input, schema_input, output_json, metadata_output, status_output]
    )

if __name__ == "__main__":
    app.launch()