annotated_casl_app.py

import gradio as gr
import json
import os
import logging
import requests
import re
import time

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Anthropic API key
ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")

if ANTHROPIC_API_KEY:
    logger.info("Claude API key found")
else:
    logger.warning("Claude API key not found - using demo mode")

def segment_response_by_sections(response_text):
    """Segment response by section titles and return a dictionary of sections"""
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    sections = {}
    current_section = None
    current_content = []
    
    lines = response_text.split('\n')
    
    for line in lines:
        # Check if this line is a section header
        is_section_header = False
        for section in required_sections:
            if section in line:
                # Save previous section if exists
                if current_section and current_content:
                    sections[current_section] = '\n'.join(current_content).strip()
                
                # Start new section
                current_section = section
                current_content = []
                is_section_header = True
                break
        
        # If not a section header, add to current section content
        if not is_section_header and current_section:
            current_content.append(line)
    
    # Save the last section
    if current_section and current_content:
        sections[current_section] = '\n'.join(current_content).strip()
    
    return sections

def combine_sections_smartly(sections_dict):
    """Combine sections in the correct order without duplicates"""
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    combined_parts = []
    combined_parts.append("COMPREHENSIVE SPEECH SAMPLE ANALYSIS")
    combined_parts.append("")
    
    for section in required_sections:
        if section in sections_dict:
            combined_parts.append(section)
            combined_parts.append("")
            combined_parts.append(sections_dict[section])
            combined_parts.append("")
    
    return '\n'.join(combined_parts)

def call_claude_api_with_continuation(prompt, max_continuations=0):
    """Call Claude API with smart continuation system - unlimited continuations until complete"""
    if not ANTHROPIC_API_KEY:
        return "❌ Claude API key not configured. Please set ANTHROPIC_API_KEY environment variable."
    
    # Define all required sections
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    # Safety limits to prevent infinite loops
    MAX_CONTINUATIONS = 30  # Increased from 20 to 30 API calls
    MAX_TIME_MINUTES = 15   # Increased from 10 to 15 minutes total
    MIN_PROGRESS_PER_CALL = 0  # Changed from 1 to 0 to allow more flexibility
    
    try:
        all_sections = {}  # Store all sections found across all parts
        continuation_count = 0
        start_time = time.time()
        last_section_count = 0  # Track progress between calls
        
        # Add continuation instruction to original prompt
        initial_prompt = prompt + "\n\nCRITICAL INSTRUCTIONS: You MUST complete ALL 12 sections of the analysis. If your response is cut off or incomplete, end with <CONTINUE> to indicate more content is needed. Do not skip any sections. Use the checklist to ensure all sections are completed."
        
        while True:  # Unlimited continuations until complete
            if continuation_count == 0:
                current_prompt = initial_prompt
            else:
                # For continuations, provide context about what was already covered
                missing_sections = [s for s in required_sections if s not in all_sections]
                missing_text = "\n".join([f"- {section}" for section in missing_sections])
                
                current_prompt = prompt + f"\n\nCONTINUATION {continuation_count + 1}: The following sections are STILL MISSING and MUST be completed:\n\n{missing_text}\n\nCRITICAL: Provide ONLY these missing sections. Do not repeat any sections that are already complete. Focus exclusively on the missing sections listed above. Complete ALL missing sections in this response."
            
            headers = {
                "Content-Type": "application/json",
                "x-api-key": ANTHROPIC_API_KEY,
                "anthropic-version": "2023-06-01"
            }
            
            data = {
                "model": "claude-3-5-sonnet-20241022",
                "max_tokens": 4096,
                "messages": [
                    {
                        "role": "user",
                        "content": current_prompt
                    }
                ]
            }
            
            response = requests.post(
                "https://api.anthropic.com/v1/messages",
                headers=headers,
                json=data,
                timeout=90
            )
            
            if response.status_code == 200:
                response_json = response.json()
                response_text = response_json['content'][0]['text']
                
                # Log response for debugging
                print(f"\n=== PART {continuation_count + 1} RESPONSE ===")
                print(f"Length: {len(response_text)} characters")
                print(f"Contains CONTINUE: {'<CONTINUE>' in response_text}")
                print(f"First 200 chars: {response_text[:200]}...")
                print(f"Last 200 chars: {response_text[-200:]}...")
                print("=" * 50)
                
                # Segment this part and add new sections to our collection
                part_sections = segment_response_by_sections(response_text)
                for section, content in part_sections.items():
                    if section not in all_sections:  # Only add if not already present
                        all_sections[section] = content
                        print(f"Added section: {section}")
                    else:
                        print(f"Skipped duplicate section: {section}")
                
                # Check completion status
                completed_sections = len(all_sections)
                missing_sections = [s for s in required_sections if s not in all_sections]
                
                print(f"Completed sections: {completed_sections}/12")
                print(f"Missing sections: {missing_sections}")
                
                # Check if response indicates continuation is needed
                needs_continuation = "<CONTINUE>" in response_text
                
                print(f"Needs continuation: {needs_continuation}")
                print(f"Continuation count: {continuation_count}")
                
                # Safety checks to prevent infinite loops
                current_time = time.time()
                elapsed_minutes = (current_time - start_time) / 60
                current_section_count = len(all_sections)
                progress_made = current_section_count - last_section_count
                
                # Check if we're making progress
                if continuation_count > 0 and progress_made < MIN_PROGRESS_PER_CALL:
                    # Only stop if we've made multiple calls with no progress
                    if continuation_count > 3:  # Allow more attempts before giving up
                        logger.warning(f"No progress made in last call (added {progress_made} sections). Stopping to prevent infinite loop.")
                        break
                    else:
                        logger.info(f"No progress in call {continuation_count}, but continuing to allow more attempts...")
                
                # Check time limit
                if elapsed_minutes > MAX_TIME_MINUTES:
                    logger.warning(f"Time limit exceeded ({elapsed_minutes:.1f} minutes). Stopping to prevent excessive API usage.")
                    break
                
                # Check continuation limit
                if continuation_count >= MAX_CONTINUATIONS:
                    logger.warning(f"Continuation limit reached ({MAX_CONTINUATIONS} calls). Stopping to prevent excessive API usage.")
                    break
                
                # Continue if <CONTINUE> is present and safety checks pass
                if needs_continuation:
                    continuation_count += 1
                    last_section_count = current_section_count
                    logger.info(f"Continuing analysis (attempt {continuation_count}/{MAX_CONTINUATIONS}, {elapsed_minutes:.1f} minutes elapsed)")
                    continue
                else:
                    break
            else:
                logger.error(f"Claude API error: {response.status_code} - {response.text}")
                return f"❌ Claude API Error: {response.status_code}"
                
    except Exception as e:
        logger.error(f"Error calling Claude API: {str(e)}")
        return f"❌ Error: {str(e)}"
    
    # Combine all sections in the correct order
    final_response = combine_sections_smartly(all_sections)
    
    # Log final results
    print(f"\n=== FINAL SMART VALIDATION ===")
    print(f"Total sections found: {len(all_sections)}")
    print(f"All sections present: {len(all_sections) == 12}")
    print(f"Missing sections: {[s for s in required_sections if s not in all_sections]}")
    print(f"Total time: {(time.time() - start_time) / 60:.1f} minutes")
    print(f"Total API calls: {continuation_count + 1}")
    print("=" * 50)
    
    # Add completion indicator with safety info
    if continuation_count > 0:
        final_response += f"\n\n[Analysis completed in {continuation_count + 1} parts over {(time.time() - start_time) / 60:.1f} minutes]"
    
    # Add warning if incomplete due to safety limits
    if len(all_sections) < 12:
        missing_sections = [s for s in required_sections if s not in all_sections]
        final_response += f"\n\n⚠️ WARNING: Analysis incomplete due to safety limits. Missing sections: {', '.join(missing_sections)}"
        final_response += f"\n\n💡 TIP: Try running the analysis again, or use the 'Targeted Analysis' tab to focus on specific areas."
        final_response += f"\nThe 'Quick Questions' tab may also provide faster results for specific areas of interest."
    
    return final_response

def call_claude_api_quick_analysis(prompt):
    """Call Claude API for quick focused analysis - single response only"""
    if not ANTHROPIC_API_KEY:
        return "❌ Claude API key not configured. Please set ANTHROPIC_API_KEY environment variable."
    
    try:
        headers = {
            "Content-Type": "application/json",
            "x-api-key": ANTHROPIC_API_KEY,
            "anthropic-version": "2023-06-01"
        }
        
        data = {
            "model": "claude-3-5-sonnet-20241022",
            "max_tokens": 4096,
            "messages": [
                {
                    "role": "user",
                    "content": prompt
                }
            ]
        }
        
        response = requests.post(
            "https://api.anthropic.com/v1/messages",
            headers=headers,
            json=data,
            timeout=90
        )
        
        if response.status_code == 200:
            response_json = response.json()
            return response_json['content'][0]['text']
        else:
            logger.error(f"Claude API error: {response.status_code} - {response.text}")
            return f"❌ Claude API Error: {response.status_code}"
            
    except Exception as e:
        logger.error(f"Error calling Claude API: {str(e)}")
        return f"❌ Error: {str(e)}"

def call_claude_api(prompt):
    """Call Claude API directly (legacy function for backward compatibility)"""
    return call_claude_api_quick_analysis(prompt)

def answer_quick_question(transcript_content, question, age, gender, slp_notes):
    """Answer a specific question about the transcript quickly"""
    if not transcript_content or len(transcript_content.strip()) < 20:
        return "Error: Please provide a transcript for analysis."
    
    if not question or len(question.strip()) < 5:
        return "Error: Please provide a specific question."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    prompt = f"""
    You are a speech-language pathologist answering a specific question about a speech sample.

    Patient: {age}-year-old {gender}
    
    TRANSCRIPT:
    {transcript_content}{notes_section}
    
    QUESTION: {question}
    
    INSTRUCTIONS:
    - Provide a focused, detailed answer to the specific question asked
    - Include specific examples from the transcript with exact quotes
    - Provide quantitative data when relevant (counts, percentages, rates)
    - Give clinical interpretation and significance
    - Keep the response focused on the question but thorough in analysis
    - If the question relates to multiple areas, address all relevant aspects
    
    Answer the question with specific evidence from the transcript:
    """
    
    return call_claude_api_quick_analysis(prompt)

def analyze_targeted_area(transcript_content, analysis_area, age, gender, slp_notes):
    """Perform targeted analysis of a specific area"""
    if not transcript_content or len(transcript_content.strip()) < 20:
        return "Error: Please provide a transcript for analysis."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    # Define analysis prompts for different areas
    analysis_prompts = {
        "Fluency and Disfluencies": """
        Conduct a comprehensive FLUENCY ANALYSIS focusing on:
        
        1. DISFLUENCY TYPES AND COUNTS:
        - Count and quote ALL filler words ("um", "uh", "like", "you know", etc.)
        - Count and quote ALL false starts and self-corrections
        - Count and quote ALL word/phrase repetitions
        - Count and quote ALL revisions and restarts
        - Count and quote ALL prolongations or blocks (if noted)
        
        2. DISFLUENCY PATTERNS:
        - Calculate disfluency rate per 100 words
        - Identify most frequent disfluency types
        - Analyze clustering patterns
        - Assess impact on communication effectiveness
        
        3. FLUENCY FACILITATORS:
        - Identify fluent segments and their characteristics
        - Note any fluency-enhancing contexts
        - Assess overall speech rhythm and flow
        
        4. CLINICAL SIGNIFICANCE:
        - Compare to age-appropriate norms
        - Assess severity level
        - Recommend intervention priorities
        """,
        
        "Grammar and Syntax": """
        Conduct a comprehensive GRAMMATICAL ANALYSIS focusing on:
        
        1. MORPHOLOGICAL ANALYSIS:
        - Count and categorize ALL grammatical morphemes (plurals, past tense, etc.)
        - Identify missing obligatory morphemes with specific examples
        - Analyze morphological error patterns
        
        2. SYNTACTIC STRUCTURES:
        - Analyze sentence types (simple, compound, complex)
        - Count and quote ALL grammatical errors
        - Assess word order and sentence construction
        - Evaluate use of conjunctions and subordination
        
        3. VERB USAGE:
        - Analyze tense consistency and accuracy
        - Count subject-verb agreement errors
        - Assess auxiliary verb usage
        
        4. CLINICAL IMPLICATIONS:
        - Identify primary grammatical targets for intervention
        - Assess developmental appropriateness
        - Recommend specific therapy goals
        """,
        
        "Vocabulary and Semantics": """
        Conduct a comprehensive VOCABULARY ANALYSIS focusing on:
        
        1. LEXICAL DIVERSITY:
        - Calculate Type-Token Ratio (TTR)
        - Identify vocabulary sophistication levels
        - Count unique vs. repeated words
        
        2. SEMANTIC ACCURACY:
        - Identify and quote ALL semantic errors or inappropriate word choices
        - Analyze word-finding difficulties and circumlocutions
        - Assess precision of vocabulary use
        
        3. VOCABULARY CATEGORIES:
        - Categorize vocabulary by semantic fields
        - Identify high-frequency vs. low-frequency words
        - Assess academic vs. conversational vocabulary
        
        4. WORD RETRIEVAL:
        - Count and analyze word-finding pauses
        - Identify compensatory strategies
        - Assess overall lexical access efficiency
        
        5. CLINICAL RECOMMENDATIONS:
        - Identify vocabulary intervention targets
        - Recommend strategies for word-finding support
        """,
        
        "Pragmatics and Discourse": """
        Conduct a comprehensive PRAGMATIC ANALYSIS focusing on:
        
        1. DISCOURSE ORGANIZATION:
        - Analyze topic initiation, maintenance, and transitions
        - Assess narrative structure and coherence
        - Evaluate logical sequencing of ideas
        
        2. CONVERSATIONAL SKILLS:
        - Assess turn-taking appropriateness
        - Analyze response relevance and appropriateness
        - Evaluate social communication effectiveness
        
        3. REFERENTIAL COMMUNICATION:
        - Count and analyze unclear pronoun references
        - Assess use of demonstratives ("this," "that")
        - Evaluate overall referential clarity
        
        4. PRAGMATIC APPROPRIATENESS:
        - Identify any inappropriate content or responses
        - Assess contextual appropriateness
        - Evaluate social awareness in communication
        
        5. CLINICAL IMPLICATIONS:
        - Identify pragmatic intervention priorities
        - Recommend social communication goals
        """,
        
        "Sentence Complexity": """
        Conduct a comprehensive SENTENCE COMPLEXITY ANALYSIS focusing on:
        
        1. SENTENCE TYPES:
        - Count and categorize simple sentences with examples
        - Count and categorize compound sentences with examples
        - Count and categorize complex sentences with examples
        
        2. CLAUSE ANALYSIS:
        - Calculate clauses per utterance
        - Analyze subordinate clause usage
        - Assess coordination patterns
        
        3. PHRASE STRUCTURES:
        - Analyze noun phrase complexity
        - Assess verb phrase elaboration
        - Evaluate prepositional phrase usage
        
        4. SYNTACTIC MATURITY:
        - Calculate Mean Length of Utterance (MLU)
        - Assess developmental appropriateness
        - Identify areas for syntactic growth
        
        5. CLINICAL RECOMMENDATIONS:
        - Identify targets for increasing complexity
        - Recommend scaffolding strategies
        """,
        
        "Word Finding and Retrieval": """
        Conduct a comprehensive WORD RETRIEVAL ANALYSIS focusing on:
        
        1. WORD-FINDING DIFFICULTIES:
        - Count and quote ALL instances of word-finding pauses
        - Identify and quote ALL circumlocutions
        - Count and quote ALL use of generic terms ("thing," "stuff")
        
        2. RETRIEVAL STRATEGIES:
        - Identify compensatory strategies used
        - Analyze self-cueing attempts
        - Assess success rate of retrieval attempts
        
        3. ERROR PATTERNS:
        - Categorize word-finding errors by type
        - Identify semantic vs. phonological retrieval issues
        - Analyze error consistency patterns
        
        4. CONTEXTUAL FACTORS:
        - Identify contexts that facilitate vs. hinder retrieval
        - Assess impact of topic familiarity
        - Evaluate effect of linguistic complexity on retrieval
        
        5. CLINICAL IMPLICATIONS:
        - Recommend word-finding intervention strategies
        - Identify cueing hierarchies to trial
        - Suggest compensatory strategy training
        """
    }
    
    if analysis_area not in analysis_prompts:
        return f"Error: Analysis area '{analysis_area}' not recognized. Available areas: {', '.join(analysis_prompts.keys())}"
    
    prompt = f"""
    You are a speech-language pathologist conducting a targeted analysis of a specific area.

    Patient: {age}-year-old {gender}
    
    TRANSCRIPT:
    {transcript_content}{notes_section}
    
    ANALYSIS FOCUS: {analysis_area}
    
    {analysis_prompts[analysis_area]}
    
    INSTRUCTIONS:
    - Provide specific examples with exact quotes from the transcript
    - Include quantitative data (counts, percentages, rates per 100 words)
    - Give detailed clinical interpretation
    - Provide specific, measurable recommendations
    - Be thorough but focused on the specified area
    
    Conduct the targeted analysis:
    """
    
    return call_claude_api_quick_analysis(prompt)

def check_annotation_completeness(original_transcript, annotated_transcript):
    """Check if annotation is complete by verifying last 3 words are present"""
    import re
    
    # Clean and extract words from original transcript
    original_words = re.findall(r'\b\w+\b', original_transcript.strip())
    if len(original_words) < 3:
        return True, "Transcript too short to validate"
    
    # Get last 3 words from original
    last_three_words = original_words[-3:]
    
    # Clean annotated transcript (remove markers but keep words)
    cleaned_annotated = re.sub(r'\[.*?\]', '', annotated_transcript)
    annotated_words = re.findall(r'\b\w+\b', cleaned_annotated.strip())
    
    # Check if all last 3 words appear in the annotated transcript
    missing_words = []
    for word in last_three_words:
        if word.lower() not in [w.lower() for w in annotated_words]:
            missing_words.append(word)
    
    if missing_words:
        return False, f"Annotation appears incomplete. Missing words from end: {', '.join(missing_words)}"
    
    # Additional check: verify the last few words appear near the end
    if len(annotated_words) > 0:
        last_annotated_words = annotated_words[-10:]  # Check last 10 words
        last_original_in_annotated = sum(1 for word in last_three_words 
                                       if word.lower() in [w.lower() for w in last_annotated_words])
        
        if last_original_in_annotated < 2:  # At least 2 of the last 3 should be near the end
            return False, f"Annotation may be incomplete. Last words '{', '.join(last_three_words)}' not found near end of annotation"
    
    return True, "Annotation appears complete"

def annotate_transcript(transcript_content, age, gender, slp_notes):
    """First step: Annotate transcript with linguistic markers"""
    if not transcript_content or len(transcript_content.strip()) < 50:
        return "Error: Please provide a longer transcript for annotation."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    annotation_prompt = f"""
    You are a speech-language pathologist preparing a transcript for detailed analysis. Your task is to ANNOTATE the ENTIRE transcript with linguistic markers at a WORD-BY-WORD level.

    Patient: {age}-year-old {gender}
    
    ORIGINAL TRANSCRIPT:
    {transcript_content}{notes_section}
    
    CRITICAL REQUIREMENT: You MUST annotate the COMPLETE transcript. Do NOT provide partial annotations or stop mid-sentence. Complete the ENTIRE transcript annotation in one response.

    DETAILED ANNOTATION INSTRUCTIONS:
    Annotate by adding markers in brackets IMMEDIATELY after each relevant word or phrase:

    FLUENCY MARKERS:
    - [FILLER] after: um[FILLER], uh[FILLER], like[FILLER], you know[FILLER], well[FILLER], so[FILLER]
    - [FALSE_START] after incomplete words: "I was go-[FALSE_START] going"
    - [REPETITION] after repeated words: "I I[REPETITION] went"
    - [REVISION] after self-corrections: "I went to the-[REVISION] I mean"
    - [PAUSE] for hesitations: "I was...[PAUSE] thinking"

    WORD RETRIEVAL MARKERS:
    - [CIRCUMLOCUTION] after roundabout descriptions: "that thing you write with[CIRCUMLOCUTION]"
    - [INCOMPLETE] after abandoned thoughts: "I was thinking about the...[INCOMPLETE]"
    - [GENERIC] after vague terms: thing[GENERIC], stuff[GENERIC], whatsit[GENERIC]
    - [WORD_SEARCH] after searching: "the... um...[WORD_SEARCH] car"

    GRAMMATICAL MARKERS:
    - [GRAM_ERROR] after mistakes: "I goed[GRAM_ERROR]", "He don't[GRAM_ERROR]"
    - [SYNTAX_ERROR] after word order problems: "Yesterday I to store went[SYNTAX_ERROR]"
    - [MORPH_ERROR] after morphological errors: "runned[MORPH_ERROR]", "childs[MORPH_ERROR]"
    - [RUN_ON] at end of run-on sentences

    VOCABULARY MARKERS:
    - [SIMPLE_VOCAB] after basic words: go[SIMPLE_VOCAB], big[SIMPLE_VOCAB], good[SIMPLE_VOCAB]
    - [COMPLEX_VOCAB] after sophisticated words: magnificent[COMPLEX_VOCAB], elaborate[COMPLEX_VOCAB]
    - [SEMANTIC_ERROR] after wrong word choices: "drove my bicycle[SEMANTIC_ERROR]"

    PRAGMATIC MARKERS:
    - [TOPIC_SHIFT] after topic changes: "Anyway, about cats[TOPIC_SHIFT]"
    - [TANGENT] after going off-topic: "Speaking of dogs, my vacation[TANGENT]"
    - [INAPPROPRIATE] after inappropriate content
    - [COHERENCE_BREAK] after illogical statements

    SENTENCE COMPLEXITY MARKERS:
    - [SIMPLE_SENT] after simple sentences: "I went home.[SIMPLE_SENT]"
    - [COMPLEX_SENT] after complex sentences: "When I got home, I made dinner.[COMPLEX_SENT]"
    - [COMPOUND_SENT] after compound sentences: "I went home, and made dinner.[COMPOUND_SENT]"
    - [FIGURATIVE] after metaphors/idioms: "raining cats and dogs[FIGURATIVE]"

    ADDITIONAL MARKERS:
    - [PRONOUN_REF] after unclear pronouns: "He told him that he[PRONOUN_REF] was wrong"
    - [MAZING] after confusing constructions
    - [PERSEVERATION] after repetitive patterns

    MANDATORY REQUIREMENTS:
    1. Do NOT stop until the entire transcript is complete
    2. Keep ALL original text intact
    3. Mark overlapping features when applicable
    4. Be consistent throughout
    5. Complete the annotation in ONE response - no partial outputs allowed

    PROVIDE THE COMPLETE ANNOTATED TRANSCRIPT - EVERY WORD MUST BE PROCESSED.
    """
    
    # Get initial annotation
    annotated_result = call_claude_api(annotation_prompt)
    
    # Check if annotation is complete
    is_complete, validation_message = check_annotation_completeness(transcript_content, annotated_result)
    
    if not is_complete:
        logger.warning(f"Annotation incomplete: {validation_message}")
        
        # Try once more with stronger emphasis on completion
        retry_prompt = f"""
        CRITICAL: The previous annotation was INCOMPLETE. You MUST complete the ENTIRE transcript.
        
        {validation_message}
        
        ORIGINAL TRANSCRIPT (COMPLETE THIS ENTIRELY):
        {transcript_content}{notes_section}
        
        MANDATORY REQUIREMENT: Annotate EVERY SINGLE WORD from start to finish. Do not stop until you reach the very last word of the transcript.
        
        {annotation_prompt.split('DETAILED ANNOTATION INSTRUCTIONS:')[1]}
        
        VERIFY: The last words of the original transcript are: {' '.join(transcript_content.strip().split()[-3:])}
        ENSURE these words appear at the END of your annotated transcript.
        """
        
        retry_result = call_claude_api(retry_prompt)
        
        # Check retry
        retry_complete, retry_message = check_annotation_completeness(transcript_content, retry_result)
        
        if retry_complete:
            logger.info("Retry successful - annotation now complete")
            return retry_result
        else:
            logger.error(f"Retry failed: {retry_message}")
            return f"⚠️ ANNOTATION INCOMPLETE: {retry_message}\n\nPartial annotation:\n{retry_result}"
    
    logger.info("Annotation completed successfully")
    return annotated_result

def analyze_annotated_transcript(annotated_transcript, age, gender, slp_notes):
    """Second step: Analyze the annotated transcript with comprehensive quantification"""
    if not annotated_transcript or len(annotated_transcript.strip()) < 50:
        return "Error: Please provide an annotated transcript for analysis."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    analysis_prompt = f"""
    You are a speech-language pathologist conducting a COMPREHENSIVE analysis of a word-by-word annotated speech sample. Count EVERY marker precisely and provide detailed quantitative analysis.

    Patient: {age}-year-old {gender}
    
    ANNOTATED TRANSCRIPT:
    {annotated_transcript}{notes_section}
    
    ORIGINAL TRANSCRIPT (for reference and backup analysis):
    {annotated_transcript.replace('[FILLER]', '').replace('[FALSE_START]', '').replace('[REPETITION]', '').replace('[REVISION]', '').replace('[PAUSE]', '').replace('[CIRCUMLOCUTION]', '').replace('[INCOMPLETE]', '').replace('[GENERIC]', '').replace('[WORD_SEARCH]', '').replace('[GRAM_ERROR]', '').replace('[SYNTAX_ERROR]', '').replace('[MORPH_ERROR]', '').replace('[RUN_ON]', '').replace('[SIMPLE_VOCAB]', '').replace('[COMPLEX_VOCAB]', '').replace('[SEMANTIC_ERROR]', '').replace('[TOPIC_SHIFT]', '').replace('[TANGENT]', '').replace('[INAPPROPRIATE]', '').replace('[COHERENCE_BREAK]', '').replace('[SIMPLE_SENT]', '').replace('[COMPLEX_SENT]', '').replace('[COMPOUND_SENT]', '').replace('[FIGURATIVE]', '').replace('[PRONOUN_REF]', '').replace('[MAZING]', '').replace('[PERSEVERATION]', '')}
    
    ANALYSIS INSTRUCTIONS:
    Using the detailed linguistic markers in the annotated transcript, provide a comprehensive analysis with EXACT counts, percentages, and specific examples. If markers are missing or unclear, use the original transcript for backup analysis. Complete ALL sections below:

    COMPREHENSIVE SPEECH SAMPLE ANALYSIS:

    1. FLUENCY ANALYSIS (count each marker type):
    - Count [FILLER] markers: List each instance and calculate rate per 100 words
    - Count [FALSE_START] markers: List examples and analyze patterns
    - Count [REPETITION] markers: Categorize by type (word, phrase, sound)
    - Count [REVISION] markers: Analyze self-correction patterns
    - Count [PAUSE] markers: Assess hesitation frequency
    - Calculate total disfluency rate and severity level
    - Determine impact on communication effectiveness

    2. WORD RETRIEVAL ANALYSIS (precise counting):
    - Count [CIRCUMLOCUTION] markers: List each roundabout description
    - Count [INCOMPLETE] markers: Analyze abandoned thought patterns
    - Count [GENERIC] markers: Calculate specificity ratio
    - Count [WORD_SEARCH] markers: Identify retrieval difficulty areas
    - Count [WORD_FINDING] markers: Assess overall retrieval efficiency
    - Calculate word-finding accuracy percentage

    3. GRAMMATICAL ANALYSIS (detailed error counting):
    - Count [GRAM_ERROR] markers by subcategory:
      * Verb tense errors
      * Subject-verb agreement errors  
      * Pronoun errors
      * Article errors
    - Count [SYNTAX_ERROR] markers: Analyze word order problems
    - Count [MORPH_ERROR] markers: Categorize morphological mistakes
    - Count [RUN_ON] markers: Assess sentence boundary awareness
    - Calculate grammatical accuracy rate (correct vs. total attempts)

    4. VOCABULARY ANALYSIS (sophistication assessment):
    - Count [SIMPLE_VOCAB] markers: List basic vocabulary used
    - Count [COMPLEX_VOCAB] markers: List sophisticated vocabulary
    - Count [SEMANTIC_ERROR] markers: Analyze word choice accuracy
    - Calculate vocabulary sophistication ratio (complex/simple)
    - Assess semantic appropriateness and precision
    - Determine vocabulary diversity (type-token ratio)

    5. PRAGMATIC LANGUAGE ANALYSIS (coherence assessment):
    - Count [TOPIC_SHIFT] markers: Assess transition appropriateness
    - Count [TANGENT] markers: Analyze tangential speech patterns
    - Count [INAPPROPRIATE] markers: Evaluate contextual appropriateness
    - Count [COHERENCE_BREAK] markers: Assess logical flow
    - Count [PRONOUN_REF] markers: Analyze referential clarity
    - Evaluate overall discourse coherence and organization

    6. SENTENCE COMPLEXITY ANALYSIS (structural assessment):
    - Count [SIMPLE_SENT] markers: Calculate simple sentence percentage
    - Count [COMPLEX_SENT] markers: Analyze subordination use
    - Count [COMPOUND_SENT] markers: Assess coordination patterns
    - Count [FIGURATIVE] markers: Evaluate figurative language use
    - Count [MAZING] markers: Assess confusing constructions
    - Calculate syntactic complexity index

    7. QUANTITATIVE METRICS (comprehensive calculations):
    - Total word count and morpheme count
    - Mean Length of Utterance (MLU) in words and morphemes
    - Type-Token Ratio (TTR) for vocabulary diversity
    - Clauses per utterance ratio
    - Error rate per linguistic domain
    - Communication efficiency index

    8. ERROR PATTERN ANALYSIS:
    - Most frequent error types with exact counts
    - Error consistency vs. variability patterns
    - Developmental appropriateness of errors
    - Severity ranking of different error types
    - Compensatory strategies observed

    9. CLINICAL IMPLICATIONS:
    - Primary strengths: List with supporting evidence
    - Primary weaknesses: Rank by severity with counts
    - Intervention priorities: Based on error frequency and impact
    - Therapy targets: Specific, measurable goals
    - Prognosis indicators: Based on error patterns and consistency

    10. SUMMARY AND RECOMMENDATIONS:
    - Overall communication profile with percentile estimates
    - Priority treatment goals ranked by importance
    - Functional communication impact assessment
    - Recommended therapy approaches and frequency
    - Follow-up assessment timeline

    CRITICAL: Provide EXACT counts for every marker type, calculate precise percentages, and give specific examples from the transcript. Show your counting work clearly. Complete ALL 12 sections - use <CONTINUE> if needed.
    """
    
    return call_claude_api_with_continuation(analysis_prompt)

def calculate_linguistic_metrics(transcript_text):
    """Calculate comprehensive linguistic metrics from transcript"""
    import re
    import numpy as np
    
    if not transcript_text or not transcript_text.strip():
        return {}
    
    # Clean text and extract words
    cleaned_text = re.sub(r'\[.*?\]', '', transcript_text)  # Remove annotation markers
    sentences = re.split(r'[.!?]+', cleaned_text)
    sentences = [s.strip() for s in sentences if s.strip()]
    
    # Extract all words
    all_words = []
    for sentence in sentences:
        words = re.findall(r'\b\w+\b', sentence.lower())
        all_words.extend(words)
    
    if not all_words:
        return {}
    
    # Basic counts
    total_words = len(all_words)
    total_sentences = len(sentences)
    unique_words = len(set(all_words))
    
    # Type-Token Ratio
    ttr = unique_words / total_words if total_words > 0 else 0
    
    # Mean Length of Utterance (MLU)
    mlu_words = total_words / total_sentences if total_sentences > 0 else 0
    
    # Word frequency analysis
    word_freq = {}
    for word in all_words:
        word_freq[word] = word_freq.get(word, 0) + 1
    
    # Sort by frequency
    sorted_word_freq = dict(sorted(word_freq.items(), key=lambda x: x[1], reverse=True))
    
    # Sentence length statistics
    sentence_lengths = []
    for sentence in sentences:
        words_in_sentence = len(re.findall(r'\b\w+\b', sentence))
        sentence_lengths.append(words_in_sentence)
    
    avg_sentence_length = np.mean(sentence_lengths) if sentence_lengths else 0
    std_sentence_length = np.std(sentence_lengths) if sentence_lengths else 0
    
    # Vocabulary sophistication (words > 6 characters as proxy for complex vocabulary)
    complex_words = [word for word in all_words if len(word) > 6]
    vocabulary_sophistication = len(complex_words) / total_words if total_words > 0 else 0
    
    # Calculate morpheme count (approximate)
    morpheme_count = 0
    for word in all_words:
        # Basic morpheme counting (word + common suffixes)
        morpheme_count += 1
        if word.endswith(('s', 'ed', 'ing', 'er', 'est', 'ly')):
            morpheme_count += 1
        if word.endswith(('tion', 'sion', 'ness', 'ment', 'able', 'ible')):
            morpheme_count += 1
    
    mlu_morphemes = morpheme_count / total_sentences if total_sentences > 0 else 0
    
    return {
        'total_words': total_words,
        'total_sentences': total_sentences,
        'unique_words': unique_words,
        'type_token_ratio': round(ttr, 3),
        'mlu_words': round(mlu_words, 2),
        'mlu_morphemes': round(mlu_morphemes, 2),
        'avg_sentence_length': round(avg_sentence_length, 2),
        'sentence_length_std': round(std_sentence_length, 2),
        'vocabulary_sophistication': round(vocabulary_sophistication, 3),
        'word_frequency': dict(list(sorted_word_freq.items())[:20]),  # Top 20 most frequent
        'sentence_lengths': sentence_lengths,
        'complex_word_count': len(complex_words),
        'morpheme_count': morpheme_count,
        'tokenized_words': all_words,  # Add for lexical diversity analysis
        'cleaned_text': cleaned_text   # Add for lexical diversity analysis
    }

def calculate_advanced_lexical_diversity(transcript_text):
    """Calculate advanced lexical diversity measures using lexical-diversity library"""
    import re
    
    try:
        from lexical_diversity import lex_div as ld
        lexdiv_available = True
    except ImportError:
        lexdiv_available = False
        
    if not lexdiv_available:
        return {
            'library_available': False,
            'error': 'lexical-diversity library not installed. Install with: pip install lexical-diversity'
        }
    
    if not transcript_text or not transcript_text.strip():
        return {'library_available': True, 'error': 'No text provided'}
    
    # Clean text and prepare for lexical diversity analysis
    cleaned_text = re.sub(r'\[.*?\]', '', transcript_text)  # Remove annotation markers
    
    try:
        # Tokenize using lexical-diversity
        tokens = ld.tokenize(cleaned_text)
        
        if len(tokens) < 10:  # Need minimum tokens for meaningful analysis
            return {
                'library_available': True,
                'error': f'Insufficient tokens for analysis (need ≥10, got {len(tokens)})'
            }
        
        # Calculate various lexical diversity measures
        diversity_measures = {}
        
        # Basic TTR (included for comparison, but noted as unreliable)
        diversity_measures['simple_ttr'] = round(ld.ttr(tokens), 4)
        
        # Recommended measures
        try:
            diversity_measures['root_ttr'] = round(ld.root_ttr(tokens), 4)
        except:
            diversity_measures['root_ttr'] = None
            
        try:
            diversity_measures['log_ttr'] = round(ld.log_ttr(tokens), 4)
        except:
            diversity_measures['log_ttr'] = None
            
        try:
            diversity_measures['maas_ttr'] = round(ld.maas_ttr(tokens), 4)
        except:
            diversity_measures['maas_ttr'] = None
        
        # MSTTR (Mean Segmental TTR) - only if enough tokens
        if len(tokens) >= 50:
            try:
                diversity_measures['msttr_50'] = round(ld.msttr(tokens, window_length=50), 4)
            except:
                diversity_measures['msttr_50'] = None
        
        if len(tokens) >= 25:
            try:
                diversity_measures['msttr_25'] = round(ld.msttr(tokens, window_length=25), 4)
            except:
                diversity_measures['msttr_25'] = None
        
        # MATTR (Moving Average TTR) - only if enough tokens
        if len(tokens) >= 50:
            try:
                diversity_measures['mattr_50'] = round(ld.mattr(tokens, window_length=50), 4)
            except:
                diversity_measures['mattr_50'] = None
                
        if len(tokens) >= 25:
            try:
                diversity_measures['mattr_25'] = round(ld.mattr(tokens, window_length=25), 4)
            except:
                diversity_measures['mattr_25'] = None
        
        # HDD (Hypergeometric Distribution D)
        try:
            diversity_measures['hdd'] = round(ld.hdd(tokens), 4)
        except:
            diversity_measures['hdd'] = None
        
        # MTLD (Measure of Textual Lexical Diversity) - only if enough tokens
        if len(tokens) >= 50:
            try:
                diversity_measures['mtld'] = round(ld.mtld(tokens), 4)
            except:
                diversity_measures['mtld'] = None
                
            try:
                diversity_measures['mtld_ma_wrap'] = round(ld.mtld_ma_wrap(tokens), 4)
            except:
                diversity_measures['mtld_ma_wrap'] = None
                
            try:
                diversity_measures['mtld_ma_bid'] = round(ld.mtld_ma_bid(tokens), 4)
            except:
                diversity_measures['mtld_ma_bid'] = None
        
        return {
            'library_available': True,
            'token_count': len(tokens),
            'diversity_measures': diversity_measures,
            'tokens': tokens[:50]  # First 50 tokens for verification
        }
        
    except Exception as e:
        return {
            'library_available': True,
            'error': f'Error calculating lexical diversity: {str(e)}'
        }

def analyze_annotation_markers(annotated_transcript):
    """Analyze and count all annotation markers in the transcript with detailed word-level analysis"""
    import re
    
    if not annotated_transcript:
        return {}
    
    # Define all marker types
    marker_types = {
        'FILLER': r'\[FILLER\]',
        'FALSE_START': r'\[FALSE_START\]',
        'REPETITION': r'\[REPETITION\]',
        'REVISION': r'\[REVISION\]',
        'PAUSE': r'\[PAUSE\]',
        'CIRCUMLOCUTION': r'\[CIRCUMLOCUTION\]',
        'INCOMPLETE': r'\[INCOMPLETE\]',
        'GENERIC': r'\[GENERIC\]',
        'WORD_SEARCH': r'\[WORD_SEARCH\]',
        'GRAM_ERROR': r'\[GRAM_ERROR\]',
        'SYNTAX_ERROR': r'\[SYNTAX_ERROR\]',
        'MORPH_ERROR': r'\[MORPH_ERROR\]',
        'RUN_ON': r'\[RUN_ON\]',
        'SIMPLE_VOCAB': r'\[SIMPLE_VOCAB\]',
        'COMPLEX_VOCAB': r'\[COMPLEX_VOCAB\]',
        'SEMANTIC_ERROR': r'\[SEMANTIC_ERROR\]',
        'TOPIC_SHIFT': r'\[TOPIC_SHIFT\]',
        'TANGENT': r'\[TANGENT\]',
        'INAPPROPRIATE': r'\[INAPPROPRIATE\]',
        'COHERENCE_BREAK': r'\[COHERENCE_BREAK\]',
        'SIMPLE_SENT': r'\[SIMPLE_SENT\]',
        'COMPLEX_SENT': r'\[COMPLEX_SENT\]',
        'COMPOUND_SENT': r'\[COMPOUND_SENT\]',
        'FIGURATIVE': r'\[FIGURATIVE\]',
        'PRONOUN_REF': r'\[PRONOUN_REF\]',
        'MAZING': r'\[MAZING\]',
        'PERSEVERATION': r'\[PERSEVERATION\]'
    }
    
    # Count each marker type and extract the actual words
    marker_counts = {}
    marker_examples = {}
    marker_words = {}
    
    for marker_name, pattern in marker_types.items():
        matches = re.findall(pattern, annotated_transcript)
        marker_counts[marker_name] = len(matches)
        
        # Find examples with context and extract the actual words
        examples = []
        words = []
        
        # Find all instances of word[MARKER] pattern
        word_pattern = r'(\w+)' + pattern
        word_matches = re.finditer(word_pattern, annotated_transcript)
        
        for match in word_matches:
            word = match.group(1)
            words.append(word)
            
            # Get context around the match
            start = max(0, match.start() - 30)
            end = min(len(annotated_transcript), match.end() + 30)
            context = annotated_transcript[start:end].strip()
            examples.append(f'"{word}" in context: {context}')
        
        marker_examples[marker_name] = examples[:10]  # Keep first 10 examples
        marker_words[marker_name] = words
    
    # Calculate totals by category
    fluency_total = sum([marker_counts.get(m, 0) for m in ['FILLER', 'FALSE_START', 'REPETITION', 'REVISION', 'PAUSE']])
    grammar_total = sum([marker_counts.get(m, 0) for m in ['GRAM_ERROR', 'SYNTAX_ERROR', 'MORPH_ERROR', 'RUN_ON']])
    vocab_simple = marker_counts.get('SIMPLE_VOCAB', 0)
    vocab_complex = marker_counts.get('COMPLEX_VOCAB', 0)
    
    return {
        'marker_counts': marker_counts,
        'marker_examples': marker_examples,
        'marker_words': marker_words,
        'category_totals': {
            'fluency_issues': fluency_total,
            'grammar_errors': grammar_total,
            'simple_vocabulary': vocab_simple,
            'complex_vocabulary': vocab_complex,
            'vocab_sophistication_ratio': vocab_complex / (vocab_simple + vocab_complex) if (vocab_simple + vocab_complex) > 0 else 0
        }
    }

def generate_comprehensive_analysis_report(annotated_transcript, original_transcript):
    """Generate the most comprehensive analysis combining manual counts, lexical diversity, and linguistic metrics"""
    import re
    
    if not annotated_transcript:
        return "No annotated transcript provided."
    
    # Get all three types of analysis
    linguistic_metrics = calculate_linguistic_metrics(original_transcript)
    marker_analysis = analyze_annotation_markers(annotated_transcript)
    lexical_diversity = calculate_advanced_lexical_diversity(original_transcript)
    
    # Calculate rates per 100 words
    total_words = linguistic_metrics.get('total_words', 0)
    
    report_lines = []
    report_lines.append("=" * 100)
    report_lines.append("COMPREHENSIVE SPEECH ANALYSIS REPORT")
    report_lines.append("Combining Manual Counts + Advanced Lexical Diversity + Linguistic Metrics")
    report_lines.append("=" * 100)
    report_lines.append("")
    
    # SECTION 1: BASIC STATISTICS
    report_lines.append("1. BASIC STATISTICS:")
    report_lines.append(f"   • Total words: {total_words}")
    report_lines.append(f"   • Total sentences: {linguistic_metrics.get('total_sentences', 0)}")
    report_lines.append(f"   • Unique words: {linguistic_metrics.get('unique_words', 0)}")
    report_lines.append(f"   • MLU (words): {linguistic_metrics.get('mlu_words', 0):.2f}")
    report_lines.append(f"   • MLU (morphemes): {linguistic_metrics.get('mlu_morphemes', 0):.2f}")
    report_lines.append(f"   • Average sentence length: {linguistic_metrics.get('avg_sentence_length', 0):.2f}")
    report_lines.append("")
    
    # SECTION 2: ADVANCED LEXICAL DIVERSITY MEASURES
    report_lines.append("2. ADVANCED LEXICAL DIVERSITY MEASURES:")
    if lexical_diversity.get('library_available', False) and 'diversity_measures' in lexical_diversity:
        measures = lexical_diversity['diversity_measures']
        report_lines.append(f"   • Token count for analysis: {lexical_diversity.get('token_count', 0)}")
        report_lines.append("")
        report_lines.append("   RECOMMENDED MEASURES:")
        
        if measures.get('root_ttr') is not None:
            report_lines.append(f"   • Root TTR: {measures['root_ttr']:.4f}")
        if measures.get('log_ttr') is not None:
            report_lines.append(f"   • Log TTR: {measures['log_ttr']:.4f}")
        if measures.get('maas_ttr') is not None:
            report_lines.append(f"   • Maas TTR: {measures['maas_ttr']:.4f}")
        if measures.get('hdd') is not None:
            report_lines.append(f"   • HDD (Hypergeometric Distribution D): {measures['hdd']:.4f}")
        
        report_lines.append("")
        report_lines.append("   MOVING WINDOW MEASURES:")
        if measures.get('msttr_25') is not None:
            report_lines.append(f"   • MSTTR (25-word window): {measures['msttr_25']:.4f}")
        if measures.get('msttr_50') is not None:
            report_lines.append(f"   • MSTTR (50-word window): {measures['msttr_50']:.4f}")
        if measures.get('mattr_25') is not None:
            report_lines.append(f"   • MATTR (25-word window): {measures['mattr_25']:.4f}")
        if measures.get('mattr_50') is not None:
            report_lines.append(f"   • MATTR (50-word window): {measures['mattr_50']:.4f}")
        
        report_lines.append("")
        report_lines.append("   MTLD MEASURES:")
        if measures.get('mtld') is not None:
            report_lines.append(f"   • MTLD: {measures['mtld']:.4f}")
        if measures.get('mtld_ma_wrap') is not None:
            report_lines.append(f"   • MTLD (moving average, wrap): {measures['mtld_ma_wrap']:.4f}")
        if measures.get('mtld_ma_bid') is not None:
            report_lines.append(f"   • MTLD (moving average, bidirectional): {measures['mtld_ma_bid']:.4f}")
        
        report_lines.append("")
        report_lines.append("   COMPARISON MEASURE:")
        report_lines.append(f"   • Simple TTR (not recommended): {measures.get('simple_ttr', 0):.4f}")
        
    else:
        report_lines.append("   ⚠️ Advanced lexical diversity measures not available")
        if 'error' in lexical_diversity:
            report_lines.append(f"   Error: {lexical_diversity['error']}")
    
    report_lines.append("")
    
    # SECTION 3: MANUAL ANNOTATION COUNTS
    report_lines.append("3. MANUAL ANNOTATION COUNTS:")
    marker_counts = marker_analysis['marker_counts']
    marker_words = marker_analysis['marker_words']
    
    # Group markers by category for organized reporting
    categories = {
        'FLUENCY MARKERS': ['FILLER', 'FALSE_START', 'REPETITION', 'REVISION', 'PAUSE'],
        'WORD RETRIEVAL MARKERS': ['CIRCUMLOCUTION', 'INCOMPLETE', 'GENERIC', 'WORD_SEARCH'],
        'GRAMMAR MARKERS': ['GRAM_ERROR', 'SYNTAX_ERROR', 'MORPH_ERROR', 'RUN_ON'],
        'VOCABULARY MARKERS': ['SIMPLE_VOCAB', 'COMPLEX_VOCAB', 'SEMANTIC_ERROR'],
        'PRAGMATIC MARKERS': ['TOPIC_SHIFT', 'TANGENT', 'INAPPROPRIATE', 'COHERENCE_BREAK', 'PRONOUN_REF'],
        'SENTENCE COMPLEXITY MARKERS': ['SIMPLE_SENT', 'COMPLEX_SENT', 'COMPOUND_SENT', 'FIGURATIVE'],
        'OTHER MARKERS': ['MAZING', 'PERSEVERATION']
    }
    
    for category, markers in categories.items():
        category_total = sum(marker_counts.get(marker, 0) for marker in markers)
        if category_total > 0:
            report_lines.append(f"   {category}:")
            
            for marker in markers:
                count = marker_counts.get(marker, 0)
                if count > 0:
                    rate = (count / total_words * 100) if total_words > 0 else 0
                    words_list = marker_words.get(marker, [])
                    
                    report_lines.append(f"     • {marker}: {count} instances ({rate:.2f} per 100 words)")
                    
                    if words_list:
                        # Count frequency of each word
                        word_freq = {}
                        for word in words_list:
                            word_freq[word] = word_freq.get(word, 0) + 1
                        
                        # Sort by frequency
                        sorted_words = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)
                        word_summary = []
                        for word, freq in sorted_words[:8]:  # Top 8 most frequent
                            if freq > 1:
                                word_summary.append(f'"{word}" ({freq}x)')
                            else:
                                word_summary.append(f'"{word}"')
                        
                        report_lines.append(f"       Words: {', '.join(word_summary)}")
            
            report_lines.append(f"     CATEGORY TOTAL: {category_total} instances")
            report_lines.append("")
    
    # SECTION 4: SUMMARY STATISTICS
    report_lines.append("4. SUMMARY STATISTICS:")
    category_totals = marker_analysis['category_totals']
    
    fluency_total = category_totals['fluency_issues']
    grammar_total = category_totals['grammar_errors']
    simple_vocab = category_totals['simple_vocabulary']
    complex_vocab = category_totals['complex_vocabulary']
    
    if total_words > 0:
        report_lines.append(f"   • Total fluency issues: {fluency_total} ({fluency_total/total_words*100:.2f} per 100 words)")
        report_lines.append(f"   • Total grammar errors: {grammar_total} ({grammar_total/total_words*100:.2f} per 100 words)")
        report_lines.append(f"   • Simple vocabulary: {simple_vocab} ({simple_vocab/total_words*100:.2f} per 100 words)")
        report_lines.append(f"   • Complex vocabulary: {complex_vocab} ({complex_vocab/total_words*100:.2f} per 100 words)")
        
        if simple_vocab + complex_vocab > 0:
            vocab_ratio = complex_vocab / (simple_vocab + complex_vocab)
            report_lines.append(f"   • Vocabulary sophistication ratio: {vocab_ratio:.3f}")
    
    # SECTION 5: WORD FREQUENCY ANALYSIS
    word_freq = linguistic_metrics.get('word_frequency', {})
    if word_freq:
        report_lines.append("")
        report_lines.append("5. MOST FREQUENT WORDS:")
        for i, (word, freq) in enumerate(list(word_freq.items())[:15], 1):
            percentage = (freq / total_words * 100) if total_words > 0 else 0
            report_lines.append(f"   {i:2d}. '{word}': {freq} times ({percentage:.1f}%)")
    
    report_lines.append("")
    report_lines.append("=" * 100)
    report_lines.append("END OF COMPREHENSIVE ANALYSIS REPORT")
    report_lines.append("=" * 100)
    
    return '\n'.join(report_lines)

def generate_manual_count_report(annotated_transcript):
    """Generate a basic manual count report (legacy function for compatibility)"""
    return generate_comprehensive_analysis_report(annotated_transcript, annotated_transcript)

def process_file(file):
    """Process uploaded transcript file"""
    if file is None:
        return "Please upload a file first."
    
    try:
        with open(file.name, 'r', encoding='utf-8', errors='ignore') as f:
            content = f.read()
        
        if not content.strip():
            return "File appears to be empty."
            
        return content
    except Exception as e:
        return f"Error reading file: {str(e)}"

def segment_response_by_sections(response_text):
    """Segment response by section titles and return a dictionary of sections"""
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    sections = {}
    current_section = None
    current_content = []
    
    lines = response_text.split('\n')
    
    for line in lines:
        # Check if this line is a section header
        is_section_header = False
        for section in required_sections:
            if section in line:
                # Save previous section if exists
                if current_section and current_content:
                    sections[current_section] = '\n'.join(current_content).strip()
                
                # Start new section
                current_section = section
                current_content = []
                is_section_header = True
                break
        
        # If not a section header, add to current section content
        if not is_section_header and current_section:
            current_content.append(line)
    
    # Save the last section
    if current_section and current_content:
        sections[current_section] = '\n'.join(current_content).strip()
    
    return sections

def combine_sections_smartly(sections_dict):
    """Combine sections in the correct order without duplicates"""
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    combined_parts = []
    combined_parts.append("COMPREHENSIVE SPEECH SAMPLE ANALYSIS")
    combined_parts.append("")
    
    for section in required_sections:
        if section in sections_dict:
            combined_parts.append(section)
            combined_parts.append("")
            combined_parts.append(sections_dict[section])
            combined_parts.append("")
    
    return '\n'.join(combined_parts)

def call_claude_api_with_continuation(prompt):
    """Call Claude API with smart continuation system - unlimited continuations until complete"""
    if not ANTHROPIC_API_KEY:
        return "❌ Claude API key not configured. Please set ANTHROPIC_API_KEY environment variable."
    
    # Define all required sections
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    # Safety limits to prevent infinite loops
    MAX_CONTINUATIONS = 30  # Increased from 20 to 30 API calls
    MAX_TIME_MINUTES = 15   # Increased from 10 to 15 minutes total
    MIN_PROGRESS_PER_CALL = 0  # Changed from 1 to 0 to allow more flexibility
    
    try:
        all_sections = {}  # Store all sections found across all parts
        continuation_count = 0
        start_time = time.time()
        last_section_count = 0  # Track progress between calls
        
        # Add continuation instruction to original prompt
        initial_prompt = prompt + "\n\nCRITICAL INSTRUCTIONS: You MUST complete ALL 12 sections of the analysis. If your response is cut off or incomplete, end with <CONTINUE> to indicate more content is needed. Do not skip any sections. Use the checklist to ensure all sections are completed."
        
        while True:  # Unlimited continuations until complete
            if continuation_count == 0:
                current_prompt = initial_prompt
            else:
                # For continuations, provide context about what was already covered
                missing_sections = [s for s in required_sections if s not in all_sections]
                missing_text = "\n".join([f"- {section}" for section in missing_sections])
                
                current_prompt = prompt + f"\n\nCONTINUATION {continuation_count + 1}: The following sections are STILL MISSING and MUST be completed:\n\n{missing_text}\n\nCRITICAL: Provide ONLY these missing sections. Do not repeat any sections that are already complete. Focus exclusively on the missing sections listed above. Complete ALL missing sections in this response."
            
            headers = {
                "Content-Type": "application/json",
                "x-api-key": ANTHROPIC_API_KEY,
                "anthropic-version": "2023-06-01"
            }
            
            data = {
                "model": "claude-3-5-sonnet-20241022",
                "max_tokens": 4096,
                "messages": [
                    {
                        "role": "user",
                        "content": current_prompt
                    }
                ]
            }
            
            response = requests.post(
                "https://api.anthropic.com/v1/messages",
                headers=headers,
                json=data,
                timeout=90
            )
            
            if response.status_code == 200:
                response_json = response.json()
                response_text = response_json['content'][0]['text']
                
                # Log response for debugging
                print(f"\n=== PART {continuation_count + 1} RESPONSE ===")
                print(f"Length: {len(response_text)} characters")
                print(f"Contains CONTINUE: {'<CONTINUE>' in response_text}")
                print(f"First 200 chars: {response_text[:200]}...")
                print(f"Last 200 chars: {response_text[-200:]}...")
                print("=" * 50)
                
                # Segment this part and add new sections to our collection
                part_sections = segment_response_by_sections(response_text)
                for section, content in part_sections.items():
                    if section not in all_sections:  # Only add if not already present
                        all_sections[section] = content
                        print(f"Added section: {section}")
                    else:
                        print(f"Skipped duplicate section: {section}")
                
                # Check completion status
                completed_sections = len(all_sections)
                missing_sections = [s for s in required_sections if s not in all_sections]
                
                print(f"Completed sections: {completed_sections}/12")
                print(f"Missing sections: {missing_sections}")
                
                # Check if response indicates continuation is needed
                needs_continuation = "<CONTINUE>" in response_text
                
                print(f"Needs continuation: {needs_continuation}")
                print(f"Continuation count: {continuation_count}")
                
                # Safety checks to prevent infinite loops
                current_time = time.time()
                elapsed_minutes = (current_time - start_time) / 60
                current_section_count = len(all_sections)
                progress_made = current_section_count - last_section_count
                
                # Check if we're making progress
                if continuation_count > 0 and progress_made < MIN_PROGRESS_PER_CALL:
                    # Only stop if we've made multiple calls with no progress
                    if continuation_count > 3:  # Allow more attempts before giving up
                        logger.warning(f"No progress made in last call (added {progress_made} sections). Stopping to prevent infinite loop.")
                        break
                    else:
                        logger.info(f"No progress in call {continuation_count}, but continuing to allow more attempts...")
                
                # Check time limit
                if elapsed_minutes > MAX_TIME_MINUTES:
                    logger.warning(f"Time limit exceeded ({elapsed_minutes:.1f} minutes). Stopping to prevent excessive API usage.")
                    break
                
                # Check continuation limit
                if continuation_count >= MAX_CONTINUATIONS:
                    logger.warning(f"Continuation limit reached ({MAX_CONTINUATIONS} calls). Stopping to prevent excessive API usage.")
                    break
                
                # Continue if <CONTINUE> is present and safety checks pass
                if needs_continuation:
                    continuation_count += 1
                    last_section_count = current_section_count
                    logger.info(f"Continuing analysis (attempt {continuation_count}/{MAX_CONTINUATIONS}, {elapsed_minutes:.1f} minutes elapsed)")
                    continue
                else:
                    break
            else:
                logger.error(f"Claude API error: {response.status_code} - {response.text}")
                return f"❌ Claude API Error: {response.status_code}"
                
    except Exception as e:
        logger.error(f"Error calling Claude API: {str(e)}")
        return f"❌ Error: {str(e)}"
    
    # Combine all sections in the correct order
    final_response = combine_sections_smartly(all_sections)
    
    # Log final results
    print(f"\n=== FINAL SMART VALIDATION ===")
    print(f"Total sections found: {len(all_sections)}")
    print(f"All sections present: {len(all_sections) == 12}")
    print(f"Missing sections: {[s for s in required_sections if s not in all_sections]}")
    print(f"Total time: {(time.time() - start_time) / 60:.1f} minutes")
    print(f"Total API calls: {continuation_count + 1}")
    print("=" * 50)
    
    # Add completion indicator with safety info
    if continuation_count > 0:
        final_response += f"\n\n[Analysis completed in {continuation_count + 1} parts over {(time.time() - start_time) / 60:.1f} minutes]"
    
    # Add warning if incomplete due to safety limits
    if len(all_sections) < 12:
        missing_sections = [s for s in required_sections if s not in all_sections]
        final_response += f"\n\n⚠️ WARNING: Analysis incomplete due to safety limits. Missing sections: {', '.join(missing_sections)}"
        final_response += f"\n\n💡 TIP: Try running the analysis again, or use the 'Targeted Analysis' tab to focus on specific areas."
        final_response += f"\nThe 'Quick Questions' tab may also provide faster results for specific areas of interest."
    
    return final_response

def analyze_with_backup(annotated_transcript, original_transcript, age, gender, slp_notes):
    """Analyze annotated transcript with original as backup"""
    if not annotated_transcript or len(annotated_transcript.strip()) < 50:
        return "Error: Please provide an annotated transcript for analysis."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    # Calculate quantitative metrics
    linguistic_metrics = calculate_linguistic_metrics(original_transcript)
    marker_analysis = analyze_annotation_markers(annotated_transcript)
    
    # Format metrics for inclusion in prompt
    metrics_text = f"""
    
    CALCULATED LINGUISTIC METRICS:
    - Total Words: {linguistic_metrics.get('total_words', 0)}
    - Total Sentences: {linguistic_metrics.get('total_sentences', 0)}
    - Unique Words: {linguistic_metrics.get('unique_words', 0)}
    - Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - MLU (Words): {linguistic_metrics.get('mlu_words', 0)}
    - MLU (Morphemes): {linguistic_metrics.get('mlu_morphemes', 0)}
    - Average Sentence Length: {linguistic_metrics.get('avg_sentence_length', 0)}
    - Vocabulary Sophistication: {linguistic_metrics.get('vocabulary_sophistication', 0)}
    
    ANNOTATION MARKER COUNTS:
    - Fluency Issues: {marker_analysis.get('category_totals', {}).get('fluency_issues', 0)}
    - Grammar Errors: {marker_analysis.get('category_totals', {}).get('grammar_errors', 0)}
    - Simple Vocabulary: {marker_analysis.get('category_totals', {}).get('simple_vocabulary', 0)}
    - Complex Vocabulary: {marker_analysis.get('category_totals', {}).get('complex_vocabulary', 0)}
    - Vocabulary Sophistication Ratio: {marker_analysis.get('category_totals', {}).get('vocab_sophistication_ratio', 0):.3f}
    """
    
    analysis_prompt = f"""
    You are a speech-language pathologist conducting a COMPREHENSIVE analysis of a word-by-word annotated speech sample. Use the provided quantitative metrics and count EVERY marker precisely.

    Patient: {age}-year-old {gender}
    
    ANNOTATED TRANSCRIPT:
    {annotated_transcript}{notes_section}
    
    ORIGINAL TRANSCRIPT (for reference and backup analysis):
    {original_transcript}
    
    {metrics_text}
    
    ANALYSIS INSTRUCTIONS:
    Using the detailed linguistic markers in the annotated transcript and the calculated metrics above, provide a comprehensive analysis with EXACT counts, percentages, and specific examples. Complete ALL 12 sections below:

    COMPREHENSIVE SPEECH SAMPLE ANALYSIS:

    1. SPEECH FACTORS (with EXACT counts and specific citations):

    A. Fluency Issues:
    - Count [FILLER] markers: List each instance and calculate rate per 100 words
    - Count [FALSE_START] markers: List examples and analyze patterns
    - Count [REPETITION] markers: Categorize by type (word, phrase, sound)
    - Count [REVISION] markers: Analyze self-correction patterns
    - Count [PAUSE] markers: Assess hesitation frequency
    - Calculate total disfluency rate and severity level

    B. Word Retrieval Issues:
    - Count [CIRCUMLOCUTION] markers: List each roundabout description
    - Count [INCOMPLETE] markers: Analyze abandoned thought patterns
    - Count [GENERIC] markers: Calculate specificity ratio
    - Count [WORD_SEARCH] markers: Identify retrieval difficulty areas

    C. Grammatical Errors:
    - Count [GRAM_ERROR] markers by subcategory (verb tense, subject-verb agreement, etc.)
    - Count [SYNTAX_ERROR] markers: Analyze word order problems
    - Count [MORPH_ERROR] markers: Categorize morphological mistakes
    - Count [RUN_ON] markers: Assess sentence boundary awareness

    2. LANGUAGE SKILLS ASSESSMENT (with specific evidence):

    A. Lexical/Semantic Skills:
    - Use calculated Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - Count [SIMPLE_VOCAB] vs [COMPLEX_VOCAB] markers
    - Assess vocabulary sophistication ratio: {marker_analysis.get('category_totals', {}).get('vocab_sophistication_ratio', 0):.3f}
    - Count [SEMANTIC_ERROR] markers and analyze patterns

    B. Syntactic Skills:
    - Count [SIMPLE_SENT], [COMPLEX_SENT], [COMPOUND_SENT] markers
    - Calculate sentence complexity ratios
    - Assess clause complexity and embedding

    C. Supralinguistic Skills:
    - Identify cause-effect relationships, inferences, non-literal language
    - Assess problem-solving language and metalinguistic awareness

    3. COMPLEX SENTENCE ANALYSIS (with exact counts):

    A. Coordinating Conjunctions:
    - Count and cite EVERY use of: and, but, or, so, yet, for, nor
    - Analyze patterns and age-appropriateness

    B. Subordinating Conjunctions:
    - Count and cite EVERY use of: because, although, while, since, if, when, where, that, which, who
    - Analyze clause complexity and embedding depth

    C. Sentence Structure Analysis:
    - Use calculated MLU: {linguistic_metrics.get('mlu_words', 0)} words, {linguistic_metrics.get('mlu_morphemes', 0)} morphemes
    - Calculate complexity ratios and assess developmental appropriateness

    4. FIGURATIVE LANGUAGE ANALYSIS (with exact counts):

    A. Similes and Metaphors:
    - Count [FIGURATIVE] markers for similes (using "like" or "as")
    - Count [FIGURATIVE] markers for metaphors (direct comparisons)

    B. Idioms and Non-literal Language:
    - Count and analyze idiomatic expressions
    - Assess comprehension and appropriate use

    5. PRAGMATIC LANGUAGE ASSESSMENT (with specific examples):

    A. Discourse Management:
    - Count [TOPIC_SHIFT] markers: Assess transition appropriateness
    - Count [TANGENT] markers: Analyze tangential speech patterns
    - Count [COHERENCE_BREAK] markers: Assess logical flow

    B. Referential Communication:
    - Count [PRONOUN_REF] markers: Analyze referential clarity
    - Assess communicative effectiveness

    6. VOCABULARY AND SEMANTIC ANALYSIS (with quantification):

    A. Vocabulary Diversity:
    - Total words: {linguistic_metrics.get('total_words', 0)}
    - Unique words: {linguistic_metrics.get('unique_words', 0)}
    - Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - Vocabulary sophistication: {linguistic_metrics.get('vocabulary_sophistication', 0)}

    B. Semantic Relationships:
    - Analyze word frequency patterns
    - Assess semantic precision and relationships

    7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS (with counts):

    A. Morphological Markers:
    - Count [MORPH_ERROR] markers and categorize
    - Analyze morpheme use patterns
    - Assess morphological complexity

    B. Phonological Patterns:
    - Identify speech sound patterns from transcript
    - Assess syllable structure complexity

    8. COGNITIVE-LINGUISTIC FACTORS (with evidence):

    A. Working Memory:
    - Assess sentence length complexity using average: {linguistic_metrics.get('avg_sentence_length', 0)} words
    - Analyze information retention patterns

    B. Processing Efficiency:
    - Analyze linguistic complexity and word-finding patterns
    - Assess cognitive demands of language structures

    C. Executive Function:
    - Count self-correction patterns ([REVISION] markers)
    - Assess planning and organization in discourse

    9. FLUENCY AND RHYTHM ANALYSIS (with quantification):

    A. Disfluency Patterns:
    - Total fluency issues: {marker_analysis.get('category_totals', {}).get('fluency_issues', 0)}
    - Calculate disfluency rate per 100 words
    - Analyze impact on communication

    B. Language Flow:
    - Assess sentence length variability: std = {linguistic_metrics.get('sentence_length_std', 0)}
    - Analyze linguistic markers of hesitation

    10. QUANTITATIVE METRICS:
    - Total words: {linguistic_metrics.get('total_words', 0)}
    - Total sentences: {linguistic_metrics.get('total_sentences', 0)}
    - MLU (words): {linguistic_metrics.get('mlu_words', 0)}
    - MLU (morphemes): {linguistic_metrics.get('mlu_morphemes', 0)}
    - Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - Grammar error rate: Calculate from marker counts
    - Vocabulary sophistication ratio: {marker_analysis.get('category_totals', {}).get('vocab_sophistication_ratio', 0):.3f}

    11. CLINICAL IMPLICATIONS:
    - Primary strengths: List with supporting evidence from markers and metrics
    - Primary weaknesses: Rank by severity with exact counts
    - Intervention priorities: Based on error frequency and impact
    - Therapy targets: Specific, measurable goals

    12. PROGNOSIS AND SUMMARY:
    - Overall communication profile with percentile estimates
    - Developmental appropriateness assessment
    - Summary of key findings from quantitative analysis
    - Priority treatment goals and expected outcomes

    CRITICAL REQUIREMENTS:
    - Use the provided calculated metrics in your analysis
    - Provide EXACT counts for every marker type
    - Calculate precise percentages and show your work
    - Give specific examples from the transcript
    - If annotation is incomplete, supplement with analysis of the original transcript
    - Complete ALL 12 sections - use <CONTINUE> if needed
    """
    
    return call_claude_api_with_continuation(analysis_prompt)

def full_analysis_pipeline(transcript_content, age, gender, slp_notes, progress_callback=None):
    """Complete pipeline: annotate then analyze with progressive updates"""
    if not transcript_content or len(transcript_content.strip()) < 50:
        return "Error: Please provide a longer transcript for analysis.", ""
    
    # Step 1: Annotate transcript
    logger.info("Step 1: Annotating transcript with linguistic markers...")
    if progress_callback:
        progress_callback("🏷️ Step 1: Annotating transcript with linguistic markers...")
    
    annotated_transcript = annotate_transcript(transcript_content, age, gender, slp_notes)
    
    if annotated_transcript.startswith("❌"):
        return annotated_transcript, ""
    
    # Return annotated transcript immediately
    if progress_callback:
        progress_callback("✅ Step 1 Complete: Annotation finished! Starting analysis...")
    
    # Check if annotation was incomplete
    if annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
        logger.warning("Annotation incomplete, proceeding with analysis using original transcript as primary source")
        analysis_note = "⚠️ Note: Annotation was incomplete. Analysis primarily based on original transcript.\n\n"
    else:
        analysis_note = ""
    
    # Step 2: Analyze annotated transcript with original as backup
    logger.info("Step 2: Analyzing annotated transcript...")
    if progress_callback:
        progress_callback("📊 Step 2: Analyzing annotated transcript (this may take several minutes)...")
    
    analysis_result = analyze_with_backup(annotated_transcript, transcript_content, age, gender, slp_notes)
    
    if progress_callback:
        progress_callback("✅ Analysis Complete!")
    
    return annotated_transcript, analysis_note + analysis_result

def progressive_analysis_pipeline(transcript_content, age, gender, slp_notes):
    """Generator function for progressive analysis updates"""
    if not transcript_content or len(transcript_content.strip()) < 50:
        yield "Error: Please provide a longer transcript for analysis.", "", "❌ Error"
        return
    
    # Step 1: Annotate transcript
    logger.info("Step 1: Annotating transcript with linguistic markers...")
    yield "", "", "🏷️ Step 1: Annotating transcript with linguistic markers..."
    
    annotated_transcript = annotate_transcript(transcript_content, age, gender, slp_notes)
    
    if annotated_transcript.startswith("❌"):
        yield annotated_transcript, "", "❌ Annotation failed"
        return
    
    # Return annotated transcript immediately after completion
    yield annotated_transcript, "", "✅ Step 1 Complete! Starting analysis..."
    
    # Check if annotation was incomplete
    if annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
        logger.warning("Annotation incomplete, proceeding with analysis")
        analysis_note = "⚠️ Note: Annotation was incomplete. Analysis primarily based on original transcript.\n\n"
        yield annotated_transcript, "", "⚠️ Annotation incomplete, continuing with analysis..."
    else:
        analysis_note = ""
    
    # Step 2: Analyze annotated transcript
    logger.info("Step 2: Analyzing annotated transcript...")
    yield annotated_transcript, "", "📊 Step 2: Analyzing annotated transcript (this may take several minutes)..."
    
    analysis_result = analyze_with_backup(annotated_transcript, transcript_content, age, gender, slp_notes)
    
    # Final result
    yield annotated_transcript, analysis_note + analysis_result, "✅ Analysis Complete!"

# Example transcript data
example_transcript = """Well, um, I was thinking about, you know, the thing that happened yesterday. I was go- I mean I was going to the store and, uh, I seen this really big dog. Actually, it was more like a wolf or something. The dog, he was just standing there, and I thought to myself, "That's one magnificent creature." But then, um, I realized I forgot my wallet at home, so I had to turn around and go back. When I got home, my wife she says to me, "Where's the groceries?" And I'm like, "Well, honey, I had to come back because I forgot my thing." She wasn't too happy about that, let me tell you. Anyway, speaking of dogs, did I ever tell you about the time I went fishing? It was raining cats and dogs that day, and I caught three fishes. My brother, he don't like fishing much, but he came with me anyway. We was sitting there for hours, just waiting and waiting. The fish, they wasn't biting at all. But then, all of a sudden, I got a bite! I was so excited, I almost falled into the water. The fish was huge - well, maybe not huge, but pretty big for that lake. We cooked it up real good that night. My wife, she made some of that fancy stuff to go with it. What do you call it... that green thing... oh yeah, asparagus. She's always making these elaborate meals. Sometimes I think she tries too hard, you know? But I appreciate it. Life's been good to us, I guess. We been married for twenty-five years now. Time flies when you're having fun, as they say."""

example_annotated = """Well[FILLER], um[FILLER], I was thinking about, you[SIMPLE_VOCAB] know[FILLER], the thing[GENERIC] that happened yesterday[SIMPLE_VOCAB]. I was go-[FALSE_START] I mean I was going[SIMPLE_VOCAB] to the store[SIMPLE_VOCAB] and, uh[FILLER], I seen[GRAM_ERROR] this really big[SIMPLE_VOCAB] dog[SIMPLE_VOCAB].[SIMPLE_SENT] Actually, it was more like[FILLER] a wolf[SIMPLE_VOCAB] or something[GENERIC].[SIMPLE_SENT] The dog[SIMPLE_VOCAB], he[PRONOUN_REF] was just standing[SIMPLE_VOCAB] there, and I thought to myself, "That's one magnificent[COMPLEX_VOCAB] creature[COMPLEX_VOCAB]."[COMPLEX_SENT] But then, um[FILLER], I realized[COMPLEX_VOCAB] I forgot[SIMPLE_VOCAB] my wallet[SIMPLE_VOCAB] at home[SIMPLE_VOCAB], so I had to turn around and go[SIMPLE_VOCAB] back[SIMPLE_VOCAB].[COMPLEX_SENT] When I got home, my wife[SIMPLE_VOCAB] she[REPETITION] says[SIMPLE_VOCAB] to me, "Where's the groceries[SIMPLE_VOCAB]?"[COMPLEX_SENT] And I'm like[FILLER], "Well[FILLER], honey[SIMPLE_VOCAB], I had to come back because I forgot[SIMPLE_VOCAB] my thing[GENERIC]."[COMPLEX_SENT] She wasn't too happy[SIMPLE_VOCAB] about that, let me tell you.[SIMPLE_SENT] Anyway[TOPIC_SHIFT], speaking of dogs, did I ever tell you about the time I went fishing?[TANGENT][COMPLEX_SENT] It was raining cats and dogs[FIGURATIVE] that day, and I caught[SIMPLE_VOCAB] three fishes[MORPH_ERROR].[COMPOUND_SENT] My brother[SIMPLE_VOCAB], he[PRONOUN_REF] don't[GRAM_ERROR] like fishing[SIMPLE_VOCAB] much, but he came with me anyway[SIMPLE_VOCAB].[COMPLEX_SENT] We was[GRAM_ERROR] sitting[SIMPLE_VOCAB] there for hours[SIMPLE_VOCAB], just waiting[SIMPLE_VOCAB] and waiting[REPETITION].[SIMPLE_SENT] The fish[SIMPLE_VOCAB], they[PRONOUN_REF] wasn't[GRAM_ERROR] biting[SIMPLE_VOCAB] at all.[SIMPLE_SENT] But then, all of a sudden[SIMPLE_VOCAB], I got[SIMPLE_VOCAB] a bite[SIMPLE_VOCAB]![SIMPLE_SENT] I was so excited[SIMPLE_VOCAB], I almost falled[MORPH_ERROR] into the water[SIMPLE_VOCAB].[COMPLEX_SENT] The fish[SIMPLE_VOCAB] was huge[SIMPLE_VOCAB] - well[FILLER], maybe not huge[SIMPLE_VOCAB], but pretty big[SIMPLE_VOCAB] for that lake[SIMPLE_VOCAB].[REVISION][COMPLEX_SENT] We cooked[SIMPLE_VOCAB] it up real good[SIMPLE_VOCAB] that night[SIMPLE_VOCAB].[SIMPLE_SENT] My wife[SIMPLE_VOCAB], she[REPETITION] made some of that fancy[SIMPLE_VOCAB] stuff[GENERIC] to go[SIMPLE_VOCAB] with it.[SIMPLE_SENT] What do you call it... [WORD_SEARCH] that green[SIMPLE_VOCAB] thing[GENERIC]... [PAUSE] oh yeah, asparagus[COMPLEX_VOCAB].[CIRCUMLOCUTION] She's always making[SIMPLE_VOCAB] these elaborate[COMPLEX_VOCAB] meals[SIMPLE_VOCAB].[SIMPLE_SENT] Sometimes I think[SIMPLE_VOCAB] she tries[SIMPLE_VOCAB] too hard[SIMPLE_VOCAB], you know[FILLER]?[COMPLEX_SENT] But I appreciate[COMPLEX_VOCAB] it.[SIMPLE_SENT] Life's been good[SIMPLE_VOCAB] to us, I guess[SIMPLE_VOCAB].[SIMPLE_SENT] We been[GRAM_ERROR] married[SIMPLE_VOCAB] for twenty-five[COMPLEX_VOCAB] years[SIMPLE_VOCAB] now.[SIMPLE_SENT] Time flies when you're having fun[FIGURATIVE], as they say.[COMPLEX_SENT]"""

# Create Gradio interface
with gr.Blocks(title="Speech Analysis", theme=gr.themes.Soft()) as demo:
    gr.Markdown("""
    # 📋 Speech Analysis Tool with Annotations
    
    This tool performs a two-step comprehensive speech analysis:
    1. **Annotation**: Marks linguistic features in the transcript
    2. **Analysis**: Counts and analyzes the marked features for detailed assessment
    
    Upload a transcript or paste text below to begin the analysis.
    """)
    
    with gr.Tab("📝 Full Analysis Pipeline"):
        gr.Markdown("### Complete two-step analysis: annotation followed by comprehensive analysis")
        
        with gr.Row():
            with gr.Column(scale=2):
                transcript_input = gr.Textbox(
                    label="Speech Transcript",
                    placeholder="Paste the speech transcript here...",
                    lines=10,
                    max_lines=20
                )
                
                file_input = gr.File(
                    label="Or upload transcript file",
                    file_types=[".txt", ".doc", ".docx"]
                )
                
                with gr.Row():
                    age_input = gr.Textbox(
                        label="Age",
                        placeholder="e.g., 45",
                        value="45"
                    )
                    gender_input = gr.Dropdown(
                        label="Gender",
                        choices=["Male", "Female", "Other"],
                        value="Male"
                    )
                
                slp_notes_input = gr.Textbox(
                    label="SLP Clinical Notes (Optional)",
                    placeholder="Add any relevant clinical observations...",
                    lines=3
                )
                
                example_btn = gr.Button("📄 Load Example Transcript", variant="secondary", size="sm")
                
                # Single main analysis button
                ultimate_analysis_btn = gr.Button("🚀 Run Complete Speech Analysis", variant="primary", size="lg")
            
            with gr.Column(scale=3):
                status_display = gr.Markdown("Ready to analyze transcript")
                
                annotated_output = gr.Textbox(
                    label="Step 1: Annotated Transcript (✓ = Complete, ⚠️ = Incomplete)",
                    lines=15,
                    max_lines=25,
                    show_copy_button=True
                )
                
                analysis_output = gr.Textbox(
                    label="Step 2: Comprehensive Analysis",
                    lines=20,
                    max_lines=30,
                    show_copy_button=True
                )
    
    with gr.Tab("🏷️ Annotation Only"):
        gr.Markdown("### Step 1: Annotate transcript with linguistic markers")
        
        with gr.Row():
            with gr.Column():
                transcript_input_2 = gr.Textbox(
                    label="Speech Transcript",
                    placeholder="Paste the speech transcript here...",
                    lines=10
                )
                
                with gr.Row():
                    age_input_2 = gr.Textbox(label="Age", value="45")
                    gender_input_2 = gr.Dropdown(
                        label="Gender",
                        choices=["Male", "Female", "Other"],
                        value="Male"
                    )
                
                slp_notes_input_2 = gr.Textbox(
                    label="SLP Clinical Notes (Optional)",
                    lines=3
                )
                
                example_btn_2 = gr.Button("📄 Load Example Transcript", variant="secondary", size="sm")
                annotate_btn = gr.Button("🏷️ Annotate Transcript", variant="secondary")
            
            with gr.Column():
                annotation_output = gr.Textbox(
                    label="Annotated Transcript (✓ = Complete, ⚠️ = Incomplete)",
                    lines=20,
                    show_copy_button=True
                )
    

    with gr.Tab("❓ Quick Questions"):
        gr.Markdown("### Ask specific questions about the transcript")
        
        with gr.Row():
            with gr.Column():
                transcript_input_4 = gr.Textbox(
                    label="Speech Transcript",
                    placeholder="Paste the speech transcript here...",
                    lines=8
                )
                
                question_input = gr.Textbox(
                    label="Your Question",
                    placeholder="e.g., How many filler words are used? What grammatical errors are present?",
                    lines=2
                )
                
                with gr.Row():
                    age_input_4 = gr.Textbox(label="Age", value="45")
                    gender_input_4 = gr.Dropdown(
                        label="Gender",
                        choices=["Male", "Female", "Other"],
                        value="Male"
                    )
                
                slp_notes_input_4 = gr.Textbox(
                    label="SLP Clinical Notes (Optional)",
                    lines=2
                )
                
                # Quick question examples
                gr.Markdown("**Example Questions:**")
                with gr.Row():
                    q1_btn = gr.Button("Count filler words", size="sm", variant="secondary")
                    q2_btn = gr.Button("Grammar errors?", size="sm", variant="secondary")
                    q3_btn = gr.Button("Vocabulary level?", size="sm", variant="secondary")
                
                with gr.Row():
                    q4_btn = gr.Button("Sentence complexity?", size="sm", variant="secondary")
                    q5_btn = gr.Button("Word finding issues?", size="sm", variant="secondary")
                    q6_btn = gr.Button("Fluency problems?", size="sm", variant="secondary")
                
                example_btn_4 = gr.Button("📄 Load Example Transcript", variant="secondary", size="sm")
                ask_question_btn = gr.Button("❓ Ask Question", variant="primary")
            
            with gr.Column():
                question_output = gr.Textbox(
                    label="Answer",
                    lines=15,
                    show_copy_button=True
                )
    
    with gr.Tab("🎯 Targeted Analysis"):
        gr.Markdown("### Focus on specific areas of speech and language")
        
        with gr.Row():
            with gr.Column():
                transcript_input_5 = gr.Textbox(
                    label="Speech Transcript",
                    placeholder="Paste the speech transcript here...",
                    lines=8
                )
                
                analysis_area = gr.Dropdown(
                    label="Analysis Focus",
                    choices=[
                        "Fluency and Disfluencies",
                        "Grammar and Syntax", 
                        "Vocabulary and Semantics",
                        "Pragmatics and Discourse",
                        "Sentence Complexity",
                        "Word Finding and Retrieval"
                    ],
                    value="Fluency and Disfluencies"
                )
                
                with gr.Row():
                    age_input_5 = gr.Textbox(label="Age", value="45")
                    gender_input_5 = gr.Dropdown(
                        label="Gender",
                        choices=["Male", "Female", "Other"],
                        value="Male"
                    )
                
                slp_notes_input_5 = gr.Textbox(
                    label="SLP Clinical Notes (Optional)",
                    lines=2
                )
                
                example_btn_5 = gr.Button("📄 Load Example Transcript", variant="secondary", size="sm")
                targeted_analysis_btn = gr.Button("🎯 Run Targeted Analysis", variant="primary")
            
            with gr.Column():
                targeted_output = gr.Textbox(
                    label="Targeted Analysis Results",
                    lines=15,
                    show_copy_button=True
                )
    
    # Event handlers - now all components are defined
    example_btn.click(fn=lambda: example_transcript, outputs=[transcript_input])
    example_btn_2.click(fn=lambda: example_transcript, outputs=[transcript_input_2])
    example_btn_4.click(fn=lambda: example_transcript, outputs=[transcript_input_4])
    example_btn_5.click(fn=lambda: example_transcript, outputs=[transcript_input_5])
    
    # Quick question button handlers
    q1_btn.click(fn=lambda: "How many filler words (um, uh, like, you know) are used in this transcript? Provide exact counts and examples.", outputs=[question_input])
    q2_btn.click(fn=lambda: "What grammatical errors are present in this transcript? List all errors with specific examples and corrections.", outputs=[question_input])
    q3_btn.click(fn=lambda: "What is the vocabulary level and sophistication in this transcript? Analyze word choice and complexity.", outputs=[question_input])
    q4_btn.click(fn=lambda: "How complex are the sentences in this transcript? Analyze sentence types and structures used.", outputs=[question_input])
    q5_btn.click(fn=lambda: "Are there any word-finding difficulties or retrieval issues? Identify specific examples and patterns.", outputs=[question_input])
    q6_btn.click(fn=lambda: "What fluency problems or disfluencies are present? Count and categorize all instances.", outputs=[question_input])
    
    file_input.change(
        fn=process_file,
        inputs=[file_input],
        outputs=[transcript_input]
    )
    
    def run_annotation_step(transcript_content, age, gender, slp_notes):
        """Run just the annotation step and return immediately"""
        if not transcript_content or len(transcript_content.strip()) < 50:
            return "Error: Please provide a longer transcript for annotation.", "❌ Error"
        
        logger.info("Step 1: Annotating transcript with linguistic markers...")
        annotated_transcript = annotate_transcript(transcript_content, age, gender, slp_notes)
        
        if annotated_transcript.startswith("❌"):
            return annotated_transcript, "❌ Annotation failed"
        elif annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
            return annotated_transcript, "⚠️ Annotation incomplete but proceeding"
        else:
            return annotated_transcript, "✅ Annotation complete! Click 'Run Analysis' to continue."
    
    def run_analysis_step(annotated_transcript, original_transcript, age, gender, slp_notes):
        """Run the analysis step on the annotated transcript"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for analysis."
        
        logger.info("Step 2: Analyzing annotated transcript...")
        
        # Check if annotation was incomplete
        if annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
            analysis_note = "⚠️ Note: Annotation was incomplete. Analysis primarily based on original transcript.\n\n"
        else:
            analysis_note = ""
        
        analysis_result = analyze_with_backup(annotated_transcript, original_transcript, age, gender, slp_notes)
        return analysis_note + analysis_result
    
    def run_manual_count_only(annotated_transcript):
        """Generate only the manual count report without AI analysis"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for manual counting."
        
        return generate_manual_count_report(annotated_transcript)
    
    def run_verified_analysis(annotated_transcript, original_transcript, age, gender, slp_notes):
        """Run analysis with manual count verification"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for analysis."
        
        # Generate comprehensive analysis report first
        comprehensive_report = generate_comprehensive_analysis_report(annotated_transcript, original_transcript)
        
        # Get all the verified data
        marker_analysis = analyze_annotation_markers(annotated_transcript)
        linguistic_metrics = calculate_linguistic_metrics(original_transcript)
        lexical_diversity = calculate_advanced_lexical_diversity(original_transcript)
        
        # Create a comprehensive verified analysis prompt
        verified_prompt = f"""
        You are a speech-language pathologist conducting analysis based on COMPREHENSIVE VERIFIED DATA. 
        Do NOT recount anything - use ONLY the provided verified measurements below.

        Patient: {age}-year-old {gender}
        
        COMPREHENSIVE VERIFIED ANALYSIS DATA (DO NOT RECOUNT):
        {comprehensive_report}
        
        ANNOTATED TRANSCRIPT (for examples only, do not recount):
        {annotated_transcript}...
        
        INSTRUCTIONS: 
        Use ONLY the verified data provided above. Do NOT count or calculate anything yourself.
        
        Provide a comprehensive clinical interpretation organized into these sections:

        1. LEXICAL DIVERSITY INTERPRETATION:
        - Interpret the advanced lexical diversity measures (MTLD, HDD, MATTR, etc.)
        - Compare to age-appropriate norms
        - Clinical significance of diversity patterns

        2. FLUENCY PATTERN ANALYSIS:
        - Clinical interpretation of fluency marker counts and rates
        - Severity assessment based on verified counts
        - Impact on communication effectiveness

        3. GRAMMATICAL COMPETENCE ASSESSMENT:
        - Analysis of grammar error patterns from verified counts
        - Developmental appropriateness
        - Areas of strength vs. weakness

        4. VOCABULARY AND SEMANTIC ANALYSIS:
        - Interpretation of vocabulary sophistication measures
        - Word frequency pattern analysis
        - Semantic precision assessment

        5. PRAGMATIC LANGUAGE EVALUATION:
        - Discourse coherence based on verified markers
        - Social communication effectiveness
        - Conversational competence

        6. OVERALL COMMUNICATION PROFILE:
        - Integration of all verified measures
        - Strengths and areas of need
        - Functional communication impact

        7. CLINICAL RECOMMENDATIONS:
        - Specific intervention targets based on verified data
        - Therapy approaches and techniques
        - Progress monitoring suggestions
        - Prognosis and expected outcomes

        Focus on INTERPRETATION and CLINICAL SIGNIFICANCE, not counting. 
        All measurements are already verified and accurate.
        Cite specific examples from the transcript to support your interpretations.
        """
        
        ai_interpretation = call_claude_api(verified_prompt)
        
        return f"{comprehensive_report}\n\n{'='*100}\nCLINICAL INTERPRETATION BASED ON COMPREHENSIVE VERIFIED DATA\n{'='*100}\n\n{ai_interpretation}"
    
    def run_ultimate_analysis(annotated_transcript, original_transcript, age, gender, slp_notes):
        """The ultimate analysis: gather all statistical data, then do final 12-section clinical analysis"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for analysis."
        
        # STEP 1: Gather ALL statistical data
        linguistic_metrics = calculate_linguistic_metrics(original_transcript)
        marker_analysis = analyze_annotation_markers(annotated_transcript)
        lexical_diversity = calculate_advanced_lexical_diversity(original_transcript)
        
        # STEP 2: Get AI clinical insights (for interpretation, not counting)
        ai_clinical_insights = analyze_with_backup(annotated_transcript, original_transcript, age, gender, slp_notes)
        
        # STEP 3: Prepare all verified statistical values for final prompt
        stats_summary = f"""
        VERIFIED STATISTICAL VALUES (DO NOT RECOUNT - USE THESE EXACT NUMBERS):
        
        BASIC METRICS:
        • Total words: {linguistic_metrics.get('total_words', 0)}
        • Total sentences: {linguistic_metrics.get('total_sentences', 0)}
        • Unique words: {linguistic_metrics.get('unique_words', 0)}
        • MLU (words): {linguistic_metrics.get('mlu_words', 0):.2f}
        • MLU (morphemes): {linguistic_metrics.get('mlu_morphemes', 0):.2f}
        • Average sentence length: {linguistic_metrics.get('avg_sentence_length', 0):.2f}
        • Sentence length std: {linguistic_metrics.get('sentence_length_std', 0):.2f}
        
        LEXICAL DIVERSITY MEASURES (from lexical-diversity library):"""
        
        if lexical_diversity.get('library_available', False) and 'diversity_measures' in lexical_diversity:
            measures = lexical_diversity['diversity_measures']
            stats_summary += f"""
        • Simple TTR: {measures.get('simple_ttr', 'N/A')}
        • Root TTR: {measures.get('root_ttr', 'N/A')}
        • Log TTR: {measures.get('log_ttr', 'N/A')}
        • Maas TTR: {measures.get('maas_ttr', 'N/A')}
        • HDD: {measures.get('hdd', 'N/A')}
        • MSTTR (25-word): {measures.get('msttr_25', 'N/A')}
        • MSTTR (50-word): {measures.get('msttr_50', 'N/A')}
        • MATTR (25-word): {measures.get('mattr_25', 'N/A')}
        • MATTR (50-word): {measures.get('mattr_50', 'N/A')}
        • MTLD: {measures.get('mtld', 'N/A')}
        • MTLD (MA wrap): {measures.get('mtld_ma_wrap', 'N/A')}
        • MTLD (MA bidirectional): {measures.get('mtld_ma_bid', 'N/A')}"""
        else:
            stats_summary += "\n        • Lexical diversity measures not available"
        
        # Add manual annotation counts
        marker_counts = marker_analysis['marker_counts']
        category_totals = marker_analysis['category_totals']
        total_words = linguistic_metrics.get('total_words', 0)
        
        stats_summary += f"""
        
        MANUAL ANNOTATION COUNTS:
        • FILLER markers: {marker_counts.get('FILLER', 0)} ({marker_counts.get('FILLER', 0)/total_words*100:.2f} per 100 words)
        • FALSE_START markers: {marker_counts.get('FALSE_START', 0)}
        • REPETITION markers: {marker_counts.get('REPETITION', 0)}
        • REVISION markers: {marker_counts.get('REVISION', 0)}
        • PAUSE markers: {marker_counts.get('PAUSE', 0)}
        • GRAM_ERROR markers: {marker_counts.get('GRAM_ERROR', 0)}
        • SYNTAX_ERROR markers: {marker_counts.get('SYNTAX_ERROR', 0)}
        • MORPH_ERROR markers: {marker_counts.get('MORPH_ERROR', 0)}
        • SIMPLE_VOCAB markers: {marker_counts.get('SIMPLE_VOCAB', 0)}
        • COMPLEX_VOCAB markers: {marker_counts.get('COMPLEX_VOCAB', 0)}
        • SIMPLE_SENT markers: {marker_counts.get('SIMPLE_SENT', 0)}
        • COMPLEX_SENT markers: {marker_counts.get('COMPLEX_SENT', 0)}
        • COMPOUND_SENT markers: {marker_counts.get('COMPOUND_SENT', 0)}
        • FIGURATIVE markers: {marker_counts.get('FIGURATIVE', 0)}
        • PRONOUN_REF markers: {marker_counts.get('PRONOUN_REF', 0)}
        • TOPIC_SHIFT markers: {marker_counts.get('TOPIC_SHIFT', 0)}
        • TANGENT markers: {marker_counts.get('TANGENT', 0)}
        • CIRCUMLOCUTION markers: {marker_counts.get('CIRCUMLOCUTION', 0)}
        • GENERIC markers: {marker_counts.get('GENERIC', 0)}
        • WORD_SEARCH markers: {marker_counts.get('WORD_SEARCH', 0)}
        
        CATEGORY TOTALS:
        • Total fluency issues: {category_totals['fluency_issues']} ({category_totals['fluency_issues']/total_words*100:.2f} per 100 words)
        • Total grammar errors: {category_totals['grammar_errors']} ({category_totals['grammar_errors']/total_words*100:.2f} per 100 words)
        • Vocabulary sophistication ratio: {category_totals['vocab_sophistication_ratio']:.3f}
        """
        
        # STEP 4: Create the final comprehensive prompt
        final_prompt = f"""
        You are a speech-language pathologist conducting the FINAL COMPREHENSIVE 12-SECTION SPEECH ANALYSIS.

        Patient: {age}-year-old {gender}
        
        {stats_summary}
        
        CLINICAL INSIGHTS FROM AI ANALYSIS (for interpretation guidance):
        {ai_clinical_insights[:4000]}...
        
        ANNOTATED TRANSCRIPT (for specific examples):
        {annotated_transcript}
        
        CRITICAL INSTRUCTIONS:
        1. Use ONLY the verified statistical values provided above - DO NOT recount anything
        2. Use the clinical insights for interpretation guidance
        3. Use the annotated transcript for specific examples and quotes
        4. Complete ALL 12 sections of the comprehensive analysis
        
        COMPREHENSIVE SPEECH SAMPLE ANALYSIS:
        
        CRITICAL: Provide EXTENSIVE detail with ALL possible examples for each category. Quote liberally from the transcript and provide comprehensive breakdowns.

        1. SPEECH FACTORS (with EXHAUSTIVE detail and ALL examples):
        
        A. Fluency Issues:
        - Filler words (total: [count]):
          * "um" ([count]): "Um, it has two..."
          * "like" ([count]): "is like looks silver", "like fits people" 
          * "I don't know" ([count]): "I don't know how to say..."
          * Other fillers: [list with counts]
        - False starts/self-corrections ([count]):
          * "My bike is like looks silver"
          * [List other examples]
        - Repetitions ([count]):
          * Word repetitions: "golf cart(s)" (Xx), "back" (Xx)
          * [List other patterns]

        B. Word Retrieval Issues:
        - Circumlocution ([count]):
          * "this type of fish in it" (for "anchovies")
          * "where you go golfing and the golf clubs are in back"
        - Incomplete thoughts ([count]):
          * "Like I've seen like a I don't know..."
        - Word-finding pauses ([count]): [brief description]
        - Generic language ([count]): "thing," "stuff," "something"

        C. Grammatical Errors:
        - Subject-verb agreement ([count]): "there is another type"
        - Verb tense errors ([count]): [examples]
        - Pronoun errors ([count]): [examples]
        - Run-on sentences ([count]): [examples]

        2. LANGUAGE SKILLS ASSESSMENT (with comprehensive evidence):
        
        A. Lexical/Semantic Skills:
        - Type-Token Ratio: [number] unique words/[number] total words
        - Vocabulary examples:
          * Advanced vocabulary: "churrasco," "lo mein," "anchovies"
          * Transportation terms: "bike," "golf cart," "wheels"
          * Food terms: "caesar salad," "hummus," "pita bread"
        - Semantic relationships:
          * Categories: [examples of categorization]
          * Part-whole: bike parts, food components
          * Cause-effect: "pumped up → feels better"

        B. Syntactic Skills:
        - Sentence types:
          * Simple sentences: [count]
          * Compound sentences: [count] 
          * Complex sentences: [count]
        - MLU: [number] words, [number] morphemes
        - Average sentence length: [number] words

        C. Supralinguistic Skills:
        - Cause-effect relationships ([count]):
          * "If you do not have a golf cart driving license you can get busted"
          * "It feels much better now that it's pumped"
        - Inferences: [count with examples]
        - Problem-solving language: [count with examples]

        3. COMPLEX SENTENCE ANALYSIS (with ALL examples and counts):
        
        A. Coordinating Conjunctions:
        - "and": [count]
        - "but": [count] 
        - "or": [count]
        - "so": [count]
        - "because": [count]

        B. Subordinating Conjunctions:
        - "because": [count]
        - "when": [count]
        - "if": [count] 
        - "that": [count]
        - "where": [count]

        C. Sentence Structure Analysis:
        - Average sentence length: [number] words
        - Sentence complexity: [brief description of patterns]

        4. FIGURATIVE LANGUAGE ANALYSIS (with ALL examples):
        
        A. Similes and Metaphors:
        - Similes ([count]): [examples]
        - Metaphors ([count]): [examples]
        - "Like" as filler vs. comparison: [brief analysis]

        B. Idioms and Non-literal Language:
        - Idioms ([count]): "get busted"
        - Colloquialisms ([count]): [examples]

        5. PRAGMATIC LANGUAGE ASSESSMENT (with detailed examples):
        
        A. Discourse Management:
        - Topic shifts: [count] - quote ALL transitions:
          * Bike → Golf carts: Quote exact transition
          * Golf carts → Food: Quote exact transition
          * Food → Cookies: Quote exact transition
        - Topic maintenance analysis:
          * Golf cart topic: [X utterances] - quote entire sequence
          * Food topic: [X utterances] - quote entire sequence
        - Topic elaboration: Count details provided per topic

        B. Referential Communication:
        - Pronoun reference errors: [count] - quote ALL unclear references
        - Demonstrative use: [count] - quote ALL "this," "that" uses
        - Referential clarity: Analyze with specific examples

        6. VOCABULARY AND SEMANTIC ANALYSIS (comprehensive breakdown):
        
        A. Vocabulary Diversity:
        - ALL lexical diversity measures with interpretations:
          * Simple TTR: [number] - age comparison
          * MTLD: [number] - clinical interpretation
          * HDD: [number] - vocabulary range assessment
          * MATTR: [number] - moving average interpretation
        - Most frequent words: List top 20 with frequencies
        - Vocabulary sophistication by domain with examples

        B. Semantic Relationships:
        - Word associations: Analyze patterns with examples
        - Semantic categories: List ALL categories used
        - Synonym/antonym use: Quote ALL instances
        - Semantic precision: Analyze accuracy with examples

        7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS (detailed breakdown):
        
        A. Morphological Markers:
        - Plurals: [count] - list ALL regular and irregular examples
        - Verb tenses: Break down by type with ALL examples:
          * Present tense: List 20+ examples
          * Past tense regular: List ALL examples
          * Past tense irregular: List ALL examples
        - Progressive forms: [count] - list ALL "-ing" examples
        - Possessives: [count] - list ALL examples
        - Compound words: [count] - list ALL examples
        - Derivational morphemes: [count] - list ALL prefixes/suffixes

        B. Phonological Patterns:
        - Articulation accuracy: Note any sound errors
        - Syllable structure: Analyze complexity with examples
        - Prosodic patterns: Describe rhythm and stress

        8. COGNITIVE-LINGUISTIC FACTORS (with specific evidence):
        
        A. Working Memory:
        - Longest successful utterance: [word count] - quote entire utterance
        - Complex information management: Quote examples of multi-part descriptions
        - Information retention across narrative: Analyze with examples

        B. Processing Speed:
        - Word-finding efficiency: [count delays] - quote ALL instances
        - Response fluency: Analyze patterns with examples
        - Processing load indicators: Identify with examples

        C. Executive Function:
        - Self-monitoring: [count] - quote ALL self-correction instances
        - Planning evidence: Analyze organization with examples
        - Cognitive flexibility: Analyze topic management with examples

        9. FLUENCY AND RHYTHM ANALYSIS (comprehensive measurement):
        
        A. Disfluency Patterns:
        - Total disfluency count: [number] with rate per 100 words
        - Disfluency types breakdown with ALL examples
        - Severity assessment compared to age norms
        - Impact on communication effectiveness

        B. Language Flow:
        - Natural pause patterns: [count] - identify ALL appropriate pauses
        - Disrupted flow instances: [count] - quote ALL with analysis
        - Rhythm variation: Describe patterns with examples

        10. QUANTITATIVE METRICS (report ALL data with calculations shown):
        - Total words: [count] (method of counting explained)
        - Total sentences: [count] (criteria for sentence boundaries)
        - Unique words: [count] (how uniqueness determined)
        - MLU words: [calculation] ([total words]/[utterances])
        - MLU morphemes: [calculation] ([total morphemes]/[utterances])
        - ALL lexical diversity measures: [list with values and interpretations]
        - Error rates: [calculations] for each error type
        - Age-appropriate comparisons for ALL measures

        11. CLINICAL IMPLICATIONS (evidence-based with priorities):
        
        A. Strengths (ranked with evidence):
        1. [Primary strength] - specific evidence with quotes and data
        2. [Secondary strength] - specific evidence with quotes and data
        3. [Continue ranking ALL strengths with supporting evidence]

        B. Areas of Need (prioritized by severity):
        1. [Highest priority] - severity data, impact analysis, examples
        2. [Second priority] - severity data, impact analysis, examples
        3. [Continue with ALL areas needing intervention]

        C. Treatment Recommendations (specific and measurable):
        1. [Primary goal] - specific techniques, frequency, duration, success criteria
        2. [Secondary goal] - specific techniques, frequency, duration, success criteria
        3. [Continue with ALL treatment recommendations]

        12. PROGNOSIS AND SUMMARY (comprehensive profile):
        - Overall severity rating: [level] with detailed justification
        - Developmental appropriateness: Compare ALL skills to age expectations
        - Functional communication impact: Real-world implications
        - Prognosis: Specific predictions with timelines and success indicators
        - Monitoring plan: Specific measures and reassessment schedule

        12. PROGNOSIS AND SUMMARY:
        Overall profile based on comprehensive verified data

        REQUIREMENTS:
        - Complete ALL 12 sections
        - Use ONLY verified statistical values (never recount)
        - Cite specific examples from annotated transcript
        - Provide clinical interpretation of the verified data
        - If response is cut off, end with <CONTINUE>
        """
        
        # STEP 5: Get the final comprehensive analysis
        final_result = call_claude_api_with_continuation(final_prompt)
        
        return final_result
    
    def run_full_pipeline(transcript_content, age, gender, slp_notes):
        """Run the complete pipeline but return annotation immediately"""
        if not transcript_content or len(transcript_content.strip()) < 50:
            return "Error: Please provide a longer transcript for analysis.", "", "❌ Error"
        
        # Step 1: Get annotation
        annotated_transcript, annotation_status = run_annotation_step(transcript_content, age, gender, slp_notes)
        
        if annotated_transcript.startswith("❌"):
            return annotated_transcript, "", annotation_status
        
        # Step 2: Run analysis
        analysis_result = run_analysis_step(annotated_transcript, transcript_content, age, gender, slp_notes)
        
        return annotated_transcript, analysis_result, "✅ Complete analysis finished!"
    
    def run_complete_speech_analysis(transcript_content, age, gender, slp_notes):
        """Run the complete speech analysis pipeline with ultimate analysis"""
        if not transcript_content or len(transcript_content.strip()) < 50:
            return "Error: Please provide a longer transcript for analysis.", "", "❌ Error"
        
        # Step 1: Annotate transcript
        annotated_transcript, annotation_status = run_annotation_step(transcript_content, age, gender, slp_notes)
        
        if annotated_transcript.startswith("❌"):
            return annotated_transcript, "", annotation_status
        
        # Step 2: Run ultimate analysis
        ultimate_result = run_ultimate_analysis(annotated_transcript, transcript_content, age, gender, slp_notes)
        
        return annotated_transcript, ultimate_result, "✅ Complete speech analysis finished!"
    
    # Single main event handler
    ultimate_analysis_btn.click(
        fn=run_complete_speech_analysis,
        inputs=[transcript_input, age_input, gender_input, slp_notes_input],
        outputs=[annotated_output, analysis_output, status_display]
    )
    
    annotate_btn.click(
        fn=annotate_transcript,
        inputs=[transcript_input_2, age_input_2, gender_input_2, slp_notes_input_2],
        outputs=[annotation_output]
    )
    

    
    # Quick Questions event handler
    ask_question_btn.click(
        fn=answer_quick_question,
        inputs=[transcript_input_4, question_input, age_input_4, gender_input_4, slp_notes_input_4],
        outputs=[question_output]
    )
    
    # Targeted Analysis event handler
    targeted_analysis_btn.click(
        fn=analyze_targeted_area,
        inputs=[transcript_input_5, analysis_area, age_input_5, gender_input_5, slp_notes_input_5],
        outputs=[targeted_output]
    )

if __name__ == "__main__":
    demo.launch(
        server_name="0.0.0.0",
        server_port=7860,
        share=True,
        show_error=True
    )