Upload 21 files

.gitattributes

@@ -53,4 +53,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -textdocs/assets/app_screenshot.png filter=lfs diff=lfs merge=lfs -text

__init__.py

@@ -0,0 +1 @@
+# Root package initialization

agent.py

@@ -0,0 +1,515 @@
+"""
+Agent module for the Fake News Detector application.
+
+This module implements a LangGraph-based agent that orchestrates
+the fact-checking process. It defines the agent setup, tools,
+and processing pipeline for claim verification.
+"""
+
+import os
+import time
+import logging
+import traceback
+import json
+from langchain_core.tools import tool
+from langchain.prompts import PromptTemplate
+from langgraph.prebuilt import create_react_agent
+
+from utils.models import get_llm_model
+from utils.performance import PerformanceTracker
+from modules.claim_extraction import extract_claims
+from modules.evidence_retrieval import retrieve_combined_evidence
+from modules.classification import classify_with_llm, aggregate_evidence
+from modules.explanation import generate_explanation
+
+# Configure logger
+logger = logging.getLogger("misinformation_detector")
+
+# Reference to global performance tracker
+performance_tracker = PerformanceTracker()
+
+# Define LangGraph Tools
+@tool
+def claim_extractor(query):
+    """
+    Tool that extracts factual claims from a given text.
+    
+    Args:
+        query (str): Text containing potential factual claims
+        
+    Returns:
+        str: Extracted factual claim
+    """
+    performance_tracker.log_claim_processed()
+    return extract_claims(query)
+
+@tool
+def evidence_retriever(query):
+    """
+    Tool that retrieves evidence from multiple sources for a claim.
+    
+    Args:
+        query (str): The factual claim to gather evidence for
+        
+    Returns:
+        list: List of evidence items from various sources
+    """
+    return retrieve_combined_evidence(query)
+
+@tool
+def truth_classifier(query, evidence):
+    """
+    Tool that classifies the truthfulness of a claim based on evidence.
+    
+    This function analyzes the provided evidence to determine if a claim is true,
+    false, or uncertain. It implements a weighted scoring approach considering
+    both the number of supporting/contradicting evidence items and their quality.
+    
+    Args:
+        query (str): The factual claim to classify
+        evidence (list): Evidence items to evaluate against the claim
+        
+    Returns:
+        str: JSON string containing verdict, confidence, and classification results
+             with a guaranteed structure for consistent downstream processing
+    """
+    # Perform classification on the evidence
+    classification_results = classify_with_llm(query, evidence)
+    
+    # Aggregate results to determine overall verdict and confidence
+    truth_label, confidence = aggregate_evidence(classification_results)
+    
+    # Debug logging
+    logger.info(f"Classification results: {len(classification_results)} items")
+    logger.info(f"Aggregate result: {truth_label}, confidence: {confidence}")
+    
+    # Ensure truth_label is never None
+    if not truth_label:
+        truth_label = "Uncertain"
+        confidence = 0.0
+    
+    # Return a structured dictionary with all needed information
+    result = {
+        "verdict": truth_label,
+        "confidence": confidence,
+        "results": classification_results
+    }
+    
+    # Convert to JSON string for consistent handling
+    return json.dumps(result)
+
+@tool
+def explanation_generator(claim, evidence_results, truth_label):
+    """
+    Tool that generates a human-readable explanation for the verdict.
+    
+    This function creates a clear, natural language explanation of why a claim
+    was classified as true, false, or uncertain based on the evidence. It handles
+    various truth label formats and extracts appropriate confidence values.
+    
+    Args:
+        claim (str): The factual claim being verified
+        evidence_results (list): Evidence items and classification results
+        truth_label (str): The verdict (True/False/Uncertain), which may come
+                          in different formats
+        
+    Returns:
+        str: Natural language explanation of the verdict with confidence
+             framing and evidence citations
+             
+    Note:
+        The function extracts confidence values from evidence when available
+        or uses appropriate defaults based on the verdict type. It includes
+        robust error handling to ensure explanations are always generated,
+        even in edge cases.
+    """
+    try:
+        # Extract confidence if available in evidence_results
+        confidence = None
+        if isinstance(evidence_results, list) and evidence_results and isinstance(evidence_results[0], dict):
+            # Try to get confidence from results
+            confidence_values = [result.get('confidence', 0) for result in evidence_results if 'confidence' in result]
+            if confidence_values:
+                confidence = max(confidence_values)
+        
+        # If confidence couldn't be extracted, use a default value based on the verdict
+        if confidence is None:
+            if truth_label and ("True" in truth_label or "False" in truth_label):
+                confidence = 0.7  # Default for definitive verdicts
+            else:
+                confidence = 0.5  # Default for uncertain verdicts
+        
+        # Generate the explanation
+        explanation = generate_explanation(claim, evidence_results, truth_label, confidence)
+        logger.info(f"Generated explanation: {explanation[:100]}...")
+        return explanation
+    except Exception as e:
+        logger.error(f"Error generating explanation: {str(e)}")
+        # Provide a fallback explanation with basic information
+        return f"The claim '{claim}' has been evaluated as {truth_label}. The available evidence provides {confidence or 'moderate'} confidence in this assessment. For more detailed information, please review the evidence provided."
+    
+def setup_agent():
+    """
+    Create and configure a ReAct agent with the fact-checking tools.
+    
+    This function configures a LangGraph ReAct agent with all the
+    necessary tools for fact checking, including claim extraction,
+    evidence retrieval, classification, and explanation generation.
+    
+    Returns:
+        object: Configured LangGraph agent ready for claim processing
+        
+    Raises:
+        ValueError: If OpenAI API key is not set
+    """
+    # Make sure OpenAI API key is set
+    if "OPENAI_API_KEY" not in os.environ or not os.environ["OPENAI_API_KEY"].strip():
+        logger.error("OPENAI_API_KEY environment variable not set or empty.")
+        raise ValueError("OpenAI API key is required")
+
+    # Define tools with any customizations
+    tools = [
+        claim_extractor,
+        evidence_retriever,
+        truth_classifier,
+        explanation_generator
+    ]
+
+    # Define the prompt template with clearer, more efficient instructions
+    FORMAT_INSTRUCTIONS_TEMPLATE = """
+    Use the following format:
+    Question: the input question you must answer
+    Action: the action to take, should be one of: {tool_names}
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Action/Action Input/Observation can repeat N times)
+    Final Answer: the final answer to the original input question
+    """
+    
+    prompt = PromptTemplate(
+        input_variables=["input", "tool_names"],
+        template=f"""
+        You are a fact-checking assistant that verifies claims by gathering evidence and 
+        determining their truthfulness. Follow these exact steps in sequence:
+
+        1. Call claim_extractor to extract the main factual claim
+        2. Call evidence_retriever to gather evidence about the claim
+        3. Call truth_classifier to evaluate the claim using the evidence
+        4. Call explanation_generator to explain the result
+        5. Provide your Final Answer that summarizes everything
+
+        Execute these steps in order without unnecessary thinking steps between tool calls.
+        Be direct and efficient in your verification process.
+        
+        {FORMAT_INSTRUCTIONS_TEMPLATE}
+        """
+    )
+    
+    try:
+        # Get the LLM model
+        model = get_llm_model()
+        
+        # Create the agent with a shorter timeout
+        graph = create_react_agent(model, tools=tools)
+        logger.info("Agent created successfully")
+        return graph
+    except Exception as e:
+        logger.error(f"Error creating agent: {str(e)}")
+        raise e
+
+def process_claim(claim, agent=None, recursion_limit=20):
+    """
+    Process a claim to determine its truthfulness using the agent.
+    
+    This function invokes the LangGraph agent to process a factual claim,
+    extract supporting evidence, evaluate the claim's truthfulness, and
+    generate a human-readable explanation.
+    
+    Args:
+        claim (str): The factual claim to be verified
+        agent (object, optional): Initialized LangGraph agent. If None, an error is logged.
+        recursion_limit (int, optional): Maximum recursion depth for agent. Default: 20.
+            Higher values allow more complex reasoning but increase processing time.
+            
+    Returns:
+        dict: Result dictionary containing:
+            - claim: Extracted factual claim
+            - evidence: List of evidence pieces
+            - evidence_count: Number of evidence pieces
+            - classification: Verdict (True/False/Uncertain)
+            - confidence: Confidence score (0-1)
+            - explanation: Human-readable explanation of the verdict
+            - final_answer: Final answer from the agent
+            - Or error information if processing failed
+    """
+    if agent is None:
+        logger.error("Agent not initialized. Call setup_agent() first.")
+        return None
+        
+    start_time = time.time()
+    logger.info(f"Processing claim with agent: {claim}")
+    
+    try:
+        # IMPORTANT: Create fresh inputs for each claim
+        # This ensures we don't carry over state from previous claims
+        inputs = {"messages": [("user", claim)]}
+        
+        # Set configuration - reduced recursion limit for faster processing
+        config = {"recursion_limit": recursion_limit}
+        
+        # Invoke the agent
+        response = agent.invoke(inputs, config)
+        
+        # Format the response
+        result = format_response(response)
+        
+        # Log performance
+        elapsed = time.time() - start_time
+        logger.info(f"Claim processed in {elapsed:.2f} seconds")
+        
+        return result
+        
+    except Exception as e:
+        logger.error(f"Error processing claim with agent: {str(e)}")
+        logger.error(traceback.format_exc())
+        return {"error": str(e)}
+
+def format_response(response):
+    """
+    Format the agent's response into a structured result.
+    
+    This function extracts key information from the agent's response,
+    including the claim, evidence, classification, and explanation.
+    It also performs error handling and provides fallback values.
+    
+    Args:
+        response (dict): Raw response from the LangGraph agent
+        
+    Returns:
+        dict: Structured result containing claim verification data
+    """
+    try:
+        if not response or "messages" not in response:
+            return {"error": "Invalid response format"}
+            
+        messages = response.get("messages", [])
+        
+        # Initialize result container with default values
+        result = {
+            "claim": None,
+            "evidence": [],
+            "evidence_count": 0,
+            "classification": "Uncertain",
+            "confidence": 0.0,  # Default zero confidence
+            "explanation": "Insufficient evidence to evaluate this claim.",
+            "final_answer": None,
+            "thoughts": []
+        }
+        
+        # Track if we found results from each tool
+        found_tools = {
+            "claim_extractor": False,
+            "evidence_retriever": False,
+            "truth_classifier": False,
+            "explanation_generator": False
+        }
+        
+        # Extract information from messages
+        tool_outputs = {}
+        
+        for idx, message in enumerate(messages):
+            # Extract agent thoughts
+            if hasattr(message, "content") and getattr(message, "type", "") == "assistant":
+                content = message.content
+                if "Thought:" in content:
+                    thought_parts = content.split("Thought:", 1)
+                    if len(thought_parts) > 1:
+                        thought = thought_parts[1].split("\n")[0].strip()
+                        result["thoughts"].append(thought)
+            
+            # Extract tool outputs
+            if hasattr(message, "type") and message.type == "tool":
+                tool_name = getattr(message, "name", "unknown")
+                
+                # Store tool outputs
+                tool_outputs[tool_name] = message.content
+                
+                # Extract specific information
+                if tool_name == "claim_extractor":
+                    found_tools["claim_extractor"] = True
+                    if message.content:
+                        result["claim"] = message.content
+                    
+                elif tool_name == "evidence_retriever":
+                    found_tools["evidence_retriever"] = True
+                    # Handle string representation of a list
+                    if message.content:
+                        if isinstance(message.content, list):
+                            result["evidence"] = message.content
+                            result["evidence_count"] = len(message.content)
+                        elif isinstance(message.content, str) and message.content.startswith("[") and message.content.endswith("]"):
+                            try:
+                                import ast
+                                parsed_content = ast.literal_eval(message.content)
+                                if isinstance(parsed_content, list):
+                                    result["evidence"] = parsed_content
+                                    result["evidence_count"] = len(parsed_content)
+                                else:
+                                    result["evidence"] = [message.content]
+                                    result["evidence_count"] = 1
+                            except:
+                                result["evidence"] = [message.content]
+                                result["evidence_count"] = 1
+                        else:
+                            result["evidence"] = [message.content]
+                            result["evidence_count"] = 1
+                            logger.warning(f"Evidence retrieved is not a list: {type(message.content)}")
+                        
+                elif tool_name == "truth_classifier":
+                    found_tools["truth_classifier"] = True
+                    
+                    # Log the incoming content for debugging
+                    logger.info(f"Truth classifier content type: {type(message.content)}")
+                    logger.info(f"Truth classifier content: {message.content}")
+                    
+                    # Handle JSON formatted result from truth_classifier
+                    if isinstance(message.content, str):
+                        try:
+                            import json
+                            # Parse the JSON string
+                            parsed_content = json.loads(message.content)
+                            
+                            # Extract the values from the parsed content
+                            result["classification"] = parsed_content.get("verdict", "Uncertain")
+                            result["confidence"] = float(parsed_content.get("confidence", 0.0))
+                            result["classification_results"] = parsed_content.get("results", [])
+                            
+                            # Add low confidence warning for results < 10%
+                            if 0 < result["confidence"] < 0.1:
+                                result["low_confidence_warning"] = True
+                            
+                            logger.info(f"Extracted from JSON: verdict={result['classification']}, confidence={result['confidence']}")
+                        except json.JSONDecodeError:
+                            logger.warning(f"Could not parse truth classifier JSON: {message.content}")
+                        except Exception as e:
+                            logger.warning(f"Error extracting from truth classifier output: {e}")
+                    else:
+                        logger.warning(f"Unexpected truth_classifier content format: {message.content}")
+                        
+                elif tool_name == "explanation_generator":
+                    found_tools["explanation_generator"] = True
+                    if message.content:
+                        result["explanation"] = message.content
+                        logger.info(f"Found explanation from tool: {message.content[:100]}...")
+                    
+            # Get final answer from last message
+            elif idx == len(messages) - 1 and hasattr(message, "content"):
+                result["final_answer"] = message.content
+        
+        # Log which tools weren't found
+        missing_tools = [tool for tool, found in found_tools.items() if not found]
+        if missing_tools:
+            logger.warning(f"Missing tool outputs in response: {', '.join(missing_tools)}")
+        
+        # IMPORTANT: ENHANCED FALLBACK MECHANISM
+        # Always run truth classification if evidence was collected but classifier wasn't called
+        if found_tools["evidence_retriever"] and not found_tools["truth_classifier"]:
+            logger.info("Truth classifier was not called by the agent, executing fallback classification")
+            
+            try:
+                from modules.classification import classify_with_llm, aggregate_evidence
+                
+                # Get the evidence from the results
+                evidence = result["evidence"]
+                claim = result["claim"] or "Unknown claim"
+                
+                # Force classification even with minimal evidence
+                if evidence:
+                    # Classify with available evidence
+                    classification_results = classify_with_llm(claim, evidence)
+                    truth_label, confidence = aggregate_evidence(classification_results)
+                    
+                    # Update result with classification results
+                    result["classification"] = truth_label
+                    result["confidence"] = confidence
+                    result["classification_results"] = classification_results
+                    
+                    # Add low confidence warning if needed
+                    if 0 < confidence < 0.1:
+                        result["low_confidence_warning"] = True
+                    
+                    logger.info(f"Fallback classification: {truth_label}, confidence: {confidence}")
+                else:
+                    # If no evidence at all, maintain uncertain with zero confidence
+                    result["classification"] = "Uncertain"
+                    result["confidence"] = 0.0
+                    logger.info("No evidence available for fallback classification")
+            except Exception as e:
+                logger.error(f"Error in fallback truth classification: {e}")
+        
+        # ENHANCED: Always generate explanation if classification exists but explanation wasn't called
+        if (found_tools["truth_classifier"] or result["classification"] != "Uncertain") and not found_tools["explanation_generator"]:
+            logger.info("Explanation generator was not called by the agent, using fallback explanation generation")
+            
+            try:
+                from modules.explanation import generate_explanation
+                
+                # Get the necessary inputs for explanation generation
+                claim = result["claim"] or "Unknown claim"
+                evidence = result["evidence"]
+                truth_label = result["classification"] 
+                confidence_value = result["confidence"]
+                classification_results = result.get("classification_results", [])
+                
+                # Choose the best available evidence for explanation
+                explanation_evidence = classification_results if classification_results else evidence
+                
+                # Force explanation generation even with minimal evidence
+                explanation = generate_explanation(claim, explanation_evidence, truth_label, confidence_value)
+                
+                # Use the generated explanation
+                if explanation:
+                    logger.info(f"Generated fallback explanation: {explanation[:100]}...")
+                    result["explanation"] = explanation
+            except Exception as e:
+                logger.error(f"Error generating fallback explanation: {e}")
+        
+        # Make sure evidence exists
+        if result["evidence_count"] > 0 and (not result["evidence"] or len(result["evidence"]) == 0):
+            logger.warning("Evidence count is non-zero but evidence list is empty. This is a data inconsistency.")
+            result["evidence_count"] = 0
+        
+        # Add debug info about the final result
+        logger.info(f"Final classification: {result['classification']}, confidence: {result['confidence']}")
+        logger.info(f"Final explanation: {result['explanation'][:100]}...")
+        
+        # Add performance metrics
+        result["performance"] = performance_tracker.get_summary()
+        
+        # Memory management - limit the size of evidence and thoughts
+        # To keep memory usage reasonable for web deployment
+        if "evidence" in result and isinstance(result["evidence"], list):
+            limited_evidence = []
+            for ev in result["evidence"]:
+                if isinstance(ev, str) and len(ev) > 500:
+                    limited_evidence.append(ev[:497] + "...")
+                else:
+                    limited_evidence.append(ev)
+            result["evidence"] = limited_evidence
+            
+        # Limit thoughts to conserve memory
+        if "thoughts" in result and len(result["thoughts"]) > 10:
+            result["thoughts"] = result["thoughts"][:10]
+        
+        return result
+        
+    except Exception as e:
+        logger.error(f"Error formatting agent response: {str(e)}")
+        logger.error(traceback.format_exc())
+        return {
+            "error": str(e), 
+            "traceback": traceback.format_exc(),
+            "classification": "Error",
+            "confidence": 0.0,
+            "explanation": "An error occurred while processing this claim."
+        }

app.py

@@ -0,0 +1,613 @@
+"""
+Main Streamlit application for the Fake News Detector.
+
+This module implements the user interface for claim verification,
+rendering the results and handling user interactions. It also
+manages the application lifecycle including initialization and cleanup.
+"""
+
+import streamlit as st
+import time
+import json
+import os
+import logging
+import atexit
+import sys
+from pathlib import Path
+
+# Configure logging first, before other imports
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[logging.StreamHandler()]
+)
+logger = logging.getLogger("misinformation_detector")
+
+# Check for critical environment variables
+if not os.environ.get("OPENAI_API_KEY"):
+    logger.warning("OPENAI_API_KEY not set. Please configure this in your Hugging Face Spaces secrets.")
+    
+# Import our modules
+from utils.models import initialize_models
+from utils.performance import PerformanceTracker
+
+# Import agent functionality
+import agent
+
+# Initialize performance tracker
+performance_tracker = PerformanceTracker()
+
+# Ensure data directory exists
+data_dir = Path("data")
+if not data_dir.exists():
+    logger.info("Creating data directory")
+    data_dir.mkdir(exist_ok=True)
+
+# Set page configuration
+st.set_page_config(
+    page_title="AskVeracity",
+    page_icon="🔍",
+    layout="wide",
+)
+
+# Hide the "Press ⌘+Enter to apply" text with CSS
+st.markdown("""
+<style>
+/* Hide the shortcut text that appears at the bottom of text areas */
+.stTextArea div:has(textarea) + div {
+    visibility: hidden !important;
+    height: 0px !important;
+    position: absolute !important;
+}
+</style>
+""", unsafe_allow_html=True)
+
+def reset_claim_specific_state():
+    """
+    Reset claim-specific state while preserving model caching.
+    
+    This function resets only the state variables related to the processing
+    of a specific claim, without clearing cached models to maintain efficiency.
+    """
+    logger.info("Resetting claim-specific state")
+    
+    # Reset performance tracker metrics but not the instance itself
+    global performance_tracker
+    performance_tracker.reset()
+    
+    # Clear session state variables related to the current claim
+    if 'result' in st.session_state:
+        st.session_state.result = None
+    
+    if 'has_result' in st.session_state:
+        st.session_state.has_result = False
+        
+    if 'claim_to_process' in st.session_state:
+        st.session_state.claim_to_process = ""
+        
+    # If we already have an agent, keep the instance but ensure it starts fresh
+    if hasattr(st.session_state, 'agent') and st.session_state.agent:
+        # Recreate the agent to ensure fresh state
+        try:
+            logger.info("Refreshing agent state for new claim processing")
+            # We're keeping the cached models but reinitializing the agent
+            st.session_state.agent = agent.setup_agent()
+        except Exception as e:
+            logger.error(f"Error refreshing agent: {e}")
+
+@st.cache_resource
+def get_agent():
+    """
+    Initialize and cache the agent for reuse across requests.
+    
+    This function creates and caches the fact-checking agent to avoid
+    recreating it for every request. It's decorated with st.cache_resource
+    to ensure the agent is only initialized once per session.
+    
+    Returns:
+        object: Initialized LangGraph agent for fact checking
+    """
+    logger.info("Initializing models and agent (cached)")
+    initialize_models()
+    return agent.setup_agent()
+
+def cleanup_resources():
+    """
+    Clean up resources when app is closed.
+    
+    This function is registered with atexit to ensure resources
+    are properly released when the application terminates.
+    """
+    try:
+        # Clear any cached data
+        st.cache_data.clear()
+        
+        # Reset performance tracker
+        performance_tracker.reset()
+        
+        # Log cleanup
+        logger.info("Resources cleaned up successfully")
+    except Exception as e:
+        logger.error(f"Error during cleanup: {e}")
+
+# Register cleanup handler
+atexit.register(cleanup_resources)
+
+# App title and description
+st.title("🔍 AskVeracity")
+st.markdown("""
+            This is a simple AI-powered agentic tool - a fact-checking system that analyzes claims to determine 
+            their truthfulness by gathering and analyzing evidence from various sources, such as Wikipedia, 
+            news outlets, and academic repositories. The application aims to support broader efforts in misinformation detection.
+""")
+
+# Sidebar with app information
+with st.sidebar:
+    st.header("About")
+    st.info(
+        "This system uses a combination of NLP techniques and LLMs to "
+        "extract claims, gather evidence, and classify the truthfulness of statements.\n\n"
+        "**Technical:** Built with Python, Streamlit, LangGraph, and OpenAI, leveraging spaCy for NLP and various APIs for retrieving evidence from diverse sources."
+    )
+    
+    # Application information
+    st.markdown("### How It Works")
+    st.info(
+        "1. Enter any recent news or a factual claim\n"
+        "2. Our AI gathers evidence from Wikipedia, news sources, and academic repositories\n"
+        "3. The system analyzes the evidence to determine truthfulness\n"
+        "4. Results show the verdict with supporting evidence"
+    )
+    
+    # Our Mission
+    st.markdown("### Our Mission")
+    st.info(
+        "AskVeracity aims to combat misinformation in real-time through an open-source application built with accessible tools. "
+        "We believe in empowering people with factual information to make informed decisions."
+    )
+    
+    # Limitations and Usage
+    st.markdown("### Limitations")
+    st.warning(
+        "Due to resource constraints, AskVeracity may not always provide real-time results with perfect accuracy. "
+        "Performance is typically best with widely-reported news and information published within the last 48 hours. "
+        "Additionally, the system evaluates claims based on current evidence - a claim that was true in the past "
+        "may be judged false if circumstances have changed, and vice versa."
+        "Currently, AskVeracity is only available in English."
+    )
+    
+    # Best Practices
+    st.markdown("### Best Practices")
+    st.success(
+        "For optimal results:\n\n"
+        "• Keep claims short and precise\n\n"
+        "• Each part of the claim is important\n\n"        
+        "• Include key details in your claim\n\n"
+        "• Phrase claims as direct statements rather than questions\n\n"
+        "• Be specific about who said what\n\n"
+        "• For very recent announcements or technical features, try checking company blogs, official documentation, or specialized tech news sites directly\n\n"
+        "• If receiving an \"Uncertain\" verdict, try alternative phrasings or more general versions of the claim\n\n"
+        "• Consider that some technical features might be in limited preview programs with minimal public documentation"
+    )
+    
+    # Example comparison
+    with st.expander("📝 Examples of Effective Claims"):
+        st.markdown("""
+        **Less precise:** "Country A-Country B Relations Are Moving in Positive Direction as per Country B Minister John Doe."
+        
+        **More precise:** "Country B's External Affairs Minister John Doe has claimed that Country A-Country B Relations Are Moving in Positive Direction."
+        """)
+    
+    # Important Notes
+    st.markdown("### Important Notes")
+    st.info(
+        "• AskVeracity covers general topics and is not specialized in any single domain or location\n\n"
+        "• Results can vary based on available evidence and LLM behavior\n\n"
+        "• The system is designed to indicate uncertainty when evidence is insufficient\n\n"
+        "• AskVeracity is not a chatbot and does not maintain conversation history\n\n"
+        "• We recommend cross-verifying critical information with additional sources"
+    )
+    
+    # Privacy Information
+    st.markdown("### Data Privacy")
+    st.info(
+        "We do not collect or store any data about the claims you submit. "
+        "Your interactions are processed by OpenAI's API. Please refer to "
+        "[OpenAI's privacy policy](https://openai.com/policies/privacy-policy) for details on their data handling practices."
+    )
+    
+    # Feedback Section
+    st.markdown("### Feedback")
+    st.success(
+        "AskVeracity is evolving and we welcome your feedback to help us improve. "
+        "Please reach out to us with questions, suggestions, or concerns."
+    )
+
+# Initialize session state variables
+if 'processing' not in st.session_state:
+    st.session_state.processing = False
+if 'claim_to_process' not in st.session_state:
+    st.session_state.claim_to_process = ""
+if 'has_result' not in st.session_state:
+    st.session_state.has_result = False
+if 'result' not in st.session_state:
+    st.session_state.result = None
+if 'total_time' not in st.session_state:
+    st.session_state.total_time = 0
+if 'fresh_state' not in st.session_state:
+    st.session_state.fresh_state = True
+# Initialize verify button disabled state
+if 'verify_btn_disabled' not in st.session_state:
+    st.session_state.verify_btn_disabled = False
+# Add a new state to track input content
+if 'input_content' not in st.session_state:
+    st.session_state.input_content = ""
+
+# Main interface
+st.markdown("### Enter a claim to verify")
+
+# Define a callback for input change
+def on_input_change():
+    st.session_state.input_content = st.session_state.claim_input_area
+
+# Input area with callback - key fix here!
+claim_input = st.text_area("", 
+                         value=st.session_state.input_content,
+                         height=100,
+                         placeholder=(
+                             "Examples: The Eiffel Tower is located in Rome, Italy. "
+                             "Meta recently released its Llama 4 large language model. "
+                             "Justin Trudeau is not the Canadian Prime Minister anymore. "
+                             "A recent piece of news."
+                         ),
+                         key="claim_input_area",
+                         on_change=on_input_change,
+                         label_visibility="collapsed", 
+                         max_chars=None,
+                         disabled=st.session_state.processing)
+
+# Add information about claim formatting
+st.info("""
+**Tip for more accurate results:**
+- As older news tends to get deprioritized by sources, trying recent news may yield better results       
+- Try news claims as they appear in the sources
+- For claims older than 36 hours, consider rephrasing the claim by removing time-sensitive words like "recently," "today," "now," etc.
+- Rephrase verbs from present tense to past tense for older events. Examples below:
+  - Instead of "launches/unveils/releases" → use "has launched/unveiled/released"
+  - Instead of "announces/invites/retaliates/ends" → use "has announced/invited/retaliated/ended"
+""")
+
+# Information about result variability
+st.caption("""
+💡 **Note:** Results may vary slightly each time, even for the same claim. This is by design, allowing our system to:
+- Incorporate the most recent evidence available
+- Benefit from the AI's ability to consider multiple perspectives
+- Adapt to evolving information landscapes
+""")
+
+st.warning("⏱️ **Note:** Processing times may vary from 10 seconds to 3 minutes depending on query complexity, available evidence, and current API response times.")
+
+# Create a clean interface based on state
+if st.session_state.fresh_state:
+    # Only show the verify button in fresh state
+    verify_button = st.button(
+        "Verify Claim", 
+        type="primary", 
+        key="verify_btn",
+        disabled=st.session_state.verify_btn_disabled
+    )
+    
+    # When button is clicked and not already processing
+    if verify_button and not st.session_state.processing:
+        # Only show error if claim input is completely empty
+        if not claim_input or claim_input.strip() == "":
+            st.error("Please enter a claim to verify.")
+        else:
+            # Reset claim-specific state before processing a new claim
+            reset_claim_specific_state()
+            
+            # Store the claim and set processing state
+            st.session_state.claim_to_process = claim_input
+            st.session_state.processing = True
+            st.session_state.fresh_state = False
+            st.session_state.verify_btn_disabled = True
+            # Force a rerun to refresh UI
+            st.rerun()
+            
+else:
+    # This is either during processing or showing results
+    
+    # Create a container for processing and results
+    analysis_container = st.container()
+    
+    with analysis_container:
+        # If we're processing, show the processing UI
+        if st.session_state.processing:
+            st.subheader("🔄 Processing...")
+            status = st.empty()
+            status.text("Verifying claim... (this may take a while)")
+            progress_bar = st.progress(0)
+            
+            # Initialize models and agent if needed
+            if not hasattr(st.session_state, 'agent_initialized'):
+                with st.spinner("Initializing system..."):
+                    st.session_state.agent = get_agent()
+                    st.session_state.agent_initialized = True
+            
+            try:
+                # Use the stored claim for processing
+                claim_to_process = st.session_state.claim_to_process
+                
+                # Process the claim with the agent
+                start_time = time.time()
+                result = agent.process_claim(claim_to_process, st.session_state.agent)
+                total_time = time.time() - start_time
+                
+                # Update progress as claim processing completes
+                progress_bar.progress(100)
+                
+                # Check for None result
+                if result is None:
+                    st.error("Failed to process the claim. Please try again.")
+                    st.session_state.processing = False
+                    st.session_state.fresh_state = True
+                    st.session_state.verify_btn_disabled = False
+                else:
+                    # If result exists but key values are missing, provide default values
+                    if "classification" not in result or result["classification"] is None:
+                        result["classification"] = "Uncertain"
+                        
+                    if "confidence" not in result or result["confidence"] is None:
+                        result["confidence"] = 0.0  # Default to 0.0
+                        
+                    if "explanation" not in result or result["explanation"] is None:
+                        result["explanation"] = "Insufficient evidence was found to determine the truthfulness of this claim."
+                    
+                    # Update result with timing information
+                    if "processing_times" not in result:
+                        result["processing_times"] = {"total": total_time}
+                    
+                    # Store the result and timing information
+                    st.session_state.result = result
+                    st.session_state.total_time = total_time
+                    st.session_state.has_result = True
+                    st.session_state.processing = False
+                    
+                    # Clear processing indicators before showing results
+                    status.empty()
+                    progress_bar.empty()
+                    
+                    # Force rerun to display results
+                    st.rerun()
+                    
+            except Exception as e:
+                # Handle any exceptions and reset processing state
+                logger.error(f"Error during claim processing: {str(e)}")
+                st.error(f"An error occurred: {str(e)}")
+                st.session_state.processing = False
+                st.session_state.fresh_state = True
+                st.session_state.verify_btn_disabled = False
+                # Force rerun to re-enable button
+                st.rerun()
+                
+        # Display results if available
+        elif st.session_state.has_result and st.session_state.result:
+            result = st.session_state.result
+            total_time = st.session_state.total_time
+            claim_to_process = st.session_state.claim_to_process
+            
+            st.subheader("📊 Verification Results")
+            
+            result_col1, result_col2 = st.columns([2, 1])
+            
+            with result_col1:
+                # Display only the original claim
+                st.markdown(f"**Claim:** {claim_to_process}")
+                
+                # Make verdict colorful based on classification
+                truth_label = result.get('classification', 'Uncertain')
+                if truth_label and "True" in truth_label:
+                    verdict_color = "green"
+                elif truth_label and "False" in truth_label:
+                    verdict_color = "red"
+                else:
+                    verdict_color = "gray"
+                    
+                st.markdown(f"**Verdict:** <span style='color:{verdict_color};font-size:1.2em'>{truth_label}</span>", unsafe_allow_html=True)
+                
+                # Ensure confidence value is used
+                if "confidence" in result and result["confidence"] is not None:
+                    confidence_value = result["confidence"]
+                    # Make sure confidence is a numeric value between 0 and 1
+                    try:
+                        confidence_value = float(confidence_value)
+                        if confidence_value < 0:
+                            confidence_value = 0.0
+                        elif confidence_value > 1:
+                            confidence_value = 1.0
+                    except (ValueError, TypeError):
+                        confidence_value = 0.0  # Fallback to zero confidence
+                else:
+                    confidence_value = 0.0  # Default confidence
+
+                # Display the confidence
+                st.markdown(f"**Confidence:** {confidence_value:.2%}")
+                
+                # Display low confidence warning if applicable
+                if 0 < confidence_value < 0.1:
+                    st.warning("⚠️ **Very Low Confidence:** This result has very low confidence. Please verify with other authoritative sources.")
+                
+                # Display explanation
+                st.markdown(f"**Explanation:** {result.get('explanation', 'No explanation available.')}")
+                
+                # Add disclaimer about cross-verification
+                st.info("⚠️ **Note:** Please cross-verify important information with additional reliable sources.")
+                
+                if truth_label == "Uncertain":
+                    st.info("💡 **Tip for Uncertain Results:** This claim might be too recent for our sources, too specialized, or might not be widely reported or supported with evidence. It is also possible that the claim does not fall into any of these categories and our system may have failed to fetch the correct evidence. Try checking official sites and blogs, news sites, or other related documentation for more information.")
+            
+            with result_col2:
+                st.markdown("**Processing Time**")
+                times = result.get("processing_times", {"total": total_time})
+                st.markdown(f"- **Total:** {times.get('total', total_time):.2f}s")
+                
+                # Show agent thoughts
+                if "thoughts" in result and result["thoughts"]:
+                    st.markdown("**AI Reasoning Process**")
+                    thoughts = result.get("thoughts", [])
+                    for i, thought in enumerate(thoughts[:5]):  # Show top 5 thoughts
+                        st.markdown(f"{i+1}. {thought}")
+                    if len(thoughts) > 5:
+                        with st.expander("Show all reasoning steps"):
+                            for i, thought in enumerate(thoughts):
+                                st.markdown(f"{i+1}. {thought}")
+            
+            # Display evidence
+            st.subheader("📝 Evidence")
+            evidence_count = result.get("evidence_count", 0)
+            evidence = result.get("evidence", [])
+            
+            # Ensure evidence is a list
+            if not isinstance(evidence, list):
+                if isinstance(evidence, str):
+                    # Try to parse string as a list
+                    try:
+                        import ast
+                        parsed_evidence = ast.literal_eval(evidence)
+                        if isinstance(parsed_evidence, list):
+                            evidence = parsed_evidence
+                        else:
+                            evidence = [evidence]
+                    except:
+                        evidence = [evidence]
+                else:
+                    evidence = [str(evidence)] if evidence else []
+            
+            # Update evidence count based on actual evidence list
+            evidence_count = len(evidence)
+            
+            # Get classification results
+            classification_results = result.get("classification_results", [])
+            
+            # Check for empty evidence
+            if evidence_count == 0 or not any(ev for ev in evidence if ev):
+                st.warning("No relevant evidence was found for this claim. The verdict may not be reliable.")
+            else:
+                # Add message about processing large number of evidence items
+                st.info("The system processes a large number of evidence items across multiple sources and provides a response based on the top relevant evidence items.")
+                
+                # Filter to only show support and contradict evidence
+                if classification_results:
+                    support_evidence = []
+                    contradict_evidence = []
+                    
+                    # Extract supporting and contradicting evidence
+                    for res in classification_results:
+                        if isinstance(res, dict) and "label" in res and "evidence" in res:
+                            if res.get("label") == "support":
+                                support_evidence.append(res)
+                            elif res.get("label") == "contradict":
+                                contradict_evidence.append(res)
+                    
+                    # Sort by confidence
+                    support_evidence.sort(key=lambda x: x.get("confidence", 0), reverse=True)
+                    contradict_evidence.sort(key=lambda x: x.get("confidence", 0), reverse=True)
+                    
+                    # Show counts of relevant evidence
+                    st.markdown(f"Found {len(support_evidence)} supporting and {len(contradict_evidence)} contradicting evidence items")
+                    
+                    # Only show evidence tabs if we have evidence
+                    if evidence and any(ev for ev in evidence if ev):
+                        # Create tabs for supporting and contradicting evidence only (removed All Evidence tab)
+                        evidence_tabs = st.tabs(["Supporting Evidence", "Contradicting Evidence", "Source Details"])
+                        
+                        # Supporting Evidence tab
+                        with evidence_tabs[0]:
+                            if support_evidence:
+                                for i, res in enumerate(support_evidence):
+                                    evidence_text = res.get("evidence", "")
+                                    confidence = res.get("confidence", 0)
+                                    reason = res.get("reason", "No reason provided")
+                                    
+                                    if evidence_text and isinstance(evidence_text, str) and evidence_text.strip():
+                                        with st.expander(f"Supporting Evidence {i+1} (Confidence: {confidence:.2%})", expanded=i==0):
+                                            st.text(evidence_text)
+                                            st.markdown(f"**Reason:** {reason}")
+                            else:
+                                st.info("No supporting evidence was found for this claim.")
+                        
+                        # Contradicting Evidence tab
+                        with evidence_tabs[1]:
+                            if contradict_evidence:
+                                for i, res in enumerate(contradict_evidence):
+                                    evidence_text = res.get("evidence", "")
+                                    confidence = res.get("confidence", 0)
+                                    reason = res.get("reason", "No reason provided")
+                                    
+                                    if evidence_text and isinstance(evidence_text, str) and evidence_text.strip():
+                                        with st.expander(f"Contradicting Evidence {i+1} (Confidence: {confidence:.2%})", expanded=i==0):
+                                            st.text(evidence_text)
+                                            st.markdown(f"**Reason:** {reason}")
+                            else:
+                                st.info("No contradicting evidence was found for this claim.")
+                        
+                        # Source Details tab (keeping original functionality)
+                        with evidence_tabs[2]:
+                            st.markdown("The system evaluates evidence from various sources to determine the verdict.")
+                            
+                            evidence_sources = {}
+                            for ev in evidence:
+                                if not ev or not isinstance(ev, str):
+                                    continue
+                                    
+                                source = "Unknown"
+                                # Extract source info from evidence text
+                                if "URL:" in ev:
+                                    import re
+                                    url_match = re.search(r'URL: https?://(?:www\.)?([^/]+)', ev)
+                                    if url_match:
+                                        source = url_match.group(1)
+                                elif "Source:" in ev:
+                                    import re
+                                    source_match = re.search(r'Source: ([^,]+)', ev)
+                                    if source_match:
+                                        source = source_match.group(1)
+                                
+                                if source in evidence_sources:
+                                    evidence_sources[source] += 1
+                                else:
+                                    evidence_sources[source] = 1
+                            
+                            # Display evidence source distribution
+                            if evidence_sources:
+                                st.markdown("**Evidence Source Distribution**")
+                                for source, count in evidence_sources.items():
+                                    st.markdown(f"- {source}: {count} item(s)")
+                            else:
+                                st.info("No source information available in the evidence.")
+                    else:
+                        st.warning("No evidence was retrieved for this claim.")
+                else:
+                    # Fallback if no classification results
+                    st.markdown(f"Retrieved {evidence_count} pieces of evidence, but none were classified as supporting or contradicting.")
+                    st.warning("No supporting or contradicting evidence was found for this claim.")
+                
+            # Button to start a new verification
+            if st.button("Verify Another Claim", type="primary", key="new_verify_btn"):
+                # Reset all necessary state variables
+                st.session_state.fresh_state = True
+                st.session_state.has_result = False
+                st.session_state.result = None
+                st.session_state.processing = False
+                st.session_state.claim_to_process = ""
+                st.session_state.verify_btn_disabled = False
+                # Clear the input field by resetting the input_content
+                st.session_state.input_content = ""
+                st.rerun()
+
+# Footer with additional information
+st.markdown("---")
+st.caption("""
+**AskVeracity** is an open-source tool designed to help combat misinformation through transparent evidence gathering and analysis. 
+While we strive for accuracy, the system has inherent limitations based on available data sources, API constraints, and the evolving nature of information.
+""")

config.py

@@ -0,0 +1,79 @@
+"""
+Configuration module for the Fake News Detector application.
+
+This module handles loading configuration parameters, API keys,
+and source credibility data needed for the fact checking system.
+It manages environment variables and file-based configurations.
+"""
+
+import os
+import json
+import logging
+from pathlib import Path
+
+# Configure logger
+logger = logging.getLogger("misinformation_detector")
+
+# Base paths
+ROOT_DIR = Path(__file__).parent.absolute()
+DATA_DIR = ROOT_DIR / "data"
+
+# Ensure data directory exists
+DATA_DIR.mkdir(exist_ok=True)
+
+# First try to get API keys from Streamlit secrets, then fall back to environment variables
+try:
+    import streamlit as st
+    OPENAI_API_KEY = st.secrets.get("OPENAI_API_KEY", os.environ.get("OPENAI_API_KEY", ""))
+    NEWS_API_KEY = st.secrets.get("NEWS_API_KEY", os.environ.get("NEWS_API_KEY", ""))
+    FACTCHECK_API_KEY = st.secrets.get("FACTCHECK_API_KEY", os.environ.get("FACTCHECK_API_KEY", ""))
+except (AttributeError, ImportError):
+    # Fall back to environment variables if Streamlit secrets aren't available
+    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "")
+    NEWS_API_KEY = os.environ.get("NEWS_API_KEY", "")
+    FACTCHECK_API_KEY = os.environ.get("FACTCHECK_API_KEY", "")
+
+# Log secrets status (but not the values)
+if OPENAI_API_KEY:
+    logger.info("OPENAI_API_KEY is set")
+else:
+    logger.warning("OPENAI_API_KEY not set. The application will not function properly.")
+    
+if NEWS_API_KEY:
+    logger.info("NEWS_API_KEY is set")
+else:
+    logger.warning("NEWS_API_KEY not set. News evidence retrieval will be limited.")
+    
+if FACTCHECK_API_KEY:
+    logger.info("FACTCHECK_API_KEY is set")
+else:
+    logger.warning("FACTCHECK_API_KEY not set. Fact-checking evidence will be limited.")
+
+# Set API key in environment to ensure it's available to all components
+os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+
+# Rate limiting configuration
+RATE_LIMITS = {
+    # api_name: {"requests": max_requests, "period": period_in_seconds}
+    "newsapi": {"requests": 100, "period": 3600},  # 100 requests per hour
+    "factcheck": {"requests": 1000, "period": 86400},  # 1000 requests per day
+    "semantic_scholar": {"requests": 10, "period": 300},  # 10 requests per 5 minutes
+    "wikidata": {"requests": 60, "period": 60},  # 60 requests per minute
+    "wikipedia": {"requests": 200, "period": 60},  # 200 requests per minute
+    "rss": {"requests": 300, "period": 3600}  # 300 RSS requests per hour
+}
+
+# Error backoff settings
+ERROR_BACKOFF = {
+    "max_retries": 5,
+    "initial_backoff": 1,  # seconds
+    "backoff_factor": 2,  # exponential backoff
+}
+
+# RSS feed settings
+RSS_SETTINGS = {
+    "max_feeds_per_request": 10,  # Maximum number of feeds to try per request
+    "max_age_days": 3,            # Maximum age of RSS items to consider
+    "timeout_seconds": 5,         # Timeout for RSS feed requests
+    "max_workers": 5              # Number of parallel workers for fetching feeds
+}

docs/architecture.md

@@ -0,0 +1,176 @@
+# AskVeracity Architecture
+
+## System Overview
+
+AskVeracity is a fact-checking and misinformation detection application that verifies factual claims by gathering and analyzing evidence from multiple sources. The system follows an agentic approach using LangGraph's ReAct agent framework for orchestrating the verification process.
+
+## Core Components
+
+### 1. Agent System
+
+The system implements a LangGraph-based agent that orchestrates the entire fact-checking process:
+
+- **Core Agent:** Defined in `agent.py`, the ReAct agent coordinates the execution of individual tools in a logical sequence to verify claims.
+- **Agent Tools:** Implemented as callable functions that the agent can invoke:
+  - `claim_extractor`: Extracts the main factual claim from user input
+  - `evidence_retriever`: Gathers evidence from multiple sources
+  - `truth_classifier`: Evaluates the claim against evidence
+  - `explanation_generator`: Creates human-readable explanations
+
+### 2. Web Interface
+
+The user interface is implemented using Streamlit:
+
+- **Main App:** Defined in `app.py`, provides the interface for users to submit claims and view results
+- **Caching:** Uses Streamlit's caching mechanisms to optimize performance
+- **Results Display:** Shows verdict, confidence, explanation, and evidence details
+
+## Module Architecture
+
+```
+askveracity/
+│
+├── agent.py                   # LangGraph agent implementation
+├── app.py                     # Main Streamlit application
+├── config.py                  # Configuration and API keys
+├── evaluate_performance.py    # Performance evaluation script
+│
+├── modules/                   # Core functionality modules
+│   ├── claim_extraction.py    # Claim extraction functionality  
+│   ├── evidence_retrieval.py  # Evidence gathering from various sources
+│   ├── classification.py      # Truth classification logic
+│   ├── explanation.py         # Explanation generation
+│   ├── rss_feed.py            # RSS feed evidence retrieval
+│   └── category_detection.py  # Claim category detection
+│
+├── utils/                     # Utility functions
+│   ├── api_utils.py           # API rate limiting and error handling
+│   ├── performance.py         # Performance tracking utilities
+│   └── models.py              # Model initialization functions
+│
+├── results/                   # Performance evaluation results
+│   ├── performance_results.json # Evaluation metrics
+│   └── *.png                  # Performance visualization charts
+│
+└── docs/ # Documentation
+   ├── assets/ # Images and other media
+   │   └── app_screenshot.png # Application screenshot
+   ├── architecture.md # System design and component interactions
+   ├── configuration.md # Setup and environment configuration
+   ├── data-handling.md # Data processing and flow
+   └── changelog.md # Version history
+```
+
+## Component Interactions
+
+### Claim Verification Flow
+
+1. **User Input:** User submits a claim via the Streamlit interface
+2. **Agent Initialization:** The ReAct agent is initialized with fact-checking tools
+3. **Claim Extraction:** The agent extracts the main factual claim
+4. **Category Detection:** The system detects the category of the claim (ai, science, technology, politics, business, world, sports, entertainment)
+5. **Evidence Retrieval:** Multi-source evidence gathering with priority based on claim category
+6. **Evidence Analysis:** Entity and verb matching assesses evidence relevance
+7. **Truthfulness Classification:** The agent evaluates the claim against the evidence
+8. **Explanation Generation:** Human-readable explanation is generated
+9. **Results Display:** Results are presented to the user with evidence details
+
+### Evidence Retrieval Architecture
+
+Evidence retrieval is a core component of the misinformation detection system:
+
+1. **Multi-source Retrieval:** The system collects evidence from:
+   - Wikipedia
+   - Wikidata
+   - News API
+   - RSS feeds
+   - Fact-checking sites (via Google Fact Check Tools API)
+   - Academic sources (via OpenAlex)
+
+2. **Category-aware Prioritization:** Sources are prioritized based on the detected category of the claim:
+   - Each category (ai, science, technology, politics, business, world, sports, entertainment) has dedicated RSS feeds
+   - AI category falls back to technology sources when needed
+   - Other categories fall back to default RSS feeds
+
+3. **Parallel Processing:** Evidence retrieval uses ThreadPoolExecutor for parallel API requests with optimized timeouts
+
+4. **Rate Limiting:** API calls are managed by a token bucket rate limiter to respect API usage limits
+
+5. **Error Handling:** Robust error handling with exponential backoff for retries
+
+6. **Source Verification:** The system provides direct URLs to original sources for all evidence items, enabling users to verify information at its original source
+
+
+### Classification System
+
+The truth classification process involves:
+
+1. **Evidence Analysis:** Each evidence item is classified as supporting, contradicting, or insufficient
+2. **Confidence Scoring:** Confidence scores are assigned to each classification
+3. **Aggregation:** Individual evidence classifications are aggregated to determine the final verdict
+
+## Technical Details
+
+### Language Models
+
+- Uses OpenAI's Large Language Model GPT-3.5 Turbo via LangChain
+- Configurable model selection in `utils/models.py`
+
+### NLP Processing
+
+- spaCy for natural language processing tasks
+- Named entity recognition for claim and evidence analysis
+- Entity and verb matching for evidence relevance scoring
+
+### Performance Optimization
+
+- Caching of models and results
+- Prioritized and parallel evidence retrieval
+- Early relevance analysis during retrieval process
+
+### Error Resilience
+
+- Multiple fallback mechanisms
+- Graceful degradation when sources are unavailable
+- Comprehensive error logging
+
+## Performance Evaluation Results
+
+The system has been evaluated using a test set of 40 claims across three categories (True, False, and Uncertain). A typical performance profile shows:
+
+1. **Overall Accuracy:** ~52.5% across all claim types
+   * Accuracy: Percentage of claims correctly classified according to their ground truth label
+
+2. **Safety Rate:** ~70.0% across all claim types
+   * Safety Rate: Percentage of claims that were either correctly classified or safely categorized as "Uncertain" rather than making an incorrect assertion
+
+3. **Class-specific Metrics:**
+   * True claims: ~40-60% accuracy, ~55-85% safety rate
+   * False claims: ~15-35% accuracy, ~50-70% safety rate
+   * Uncertain claims: ~50.0% accuracy, ~50.0% safety rate (for Uncertain claims, accuracy equals safety rate)
+
+4. **Confidence Scores:**
+   * True claims: ~0.62-0.74 average confidence
+   * False claims: ~0.42-0.50 average confidence
+   * Uncertain claims: ~0.38-0.50 average confidence
+
+5. **Processing Times:**
+   * True claims: ~21-32 seconds average
+   * False claims: ~24-37 seconds average
+   * Uncertain claims: ~23-31 seconds average
+
+**Note:** The class-specific metrics, confidence scores, and processing times vary by test run.
+
+These metrics vary between evaluation runs due to the dynamic nature of evidence sources and the real-time information landscape. The system is designed to adapt to this variability, making it well-suited for real-world fact-checking scenarios where information evolves over time.
+
+## Misinformation Detection Capabilities
+
+The system's approach to detecting misinformation includes:
+
+1. **Temporal Relevance:** Checks if evidence is temporally appropriate for the claim
+2. **Contradiction Detection:** Identifies evidence that directly contradicts claims
+3. **Evidence Diversity:** Ensures diverse evidence sources for more robust verification
+4. **Domain Prioritization:** Applies a small relevance boost to content from established news and fact-checking domains in the RSS feed handling
+5. **Safety-First Classification:** Prioritizes preventing the spread of misinformation by avoiding incorrect assertions when evidence is insufficient
+
+This architecture enables AskVeracity to efficiently gather, analyze, and present evidence relevant to user claims, supporting the broader effort to detect and counteract misinformation.

docs/changelog.md

@@ -0,0 +1,114 @@
+# Changelog
+
+All notable changes to the AskVeracity fact-checking and misinformation detection system will be documented in this file.
+
+## [0.4.1] - 2025-04-25
+
+### Updated
+- Updated architecture.md to improve accuracy of system description
+- Updated README.md to better reflect current system functionality
+- Removed references to deprecated source credibility assessment
+- Clarified documentation of domain quality boost in RSS feed processing
+
+## [0.4.0] - 2025-04-24
+
+### Added
+- Added safety rate metric to performance evaluation
+  - Measures how often the system avoids making incorrect assertions
+  - Tracks when system correctly abstains from judgment by using "Uncertain" 
+  - Included in overall metrics and per-class metrics
+- New safety rate visualization chart in performance evaluation
+- Added safety flag to detailed claim results
+
+### Updated
+- Enhanced `evaluate_performance.py` script to track and calculate safety rates
+- Updated documentation to explain the safety rate metric and its importance
+- Improved tabular display of performance metrics with safety rate column
+
+## [0.3.0] - 2025-04-23
+
+### Added
+- Performance evaluation script (`evaluate_performance.py`) in root directory
+- Performance results visualization and storage in `results/` directory
+- Enhanced error handling and fallback mechanisms
+- Refined relevance scoring with entity and verb matching with keyword fallback for accurate evidence assessment
+- Enhanced evidence relevance with weighted scoring prioritization and increased gathering from 5 to 10 items
+- Added detailed confidence calculation for more reliable verdicts with better handling of low confidence cases
+- Category-specific RSS feeds for more targeted evidence retrieval
+- OpenAlex integration for scholarly evidence (replacing Semantic Scholar)
+
+### Changed
+- Improved classification output structure for consistent downstream processing
+- Added fallback mechanisms for explanation generation and classification
+- Improved evidence retrieval and classification mechanism
+- Streamlined architecture by removing source credibility and semantic analysis complexity
+- Improved classification mechanism with weighted evidence count (55%) and quality (45%)
+- Updated documentation to reflect the updated performance metrics, enhanced evidence processing pipeline, improved classification mechanism, and streamlined architecture
+
+### Fixed
+- Enhanced handling of non-standard response formats
+
+## [0.2.0] - 2025-04-22
+
+### Added
+- Created comprehensive documentation in `/docs` directory
+  - `architecture.md` for system design and component interactions
+  - `configuration.md` for setup and environment configuration
+  - `data-handling.md` for data processing and flow
+  - `changelog.md` for version history tracking
+- Updated app description to emphasize misinformation detection capabilities
+
+### Changed
+- Improved directory structure with documentation folder
+- Enhanced README with updated project structure
+- Clarified misinformation detection focus in documentation
+
+## [0.1.0] - 2025-04-21
+
+### Added
+- Initial release of AskVeracity fact-checking system
+- Streamlit web interface in `app.py`
+- LangGraph ReAct agent implementation in `agent.py`
+- Multi-source evidence retrieval system
+  - Wikipedia integration
+  - Wikidata integration
+  - News API integration
+  - RSS feed processing
+  - Google's FactCheck Tools API integration
+  - OpenAlex scholarly evidence
+- Truth classification with LLM
+- Explanation generation
+- Performance tracking utilities
+- Rate limiting and API error handling
+- Category detection for source prioritization
+
+### Features
+- User-friendly claim input interface
+- Detailed results display with evidence exploration
+- Category-aware source prioritization
+- Robust error handling and fallbacks
+- Parallel evidence retrieval for improved performance
+- Support for various claim categories:
+  - AI
+  - Science
+  - Technology
+  - Politics
+  - Business
+  - World news
+  - Sports
+  - Entertainment
+
+## Unreleased
+
+### Planned Features
+- Enhanced visualization of evidence relevance
+- Support for user feedback on verification results
+- Streamlined fact-checking using only relevant sources
+- Source weighting for improved result relevance
+- Improved verdict confidence for challenging / ambiguous claims
+- Expanded fact-checking sources
+- Improved handling of multilingual claims
+- Integration with additional academic databases
+- Custom source credibility configuration interface
+- Historical claim verification database
+- API endpoint for programmatic access

docs/configuration.md

@@ -0,0 +1,292 @@
+# AskVeracity Configuration Guide
+
+This document describes how to set up and configure the AskVeracity fact-checking and misinformation detection system.
+
+## Prerequisites
+
+Before setting up AskVeracity, ensure you have:
+
+- Python 3.8 or higher
+- pip (Python package installer)
+- Git (for cloning the repository)
+- API keys for external services
+
+## Installation
+
+### Local Development
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/yourusername/askveracity.git
+   cd askveracity
+   ```
+
+2. Install the required dependencies:
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+3. Download the required spaCy model:
+   ```bash
+   python -m spacy download en_core_web_sm
+   ```
+
+## API Key Configuration
+
+AskVeracity requires several API keys to access external services. You have two options for configuring these keys:
+
+### Option 1: Using Streamlit Secrets (Recommended for Local Development)
+
+1. Create a `.streamlit` directory if it doesn't exist:
+   ```bash
+   mkdir -p .streamlit
+   ```
+
+2. Create a `secrets.toml` file:
+   ```bash
+   cp .streamlit/secrets.toml.example .streamlit/secrets.toml
+   ```
+
+3. Edit the `.streamlit/secrets.toml` file with your API keys:
+   ```toml
+   OPENAI_API_KEY = "your_openai_api_key"
+   NEWS_API_KEY = "your_news_api_key"
+   FACTCHECK_API_KEY = "your_factcheck_api_key"
+   ```
+
+### Option 2: Using Environment Variables
+
+1. Create a `.env` file in the root directory:
+   ```bash
+   touch .env
+   ```
+
+2. Add your API keys to the `.env` file:
+   ```
+   OPENAI_API_KEY=your_openai_api_key
+   NEWS_API_KEY=your_news_api_key
+   FACTCHECK_API_KEY=your_factcheck_api_key
+   ```
+
+3. Load the environment variables:
+   ```python
+   # In Python
+   from dotenv import load_dotenv
+   load_dotenv()
+   ```
+
+   Or in your terminal:
+   ```bash
+   # Unix/Linux/MacOS
+   source .env
+   
+   # Windows
+   # Install python-dotenv[cli] and run
+   dotenv run streamlit run app.py
+   ```
+
+## Required API Keys
+
+AskVeracity uses the following external APIs:
+
+1. **OpenAI API** (Required)
+   - Used for claim extraction, classification, and explanation generation
+   - Get an API key from [OpenAI's website](https://platform.openai.com/)
+
+2. **News API** (Optional but recommended)
+   - Used for retrieving news article evidence
+   - Get an API key from [NewsAPI.org](https://newsapi.org/)
+
+3. **Google Fact Check Tools API** (Optional but recommended)
+   - Used for retrieving fact-checking evidence
+   - Get an API key from [Google Fact Check Tools API](https://developers.google.com/fact-check/tools/api)
+
+## Configuration Files
+
+### config.py
+
+The main configuration file is `config.py`, which contains:
+
+- API key handling
+- Rate limiting configuration
+- Error backoff settings
+- RSS feed settings
+
+Important configuration sections in `config.py`:
+
+```python
+# Rate limiting configuration
+RATE_LIMITS = {
+    # api_name: {"requests": max_requests, "period": period_in_seconds}
+    "newsapi": {"requests": 100, "period": 3600},  # 100 requests per hour
+    "factcheck": {"requests": 1000, "period": 86400},  # 1000 requests per day
+    "semantic_scholar": {"requests": 10, "period": 300},  # 10 requests per 5 minutes
+    "wikidata": {"requests": 60, "period": 60},  # 60 requests per minute
+    "wikipedia": {"requests": 200, "period": 60},  # 200 requests per minute
+    "rss": {"requests": 300, "period": 3600}  # 300 RSS requests per hour
+}
+
+# Error backoff settings
+ERROR_BACKOFF = {
+    "max_retries": 5,
+    "initial_backoff": 1,  # seconds
+    "backoff_factor": 2,  # exponential backoff
+}
+
+# RSS feed settings
+RSS_SETTINGS = {
+    "max_feeds_per_request": 10,  # Maximum number of feeds to try per request
+    "max_age_days": 3,            # Maximum age of RSS items to consider
+    "timeout_seconds": 5,         # Timeout for RSS feed requests
+    "max_workers": 5              # Number of parallel workers for fetching feeds
+}
+```
+
+### Category-Specific RSS Feeds
+
+Category-specific RSS feeds are defined in `modules/category_detection.py`. These feeds are used to prioritize sources based on the detected claim category:
+
+```python
+CATEGORY_SPECIFIC_FEEDS = {
+    "ai": [
+        "https://www.artificialintelligence-news.com/feed/",
+        "https://openai.com/news/rss.xml",
+        # Additional AI-specific feeds
+    ],
+    "science": [
+        "https://www.science.org/rss/news_current.xml",
+        "https://www.nature.com/nature.rss",
+        # Additional science feeds
+    ],
+    # Additional categories
+}
+```
+
+## Hugging Face Spaces Deployment
+
+### Setting Up a Space
+
+1. Create a new Space on Hugging Face:
+   - Go to https://huggingface.co/spaces
+   - Click "Create new Space"
+   - Select "Streamlit" as the SDK
+   - Choose the hardware tier (use the default 16GB RAM)
+
+2. Upload the project files:
+   - You can upload files directly through the Hugging Face web interface
+   - Alternatively, use Git to push to the Hugging Face repository
+   - Make sure to include all necessary files including requirements.txt
+
+### Setting Up Secrets
+
+1. Add API keys as secrets:
+   - Go to the "Settings" tab of your Space
+   - Navigate to the "Repository secrets" section
+   - Add your API keys:
+     - `OPENAI_API_KEY`
+     - `NEWS_API_KEY`
+     - `FACTCHECK_API_KEY`
+
+### Configuring the Space
+
+Edit the metadata in the `README.md` file:
+
+```yaml
+---
+title: Askveracity
+emoji: 📉
+colorFrom: blue
+colorTo: pink
+sdk: streamlit
+sdk_version: 1.44.1
+app_file: app.py
+pinned: false
+license: mit
+short_description: Fact-checking and misinformation detection tool.
+---
+```
+
+## Custom Configuration
+
+### Adjusting Rate Limits
+
+You can adjust the rate limits in `config.py` based on your API subscription levels:
+
+```python
+# Update for higher tier News API subscription
+RATE_LIMITS["newsapi"] = {"requests": 500, "period": 3600}  # 500 requests per hour
+```
+
+### Modifying RSS Feeds
+
+The list of RSS feeds can be found in `modules/rss_feed.py` and category-specific feeds in `modules/category_detection.py`. You can add or remove feeds as needed.
+
+### Performance Evaluation
+
+The system includes a performance evaluation script `evaluate_performance.py` that:
+
+1. Runs the fact-checking system on a predefined set of test claims
+2. Calculates accuracy, safety rate, processing time, and confidence metrics
+3. Generates visualization charts in the `results/` directory
+4. Saves detailed results to `results/performance_results.json`
+
+To run the performance evaluation:
+
+```bash
+python evaluate_performance.py [--limit N] [--output FILE]
+```
+
+- `--limit N`: Limit evaluation to first N claims (default: all)
+- `--output FILE`: Save results to FILE (default: performance_results.json)
+
+## Running the Application
+
+Start the Streamlit app:
+
+```bash
+streamlit run app.py
+```
+
+The application will be available at http://localhost:8501 by default.
+
+## Troubleshooting
+
+### API Key Issues
+
+If you encounter API key errors:
+
+1. Verify that your API keys are set correctly
+2. Check the logs for specific error messages
+3. Make sure API keys are not expired or rate-limited
+
+### Model Loading Errors
+
+If spaCy model fails to load:
+
+```bash
+# Reinstall the model
+python -m spacy download en_core_web_sm --force
+```
+
+### Rate Limiting
+
+If you encounter rate limiting issues:
+
+1. Reduce the number of requests by adjusting `RATE_LIMITS` in `config.py`
+2. Increase the backoff parameters in `ERROR_BACKOFF`
+3. Subscribe to higher API tiers if available
+
+### Memory Issues
+
+If the application crashes due to memory issues:
+
+1. Reduce the number of parallel workers in `RSS_SETTINGS`
+2. Limit the maximum number of evidence items processed
+
+## Performance Optimization
+
+For better performance:
+
+1. Upgrade to a higher-tier OpenAI model for improved accuracy
+2. Increase the number of parallel workers for evidence retrieval
+3. Add more relevant RSS feeds to improve evidence gathering

docs/data-handling.md

@@ -0,0 +1,285 @@
+# Data Handling in AskVeracity
+
+This document explains how data flows through the AskVeracity fact-checking and misinformation detection system, from user input to final verification results.
+
+## Data Flow Overview
+
+```
+User Input → Claim Extraction → Category Detection → Evidence Retrieval → Evidence Analysis → Classification → Explanation → Result Display
+```
+
+## User Input Processing
+
+### Input Sanitization and Extraction
+
+1. **Input Acceptance:** The system accepts user input as free-form text through the Streamlit interface.
+
+2. **Claim Extraction** (`modules/claim_extraction.py`):
+   - For concise inputs (<30 words), the system preserves the input as-is
+   - For longer texts, an LLM extracts the main factual claim
+   - Validation ensures the extraction doesn't add information not present in the original
+   - Entity preservation is verified using spaCy's NER
+
+3. **Claim Shortening:**
+   - For evidence retrieval, claims are shortened to preserve key entities and context
+   - Preserves entity mentions, key nouns, titles, country references, and negation contexts
+
+## Evidence Retrieval and Processing
+
+### Multi-source Evidence Gathering
+
+Evidence is collected from multiple sources in parallel (`modules/evidence_retrieval.py`):
+
+1. **Category Detection** (`modules/category_detection.py`):
+   - Detects the claim category (ai, science, technology, politics, business, world, sports, entertainment)
+   - Prioritizes sources based on category
+   - No category receives preferential weighting; assignment is based purely on keyword matching
+
+2. **Wikipedia** evidence:
+   - Search Wikipedia API for relevant articles
+   - Extract introductory paragraphs
+   - Process in parallel for up to 3 top search results
+
+3. **Wikidata** evidence:
+   - SPARQL queries for structured data
+   - Entity extraction with descriptions
+
+4. **News API** evidence:
+   - Retrieval from NewsAPI.org with date filtering
+   - Prioritizes recent articles
+   - Extracts titles, descriptions, and content snippets
+
+5. **RSS Feed** evidence (`modules/rss_feed.py`):
+   - Parallel retrieval from multiple RSS feeds
+   - Category-specific feeds selection
+   - Relevance and recency scoring
+
+6. **ClaimReview** evidence:
+   - Google's Fact Check Tools API integration
+   - Retrieves fact-checks from fact-checking organizations
+   - Includes ratings and publisher information
+
+7. **Scholarly** evidence:
+   - OpenAlex API for academic sources
+   - Extracts titles, abstracts, and publication dates
+
+8. **Category Fallback** mechanism:
+   - For AI claims, falls back to technology sources if insufficient evidence (for RSS feeds)
+   - For other categories, falls back to default RSS feeds
+   - Ensures robust evidence retrieval across related domains
+
+### Evidence Preprocessing
+
+Each evidence item is standardized to a consistent format:
+```
+Title: [title], Source: [source], Date: [date], URL: [url], Content: [content snippet]
+```
+
+Length limits are applied to reduce token usage:
+- Content snippets are limited to ~1000 characters
+- Evidence items are truncated while maintaining context
+
+## Evidence Analysis and Relevance Ranking
+
+### Relevance Assessment
+
+Evidence is analyzed and scored for relevance:
+
+1. **Component Extraction:**
+   - Extract entities, verbs, and keywords from the claim
+   - Use NLP processing to identify key claim components
+
+2. **Entity and Verb Matching:**
+   - Match entities from claim to evidence (case-sensitive and case-insensitive)
+   - Match verbs from claim to evidence
+   - Score based on matches (entity matches weighted higher than verb matches)
+
+3. **Temporal Relevance:**
+   - Detection of temporal indicators in claims
+   - Date-based filtering for time-sensitive claims
+   - Adjusts evidence retrieval window based on claim temporal context
+
+4. **Scoring Formula:**
+   ```
+   final_score = (entity_matches * 3.0) + (verb_matches * 2.0)
+   ```
+   If no entity or verb matches, fall back to keyword matching:
+   ```
+   final_score = keyword_matches * 1.0
+   ```
+
+### Evidence Selection
+
+The system selects the most relevant evidence:
+
+1. **Relevance Sorting:**
+   - Evidence items sorted by relevance score (descending)
+   - Top 10 most relevant items selected
+
+2. **Handling No Evidence:**
+   - If no evidence is found, a placeholder is returned
+   - Ensures graceful handling of edge cases
+
+## Truth Classification
+
+### Evidence Classification (`modules/classification.py`)
+
+Each evidence item is classified individually:
+
+1. **LLM Classification:**
+   - Each evidence item is analyzed by an LLM
+   - Classification categories: support, contradict, insufficient
+   - Confidence score (0-100) assigned to each classification
+   - Structured output parsing with fallback mechanisms
+
+2. **Tense Normalization:**
+   - Normalizes verb tenses in claims to ensure consistent classification
+   - Converts present simple and perfect forms to past tense equivalents
+   - Preserves semantic equivalence across tense variations
+
+### Verdict Aggregation
+
+Evidence classifications are aggregated to determine the final verdict:
+
+1. **Weighted Aggregation:**
+   - 55% weight for count of support/contradict items
+   - 45% weight for quality (confidence) of support/contradict items
+
+2. **Confidence Calculation:**
+   - Formula: `1.0 - (min_score / max_score)`
+   - Higher confidence for consistent evidence
+   - Lower confidence for mixed or insufficient evidence
+
+3. **Final Verdict Categories:**
+   - "True (Based on Evidence)"
+   - "False (Based on Evidence)"
+   - "Uncertain"
+
+## Explanation Generation
+
+### Explanation Creation (`modules/explanation.py`)
+
+Human-readable explanations are generated based on the verdict:
+
+1. **Template Selection:**
+   - Different prompts for true, false, and uncertain verdicts
+   - Special handling for claims containing negation
+
+2. **Confidence Communication:**
+   - Translation of confidence scores to descriptive language
+   - Clear communication of certainty/uncertainty
+
+3. **Very Low Confidence Handling:**
+   - Special explanations for verdicts with very low confidence (<10%)
+   - Strong recommendations to verify with authoritative sources
+
+## Result Presentation
+
+Results are presented in the Streamlit UI with multiple components:
+
+1. **Verdict Display:**
+   - Color-coded verdict (green for true, red for false, gray for uncertain)
+   - Confidence percentage
+   - Explanation text
+
+2. **Evidence Presentation:**
+   - Tabbed interface for different evidence views with URLs if available
+   - Supporting and contradicting evidence tabs
+   - Source distribution summary
+
+3. **Input Guidance:**
+   - Tips for claim formatting
+   - Guidance for time-sensitive claims
+   - Suggestions for verb tense based on claim age
+
+4. **Processing Insights:**
+   - Processing time
+   - AI reasoning steps
+   - Source distribution statistics
+
+## Data Persistence and Privacy
+
+AskVeracity prioritizes user privacy:
+
+1. **No Data Storage:**
+   - User claims are not stored persistently
+   - Results are maintained only in session state
+   - No user data is collected or retained
+
+2. **Session Management:**
+   - Session state in Streamlit manages current user interaction
+   - Session is cleared when starting a new verification
+
+3. **API Interaction:**
+   - External API calls use their respective privacy policies
+   - OpenAI API usage follows their data handling practices
+
+4. **Caching:**
+   - Model caching for performance
+   - Resource cleanup on application termination
+
+## Performance Tracking
+
+The system includes a performance tracking utility (`utils/performance.py`):
+
+1. **Metrics Tracked:**
+   - Claims processed count
+   - Evidence retrieval success rates
+   - Processing times
+   - Confidence scores
+   - Source types used
+   - Temporal relevance
+
+2. **Usage:**
+   - Performance metrics are logged during processing
+   - Summary of select metrics available in the final result
+   - Used for system optimization
+
+## Performance Evaluation
+
+The system includes a performance evaluation script (`evaluate_performance.py`):
+
+1. **Test Claims:**
+   - Predefined set of test claims with known ground truth labels
+   - Claims categorized as "True", "False", or "Uncertain"
+
+2. **Metrics:**
+   - Overall accuracy: Percentage of claims correctly classified according to ground truth
+   - Safety rate: Percentage of claims either correctly classified or safely categorized as "Uncertain" rather than making an incorrect assertion
+   - Per-class accuracy and safety rates
+   - Average processing time
+   - Average confidence score
+   - Classification distributions
+
+3. **Visualization:**
+   - Charts for accuracy by classification type
+   - Charts for safety rate by classification type
+   - Processing time by classification type
+   - Confidence scores by classification type
+
+4. **Results Storage:**
+   - Detailed results saved to JSON file
+   - Visualization charts saved as PNG files
+   - All results stored in the `results/` directory
+
+## Error Handling and Resilience
+
+The system implements robust error handling:
+
+1. **API Error Handling** (`utils/api_utils.py`):
+   - Decorator-based error handling
+   - Exponential backoff for retries
+   - Rate limiting respecting API constraints
+
+2. **Safe JSON Parsing:**
+   - Defensive parsing of API responses
+   - Fallback mechanisms for invalid responses
+
+3. **Graceful Degradation:**
+   - Multiple fallback strategies
+   - Core functionality preservation even when some sources fail
+
+4. **Fallback Mechanisms:**
+   - Fallback for truth classification when classifier is not called
+   - Fallback for explanation generation when explanation generator is not called
+   - Ensures complete results even with partial component failures

evaluate_performance.py

@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+"""
+Performance Evaluation Script for AskVeracity.
+
+This script evaluates the performance of the AskVeracity fact-checking system
+using a predefined set of test claims with known ground truth labels.
+It collects metrics on accuracy, safety rate, processing time, and confidence scores
+without modifying the core codebase.
+
+Usage:
+    python evaluate_performance.py [--limit N] [--output FILE]
+
+Options:
+    --limit N        Limit evaluation to first N claims (default: all)
+    --output FILE    Save results to FILE (default: performance_results.json)
+"""
+
+import os
+import sys
+import json
+import time
+import argparse
+from datetime import datetime
+import matplotlib.pyplot as plt
+from tabulate import tabulate
+import numpy as np
+
+# Add the parent directory to sys.path if this script is run directly
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# Import the agent and performance tracker
+import agent
+from utils.performance import PerformanceTracker
+from utils.models import initialize_models
+
+# IMPORTANT NOTE FOR DEVELOPERS:
+# The test claims below include many recent events that will become outdated.
+# When using this script for testing or evaluation, please update these claims 
+# with relevant and up-to-date examples to ensure meaningful results.
+# Performance metrics are heavily influenced by the recency and verifiability 
+# of these claims, so using outdated claims will likely lead to poor results.
+
+# Define the test claims with ground truth labels
+TEST_CLAIMS = [
+    # True claims
+    {"claim": "Dozens killed as gunmen massacre tourists in Kashmir beauty spot.", "expected": "True"},
+    {"claim": "Pope Francis dies at 88.", "expected": "True"},
+    {"claim": "OpenAI released new reasoning models called o3 and o4-mini.", "expected": "True"},
+    {"claim": "Trump And Zelensky Clash Again As US Says Crimea Now Russian Territory.", "expected": "True"},
+    {"claim": "Twelve states sue Donald Trump administration in trade court over chaotic and illegal tariff policy.", "expected": "True"},
+    {"claim": "Zomato has been renamed to Eternal Limited.", "expected": "True"},
+    {"claim": "The Taj Mahal is located in Agra.", "expected": "True"},
+    {"claim": "ISRO achieves second docking with SpaDeX satellites.", "expected": "True"},
+    {"claim": "The TV series Adolescence is streaming on Netflix.", "expected": "True"},
+    {"claim": "Vladimir Putin offers to halt Ukraine invasion.", "expected": "True"},
+    {"claim": "Meta released its Llama 4 language model.", "expected": "True"},
+    {"claim": "Google launched Gemini 2.5 Pro Experimental, the first model in the Gemini 2.5 family.", "expected": "True"},
+    {"claim": "Microsoft is rolling out improved Recall feature for Windows Insiders.", "expected": "True"},
+    {"claim": "Microsoft announced a 1-bit language model that can run on CPU.", "expected": "True"},
+    {"claim": "Royal Challengers Bengaluru beat Rajasthan Royals by 11 runs in yesterday's IPL match.", "expected": "True"},
+    {"claim": "Anthropic introduced Claude Research.", "expected": "True"},
+    {"claim": "The IMF has lowered India's growth projection for the fiscal year 2025-26 to 6.2 per cent.", "expected": "True"},
+    {"claim": "In Bundesliga, Bayern Munich beat Heidenheim 4-0 last week.", "expected": "True"},
+    {"claim": "Manchester United in Europa League semi-finals.", "expected": "True"},
+    
+    # False claims
+    {"claim": "The Eiffel Tower is in Rome.", "expected": "False"},
+    {"claim": "The earth is flat.", "expected": "False"},
+    {"claim": "Rishi Sunak is the current Prime Minister of the UK.", "expected": "False"},
+    {"claim": "New Zealand won the ICC Champions Trophy in 2025.", "expected": "False"},
+    {"claim": "US President Donald trump to visit India next week.", "expected": "False"},
+    {"claim": "Quantum computers have definitively solved the protein folding problem.", "expected": "False"},
+    {"claim": "CRISPR gene editing has successfully cured type 1 diabetes in human clinical trials.", "expected": "False"},
+    {"claim": "Google's new quantum computer, Willow, has demonstrated remarkable capabilities by solving mathematical problems far beyond the reach of the fastest supercomputers.", "expected": "False"},
+    {"claim": "NASA confirmed that the James Webb Space Telescope has found definitive evidence of alien life on an exoplanet.", "expected": "False"},
+    {"claim": "Google launched Gemini 3.", "expected": "False"},
+    {"claim": "A solar eclipse was be seen in India on October 17, 2024.", "expected": "False"},
+    {"claim": "Tom Cruise and Shah Rukh Khan have starred in a Bollywood movie in the past.", "expected": "False"},
+    {"claim": "Germany has the highest GDP in the world.", "expected": "False"},
+    
+    # Uncertain claims
+    {"claim": "Aliens have visited the Earth.", "expected": "Uncertain"},
+    {"claim": "Information that falls into a black hole is permanently lost or destroyed.", "expected": "Uncertain"},
+    {"claim": "Time travel into the past is possible.", "expected": "Uncertain"},
+    {"claim": "Bigfoot (or Yeti) exists in remote wilderness areas.", "expected": "Uncertain"},
+    {"claim": "Intelligent life exists elsewhere in the universe.", "expected": "Uncertain"},
+    {"claim": "Yogi Adityanath will be the next Prime Minister of India.", "expected": "Uncertain"},
+    {"claim": "Consciousness continues to exist after biological death.", "expected": "Uncertain"},
+    {"claim": "There are multiple parallel universes.", "expected": "Uncertain"}
+]
+
+def setup_argument_parser():
+    """
+    Set up command line argument parsing.
+    
+    Returns:
+        argparse.Namespace: Parsed command line arguments
+    """
+    parser = argparse.ArgumentParser(description="Evaluate AskVeracity performance")
+    parser.add_argument("--limit", type=int, help="Limit evaluation to first N claims")
+    parser.add_argument("--output", type=str, default="performance_results.json", 
+                        help="Output file for results (default: performance_results.json)")
+    return parser.parse_args()
+
+def initialize_system():
+    """
+    Initialize the system for evaluation.
+    
+    Returns:
+        object: Initialized LangGraph agent
+    """
+    print("Initializing models and agent...")
+    initialize_models()
+    eval_agent = agent.setup_agent()
+    return eval_agent
+
+def normalize_classification(classification):
+    """
+    Normalize classification labels for consistent comparison.
+    
+    Args:
+        classification (str): Classification label from the system
+        
+    Returns:
+        str: Normalized classification label ("True", "False", or "Uncertain")
+    """
+    if not classification:
+        return "Uncertain"
+    
+    if "true" in classification.lower():
+        return "True"
+    elif "false" in classification.lower():
+        return "False"
+    else:
+        return "Uncertain"
+
+def is_correct(actual, expected):
+    """
+    Determine if the actual classification matches the expected classification.
+    
+    Args:
+        actual (str): Actual classification from the system
+        expected (str): Expected (ground truth) classification
+        
+    Returns:
+        bool: True if classifications match, False otherwise
+    """
+    # Normalize both for comparison
+    normalized_actual = normalize_classification(actual)
+    normalized_expected = expected
+    
+    return normalized_actual == normalized_expected
+
+def is_safe(actual, expected):
+    """
+    Determine if the classification is "safe" - either correct or abstained (Uncertain)
+    instead of making an incorrect assertion.
+    
+    Args:
+        actual (str): Actual classification from the system
+        expected (str): Expected (ground truth) classification
+        
+    Returns:
+        bool: True if the classification is safe, False otherwise
+    """
+    # Normalize both for comparison
+    normalized_actual = normalize_classification(actual)
+    normalized_expected = expected
+    
+    # If the classification is correct, it's definitely safe
+    if normalized_actual == normalized_expected:
+        return True
+    
+    # If the system classified as "Uncertain", that's safe (abstaining rather than wrong assertion)
+    if normalized_actual == "Uncertain":
+        return True
+    
+    # Otherwise, the system made an incorrect assertion (False as True or True as False)
+    return False
+
+def evaluate_claims(test_claims, eval_agent, limit=None):
+    """
+    Evaluate a list of claims using the fact-checking system.
+    
+    Args:
+        test_claims (list): List of test claims with expected classifications
+        eval_agent (object): Initialized LangGraph agent
+        limit (int, optional): Maximum number of claims to evaluate
+        
+    Returns:
+        tuple: (results, metrics)
+            - results (list): Detailed results for each claim
+            - metrics (dict): Aggregated performance metrics
+    """
+    # Initialize performance tracker
+    performance_tracker = PerformanceTracker()
+    
+    # Limit the number of claims if requested
+    if limit and limit > 0:
+        claims_to_evaluate = test_claims[:limit]
+    else:
+        claims_to_evaluate = test_claims
+    
+    results = []
+    total_count = len(claims_to_evaluate)
+    correct_count = 0
+    safe_count = 0
+    
+    # Classification counts
+    classification_counts = {"True": 0, "False": 0, "Uncertain": 0}
+    
+    # Track processing times by expected classification
+    processing_times = {"True": [], "False": [], "Uncertain": []}
+    
+    # Confidence scores by expected classification
+    confidence_scores = {"True": [], "False": [], "Uncertain": []}
+    
+    # Track correct classifications by expected classification
+    correct_by_class = {"True": 0, "False": 0, "Uncertain": 0}
+    safe_by_class = {"True": 0, "False": 0, "Uncertain": 0}
+    total_by_class = {"True": 0, "False": 0, "Uncertain": 0}
+    
+    print(f"Evaluating {len(claims_to_evaluate)} claims...")
+    
+    # Process each claim
+    for idx, test_case in enumerate(claims_to_evaluate):
+        claim = test_case["claim"]
+        expected = test_case["expected"]
+        
+        print(f"\nProcessing claim {idx+1}/{len(claims_to_evaluate)}: {claim}")
+        
+        try:
+            # Process the claim and measure time
+            start_time = time.time()
+            result = agent.process_claim(claim, eval_agent)
+            total_time = time.time() - start_time
+            
+            # Extract classification and confidence
+            classification = result.get("classification", "Uncertain")
+            confidence = result.get("confidence", 0.0)
+            
+            # Normalize classification for comparison
+            normalized_classification = normalize_classification(classification)
+            
+            # Check if classification is correct
+            correct = is_correct(normalized_classification, expected)
+            if correct:
+                correct_count += 1
+                correct_by_class[expected] += 1
+            
+            # Check if classification is safe
+            safe = is_safe(normalized_classification, expected)
+            if safe:
+                safe_count += 1
+                safe_by_class[expected] += 1
+            
+            # Update classification count
+            classification_counts[normalized_classification] = classification_counts.get(normalized_classification, 0) + 1
+            
+            # Update counts by expected class
+            total_by_class[expected] += 1
+            
+            # Update processing times
+            processing_times[expected].append(total_time)
+            
+            # Update confidence scores
+            confidence_scores[expected].append(confidence)
+            
+            # Save detailed result
+            detail_result = {
+                "claim": claim,
+                "expected": expected,
+                "actual": normalized_classification,
+                "correct": correct,
+                "safe": safe,
+                "confidence": confidence,
+                "processing_time": total_time
+            }
+            
+            results.append(detail_result)
+            
+            # Print progress indicator
+            outcome = "✓" if correct else "✗"
+            safety = "(safe)" if safe and not correct else ""
+            print(f"  Result: {normalized_classification} (Expected: {expected}) {outcome} {safety}")
+            print(f"  Time: {total_time:.2f}s, Confidence: {confidence:.2f}")
+            
+        except Exception as e:
+            print(f"Error processing claim: {str(e)}")
+            results.append({
+                "claim": claim,
+                "expected": expected,
+                "error": str(e)
+            })
+    
+    # Calculate performance metrics
+    accuracy = correct_count / total_count if total_count > 0 else 0
+    safety_rate = safe_count / total_count if total_count > 0 else 0
+    
+    # Calculate per-class metrics
+    class_metrics = {}
+    for cls in ["True", "False", "Uncertain"]:
+        class_accuracy = correct_by_class[cls] / total_by_class[cls] if total_by_class[cls] > 0 else 0
+        class_safety_rate = safe_by_class[cls] / total_by_class[cls] if total_by_class[cls] > 0 else 0
+        avg_time = sum(processing_times[cls]) / len(processing_times[cls]) if processing_times[cls] else 0
+        avg_confidence = sum(confidence_scores[cls]) / len(confidence_scores[cls]) if confidence_scores[cls] else 0
+        
+        class_metrics[cls] = {
+            "accuracy": class_accuracy,
+            "safety_rate": class_safety_rate,
+            "count": total_by_class[cls],
+            "correct": correct_by_class[cls],
+            "safe": safe_by_class[cls],
+            "avg_processing_time": avg_time,
+            "avg_confidence": avg_confidence
+        }
+    
+    # Calculate overall metrics
+    all_times = [r.get("processing_time", 0) for r in results if "processing_time" in r]
+    all_confidence = [r.get("confidence", 0) for r in results if "confidence" in r]
+    
+    metrics = {
+        "total_claims": total_count,
+        "correct_claims": correct_count,
+        "safe_claims": safe_count,
+        "accuracy": accuracy,
+        "safety_rate": safety_rate,
+        "avg_processing_time": sum(all_times) / len(all_times) if all_times else 0,
+        "avg_confidence": sum(all_confidence) / len(all_confidence) if all_confidence else 0,
+        "classification_counts": classification_counts,
+        "per_class_metrics": class_metrics
+    }
+    
+    return results, metrics
+
+def save_results(results, metrics, output_file):
+    """
+    Save evaluation results to a JSON file.
+    
+    Args:
+        results (list): Detailed results for each claim
+        metrics (dict): Aggregated performance metrics
+        output_file (str): Path to output file
+    """
+    output_data = {
+        "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        "metrics": metrics,
+        "detailed_results": results
+    }
+    
+    with open(output_file, 'w') as f:
+        json.dump(output_data, f, indent=2)
+    
+    print(f"\nResults saved to {output_file}")
+
+def print_summary(metrics):
+    """
+    Print a summary of performance metrics.
+    
+    Args:
+        metrics (dict): Aggregated performance metrics
+    """
+    print("\n" + "="*70)
+    print(f"PERFORMANCE SUMMARY")
+    print("="*70)
+    
+    # Overall metrics
+    print(f"\nOverall Metrics:")
+    print(f"Total Claims: {metrics['total_claims']}")
+    print(f"Correctly Classified: {metrics['correct_claims']}")
+    print(f"Safely Classified: {metrics['safe_claims']}")
+    print(f"Accuracy: {metrics['accuracy']:.2%}")
+    print(f"Safety Rate: {metrics['safety_rate']:.2%}")
+    print(f"Average Processing Time: {metrics['avg_processing_time']:.2f} seconds")
+    print(f"Average Confidence Score: {metrics['avg_confidence']:.2f}")
+    
+    # Per-class metrics as table
+    print("\nPer-Class Performance:")
+    table_data = []
+    headers = ["Class", "Count", "Correct", "Safe", "Accuracy", "Safety Rate", "Avg Time", "Avg Confidence"]
+    
+    for cls, cls_metrics in metrics['per_class_metrics'].items():
+        table_data.append([
+            cls,
+            cls_metrics['count'],
+            cls_metrics['correct'],
+            cls_metrics['safe'],
+            f"{cls_metrics['accuracy']:.2%}",
+            f"{cls_metrics['safety_rate']:.2%}",
+            f"{cls_metrics['avg_processing_time']:.2f}s",
+            f"{cls_metrics['avg_confidence']:.2f}"
+        ])
+    
+    print(tabulate(table_data, headers=headers, tablefmt="grid"))
+
+def create_charts(metrics, output_dir="."):
+    """
+    Create visualizations of performance metrics.
+    
+    Args:
+        metrics (dict): Aggregated performance metrics
+        output_dir (str): Directory to save charts
+    """
+    try:
+        # Create output directory if it doesn't exist
+        os.makedirs(output_dir, exist_ok=True)
+        
+        # Plot 1: Accuracy by class
+        plt.figure(figsize=(10, 6))
+        classes = list(metrics['per_class_metrics'].keys())
+        accuracies = [metrics['per_class_metrics'][cls]['accuracy'] for cls in classes]
+        
+        plt.bar(classes, accuracies, color=['green', 'red', 'gray'])
+        plt.title('Accuracy by Classification Type')
+        plt.xlabel('Classification')
+        plt.ylabel('Accuracy')
+        plt.ylim(0, 1)
+        
+        for i, v in enumerate(accuracies):
+            plt.text(i, v + 0.02, f"{v:.2%}", ha='center')
+        
+        plt.tight_layout()
+        plt.savefig(os.path.join(output_dir, 'accuracy_by_class.png'))
+        plt.close()  # Close the figure to free memory
+        
+        # Plot 2: Safety Rate by class
+        plt.figure(figsize=(10, 6))
+        safety_rates = [metrics['per_class_metrics'][cls]['safety_rate'] for cls in classes]
+        
+        plt.bar(classes, safety_rates, color=['green', 'red', 'gray'])
+        plt.title('Safety Rate by Classification Type')
+        plt.xlabel('Classification')
+        plt.ylabel('Safety Rate')
+        plt.ylim(0, 1)
+        
+        for i, v in enumerate(safety_rates):
+            plt.text(i, v + 0.02, f"{v:.2%}", ha='center')
+        
+        plt.tight_layout()
+        plt.savefig(os.path.join(output_dir, 'safety_rate_by_class.png'))
+        plt.close()  # Close the figure to free memory
+        
+        # Plot 3: Processing time by class
+        plt.figure(figsize=(10, 6))
+        times = [metrics['per_class_metrics'][cls]['avg_processing_time'] for cls in classes]
+        
+        plt.bar(classes, times, color=['green', 'red', 'gray'])
+        plt.title('Average Processing Time by Classification Type')
+        plt.xlabel('Classification')
+        plt.ylabel('Time (seconds)')
+        
+        for i, v in enumerate(times):
+            plt.text(i, v + 0.5, f"{v:.2f}s", ha='center')
+        
+        plt.tight_layout()
+        plt.savefig(os.path.join(output_dir, 'processing_time_by_class.png'))
+        plt.close()  # Close the figure to free memory
+        
+        # Plot 4: Confidence scores by class
+        plt.figure(figsize=(10, 6))
+        confidence = [metrics['per_class_metrics'][cls]['avg_confidence'] for cls in classes]
+        
+        plt.bar(classes, confidence, color=['green', 'red', 'gray'])
+        plt.title('Average Confidence Score by Classification Type')
+        plt.xlabel('Classification')
+        plt.ylabel('Confidence Score')
+        plt.ylim(0, 1)
+        
+        for i, v in enumerate(confidence):
+            plt.text(i, v + 0.02, f"{v:.2f}", ha='center')
+        
+        plt.tight_layout()
+        plt.savefig(os.path.join(output_dir, 'confidence_by_class.png'))
+        plt.close()  # Close the figure to free memory
+        
+        print(f"\nCharts created in {output_dir}")
+        
+    except Exception as e:
+        print(f"Error creating charts: {str(e)}")
+        print("Continuing without charts.")
+
+def main():
+    """Main evaluation function that runs the entire evaluation process."""
+    # Parse arguments
+    args = setup_argument_parser()
+    
+    # Initialize the agent
+    eval_agent = initialize_system()
+    
+    # Create results directory if it doesn't exist
+    results_dir = "results"
+    os.makedirs(results_dir, exist_ok=True)
+    
+    # Set output file path
+    output_file = args.output
+    if not os.path.isabs(output_file):
+        output_file = os.path.join(results_dir, output_file)
+    
+    # Evaluate claims
+    results, metrics = evaluate_claims(TEST_CLAIMS, eval_agent, args.limit)
+    # results, metrics = evaluate_claims(TEST_CLAIMS, eval_agent, 1)
+    
+    # Print summary
+    print_summary(metrics)
+    
+    # Save results
+    save_results(results, metrics, output_file)
+    
+    # Create charts
+    try:
+        from tabulate import tabulate
+        import matplotlib.pyplot as plt
+        create_charts(metrics, results_dir)
+    except ImportError:
+        print("\nCould not create charts. Please install matplotlib and tabulate packages:")
+        print("pip install matplotlib tabulate")
+
+if __name__ == "__main__":
+    main()

modules/__init__.py

@@ -0,0 +1,19 @@
+"""
+Modules package initialization.
+
+This package contains the core modules for the AskVeracity fact-checking system.
+"""
+
+from .claim_extraction import extract_claims, shorten_claim_for_evidence
+from .evidence_retrieval import retrieve_combined_evidence
+from .classification import classify_with_llm, aggregate_evidence
+from .explanation import generate_explanation
+
+__all__ = [
+    'extract_claims',
+    'shorten_claim_for_evidence',
+    'retrieve_combined_evidence',
+    'classify_with_llm',
+    'aggregate_evidence',
+    'generate_explanation'
+]

modules/category_detection.py

@@ -0,0 +1,716 @@
+import logging
+import re
+from typing import Tuple, List, Dict, Optional
+import os
+import time
+
+# Set up logging
+logger = logging.getLogger("misinformation_detector")
+
+# Define categories and their keywords
+CLAIM_CATEGORIES = {
+    "ai": [
+        # General AI terms
+        "AI", "artificial intelligence", "machine learning", "ML", "deep learning", "DL", 
+        "neural network", "neural nets", "generative AI", "GenAI", "AGI", "artificial general intelligence",
+        "transformer", "attention mechanism", "fine-tuning", "pre-training", "training", "inference",
+        
+        # AI Models and Architectures
+        "language model", "large language model", "LLM", "foundation model", "multimodal model",
+        "vision language model", "VLM", "text-to-speech", "TTS", "speech-to-text", "STT",
+        "text-to-image", "image-to-text", "diffusion model", "generative model", "discriminative model",
+        "GPT", "BERT", "T5", "PaLM", "Claude", "Llama", "Gemini", "Mistral", "Mixtral", "Stable Diffusion",
+        "Dall-E", "Midjourney", "Sora", "transformer", "MoE", "mixture of experts", "sparse model", 
+        "dense model", "encoder", "decoder", "encoder-decoder", "autoencoder", "VAE",
+        "mixture of experts", "MoE", "sparse MoE", "switch transformer", "gated experts",
+        "routing network", "expert routing", "pathways", "multi-query attention", "multi-head attention",
+        "rotary position embedding", "RoPE", "grouped-query attention", "GQA", "flash attention",
+        "state space model", "SSM", "mamba", "recurrent neural network", "RNN", "LSTM", "GRU",
+        "convolutional neural network", "CNN", "residual connection", "skip connection", "normalization",
+        "layer norm", "group norm", "batch norm", "parameter efficient fine-tuning", "PEFT",
+        "LoRA", "low-rank adaptation", "QLoRA", "adapters", "prompt tuning", "prefix tuning",
+        
+        # AI Learning Paradigms
+        "supervised learning", "unsupervised learning", "reinforcement learning", "RL", 
+        "meta-learning", "transfer learning", "federated learning", "self-supervised learning", 
+        "semi-supervised learning", "few-shot learning", "zero-shot learning", "one-shot learning",
+        "contrastive learning", "curriculum learning", "imitation learning", "active learning",
+        "reinforcement learning from human feedback", "RLHF", "direct preference optimization", "DPO",
+        "constitutional AI", "red teaming", "adversarial training", "GAN", "generative adversarial network",
+        "diffusion", "latent diffusion", "flow-based model", "variational autoencoder", "VAE",
+        
+        # AI Capabilities and Applications
+        "natural language processing", "NLP", "computer vision", "CV", "speech recognition",
+        "text generation", "image generation", "video generation", "multimodal", "multi-modal",
+        "recommendation system", "recommender system", "chatbot", "conversational AI",
+        "sentiment analysis", "entity recognition", "semantic search", "vector search", "embedding",
+        "classification", "regression", "clustering", "anomaly detection", "agent", "AI agent",
+        "autonomous agent", "agentic", "RAG", "retrieval augmented generation", "tool use",
+        "function calling", "reasoning", "chain-of-thought", "CoT", "tree-of-thought", "ToT",
+        "planning", "decision making", "multi-agent", "agent swarm", "multi-agent simulation",
+        
+        # AI Technical Terms
+        "token", "tokenizer", "tokenization", "embedding", "vector", "prompt", "prompt engineering",
+        "context window", "parameter", "weights", "bias", "activation function", "loss function",
+        "gradient descent", "backpropagation", "epoch", "batch", "mini-batch", "regularization",
+        "dropout", "overfitting", "underfitting", "hyperparameter", "latent space", "latent variable",
+        "feature extraction", "dimensionality reduction", "quantization", "pruning",
+        "fine-tuning", "transfer learning", "knowledge distillation", "int4", "int8", "bfloat16",
+        "float16", "mixed precision", "GPTQ", "AWQ", "GGUF", "GGML", "KV cache", "speculative decoding",
+        "beam search", "greedy decoding", "temperature", "top-k", "top-p", "nucleus sampling",
+        
+        # AI Tools and Frameworks
+        "TensorFlow", "PyTorch", "JAX", "Keras", "Hugging Face", "Transformers", "Diffusers",
+        "LangChain", "Llama Index", "OpenAI", "Anthropic", "NVIDIA", "GPU", "TPU", "IPU", "NPU", "CUDA",
+        "MLOps", "model monitoring", "model deployment", "model serving", "inference endpoint",
+        "vLLM", "TGI", "text generation inference", "triton", "onnx", "tensorRT",
+        
+        # AI Ethics and Concerns
+        "AI ethics", "responsible AI", "AI safety", "AI alignment", "AI governance",
+        "bias", "fairness", "interpretability", "explainability", "XAI", "transparency",
+        "hallucination", "toxicity", "safe deployment", "AI risk", "AI capabilities",
+        "alignment tax", "red teaming", "jailbreak", "prompt injection", "data poisoning",
+        
+        # AI Companies and Organizations
+        "OpenAI", "Anthropic", "Google DeepMind", "Meta AI", "Microsoft", "NVIDIA", 
+        "Hugging Face", "Mistral AI", "Cohere", "AI21 Labs", "Stability AI", "Midjourney",
+        "EleutherAI", "Allen AI", "DeepMind", "Character AI", "Inflection AI", "xAI"
+    ],
+    
+    "science": [
+        # General scientific terms
+        "study", "research", "scientist", "scientific", "discovered", "experiment", 
+        "laboratory", "clinical", "trial", "hypothesis", "theory", "evidence-based",
+        "peer-reviewed", "journal", "publication", "finding", "breakthrough", "innovation",
+        "discovery", "analysis", "measurement", "observation", "empirical",
+        
+        # Biology and medicine
+        "biology", "chemistry", "physics", "genetics", "genomics", "DNA", "RNA", 
+        "medicine", "gene", "protein", "molecule", "cell", "brain", "neuro", 
+        "cancer", "disease", "cure", "treatment", "vaccine", "health", "medical",
+        "pharmaceutical", "drug", "therapy", "symptom", "diagnosis", "prognosis",
+        "patient", "doctor", "hospital", "clinic", "surgery", "immune", "antibody",
+        "virus", "bacteria", "pathogen", "infection", "epidemic", "pandemic",
+        "organism", "evolution", "mutation", "chromosome", "enzyme", "hormone",
+        
+        # Physics and astronomy
+        "quantum", "particle", "atom", "nuclear", "electron", "neutron", "proton",
+        "atomic", "subatomic", "molecular", "energy", "matter", "mass", "force",
+        "space", "NASA", "telescope", "planet", "exoplanet", "moon", "lunar", "mars",
+        "star", "galaxy", "cosmic", "astronomical", "universe", "solar", "celestial",
+        "orbit", "gravitational", "gravity", "relativity", "quantum mechanics",
+        "string theory", "dark matter", "dark energy", "black hole", "supernova",
+        "radiation", "radioactive", "isotope", "fission", "fusion", "accelerator",
+        
+        # Environmental science
+        "climate", "carbon", "environment", "ecosystem", "species", "extinct",
+        "endangered", "biodiversity", "conservation", "sustainable", "renewable",
+        "fossil fuel", "greenhouse", "global warming", "polar", "ice cap", "glacier",
+        "ozone", "atmosphere", "weather", "meteorology", "geology", "earthquake",
+        "volcanic", "ocean", "marine", "coral reef", "deforestation", "pollution",
+        
+        # Math and computer science (non-AI specific)
+        "equation", "formula", "theorem", "calculus", "statistical", "probability",
+        "variable", "matrix", "optimization",
+        
+        # Organizations
+        "CERN", "NIH", "CDC", "WHO", "NOAA", "ESA", "SpaceX", "Blue Origin", "JPL",
+        "laboratory", "institute", "university", "academic", "faculty", "professor",
+        
+        # Science tools
+        "Matlab", "SPSS", "SAS", "ImageJ", "LabVIEW", "ANSYS", "Cadence", "Origin",
+        "Avogadro", "ChemDraw", "Mathematica", "Wolfram Alpha", "COMSOL", "LAMMPS",
+        "VASP", "Gaussian", "GIS", "ArcGIS", "QGIS", "Maple", "R Studio"
+    ],
+    
+    "technology": [
+        # General tech terms
+        "computer", "hardware", "internet", "cyber", "digital", "tech", 
+        "robot", "automation", "autonomous", "code", "programming", "data", "cloud", 
+        "server", "network", "encryption", "blockchain", "crypto", "bitcoin", "ethereum",
+        "technology", "breakthrough", "prototype", "dataset",
+        "engineering", "technical", "specification", "feature", "functionality",
+        "interface", "system", "infrastructure", "integration", "implementation",
+        
+        # Devices and hardware
+        "smartphone", "device", "gadget", "laptop", "desktop", "tablet", "wearable",
+        "smartwatch", "IoT", "internet of things", "sensor", "chip", "semiconductor",
+        "processor", "CPU", "GPU", "memory", "RAM", "storage", "hard drive", "SSD",
+        "electronic", "circuit", "motherboard", "component", "peripheral", "accessory",
+        "display", "screen", "touchscreen", "camera", "lens", "microphone", "speaker",
+        "battery", "charger", "wireless", "bluetooth", "WiFi", "router", "modem",
+        
+        # Software and internet
+        "algorithm", "app", "application", "platform", "website", "online", "web", "browser",
+        "operating system", "Windows", "macOS", "Linux", "Android", "iOS", "software",
+        "program", "code", "coding", "development", "framework", "library", "API",
+        "backend", "frontend", "full-stack", "developer", "programmer", "function",
+        "database", "SQL", "NoSQL", "cloud computing", "SaaS", "PaaS", "IaaS",
+        "DevOps", "agile", "scrum", "sprint", "version control", "git", "repository",
+        
+        # Communications and networking
+        "5G", "6G", "broadband", "fiber", "network", "wireless", "cellular", "mobile",
+        "telecommunications", "telecom", "transmission", "bandwidth", "latency",
+        "protocol", "IP address", "DNS", "server", "hosting", "data center",
+        
+        # Company and product names
+        "Apple", "Google", "Microsoft", "Amazon", "Facebook", "Meta", "Tesla", 
+        "IBM", "Intel", "AMD", "Nvidia", "Qualcomm", "Cisco", "Oracle", "SAP", 
+        "Huawei", "Samsung", "Sony", "LG", "Dell", "HP", "Lenovo", "Xiaomi",
+        "iPhone", "iPad", "MacBook", "Surface", "Galaxy", "Pixel", "Windows",
+        "Android", "iOS", "Chrome", "Firefox", "Edge", "Safari", "Office",
+        "Azure", "AWS", "Google Cloud", "Gmail", "Outlook", "Teams", "Zoom",
+        
+        # Advanced technologies
+        "VR", "AR", "XR", "virtual reality", "augmented reality", "mixed reality",
+        "metaverse", "3D printing", "additive manufacturing", "quantum computing",
+        "nanotechnology", "biotechnology", "electric vehicle", "self-driving",
+        "autonomous vehicle", "drone", "UAV", "robotics", "cybersecurity",
+        
+        # Social media
+        "social media", "social network", "Facebook", "Instagram", "Twitter", "X",
+        "LinkedIn", "TikTok", "Snapchat", "YouTube", "Pinterest", "Reddit",
+        "streaming", "content creator", "influencer", "follower", "like", "share",
+        "post", "tweet", "user-generated", "viral", "trending", "engagement",
+        
+        # Technology tools
+        "NumPy", "Pandas", "Matplotlib", "Seaborn", "Scikit-learn", "Jupyter",
+        "Visual Studio", "VS Code", "IntelliJ", "PyCharm", "Eclipse", "Android Studio",
+        "Xcode", "Docker", "Kubernetes", "Jenkins", "Ansible", "Terraform", "Vagrant",
+        "AWS CLI", "Azure CLI", "GCP CLI", "PowerShell", "Bash", "npm", "pip", "conda",
+        "React", "Angular", "Vue.js", "Node.js", "Django", "Flask", "Spring", "Laravel",
+        "PostgreSQL", "MySQL", "MongoDB", "Redis", "Elasticsearch", "Kafka", "RabbitMQ",
+        
+        # Optimization terms
+        "efficiency", "performance tuning", "benchmarking", "profiling",
+        "refactoring", "scaling", "bottleneck", "throughput", "latency reduction",
+        "response time", "caching", "load balancing", "distributed computing",
+        "parallel processing", "concurrency", "asynchronous", "memory management"
+    ],
+    
+    "politics": [
+        # Government structure
+        "president", "prime minister", "government", "parliament", "congress", 
+        "senate", "house", "representative", "minister", "secretary", "cabinet",
+        "administration", "mayor", "governor", "politician", "official", "authority",
+        "federal", "state", "local", "municipal", "county", "city", "town",
+        "constituency", "district", "precinct", "ward", "judiciary", "executive",
+        "legislative", "branch", "checks and balances", "separation of powers",
+        
+        # Political activities
+        "election", "campaign", "vote", "voter", "ballot", "polling",
+        "political", "politics", "debate", "speech", "address", "press conference",
+        "approval rating", "opinion poll", "candidate", "incumbent", "challenger",
+        "primary", "caucus", "convention", "delegate", "nomination", "campaign trail",
+        "fundraising", "lobbying", "advocacy", "activism", "protest", "demonstration",
+        
+        # Political ideologies
+        "democracy", "democratic", "republican", "conservative", "liberal", 
+        "progressive", "left-wing", "right-wing", "centrist", "moderate",
+        "socialist", "capitalist", "communist", "libertarian", "populist",
+        "nationalist", "globalist", "isolationist", "hawk", "dove",
+        "ideology", "partisan", "bipartisan", "coalition", "majority", "minority",
+        
+        # Laws and regulations
+        "bill", "law", "legislation", "regulation", "policy", "statute", "code",
+        "amendment", "reform", "repeal", "enact", "implement", "enforce",
+        "constitutional", "unconstitutional", "legal", "illegal", "legalize",
+        "criminalize", "deregulate", "regulatory", "compliance", "mandate",
+        
+        # Judicial and legal
+        "court", "supreme", "justice", "judge", "ruling", "decision", "opinion",
+        "case", "lawsuit", "litigation", "plaintiff", "defendant", "prosecutor",
+        "attorney", "lawyer", "advocate", "judicial review", "precedent",
+        "constitution", "amendment", "rights", "civil rights", "human rights",
+        
+        # International relations
+        "treaty", "diplomatic", "diplomacy", "relations",
+        "foreign policy", "domestic policy", "UN", "NATO", "EU", "United Nations",
+        "sanctions", "embargo", "tariff", "trade war", "diplomat", "embassy",
+        "consulate", "ambassador", "delegation", "summit", "bilateral", "multilateral",
+        "alliance", "ally", "adversary", "geopolitical", "sovereignty", "regime",
+        
+        # Security and defense
+        "national security", "homeland security", "defense", "military", "armed forces",
+        "army", "navy", "air force", "marines", "coast guard", "intelligence",
+        "CIA", "FBI", "NSA", "Pentagon", "war", "conflict", "peacekeeping",
+        "terrorism", "counterterrorism", "insurgency", "nuclear weapon", "missile",
+        "disarmament", "nonproliferation", "surveillance", "espionage",
+        
+        # Political institutions
+        "White House", "Kremlin", "Downing Street", "Capitol Hill", "Westminster",
+        "United Nations", "European Union", "NATO", "World Bank", "IMF", "WTO",
+        "ASEAN", "African Union", "BRICS", "G7", "G20",
+        
+        # Political parties and movements
+        "Democrat", "Republican", "Labour", "Conservative", "Green Party",
+        "Socialist", "Communist", "Libertarian", "Independent", "Tea Party",
+        "progressive movement", "civil rights movement", "womens rights",
+        "LGBTQ rights", "Black Lives Matter", "environmental movement"
+    ],
+    
+    "business": [
+        # Companies and organization types
+        "company", "corporation", "business", "startup", "firm", "enterprise", 
+        "corporate", "industry", "sector", "conglomerate", "multinational",
+        "organization", "entity", "private", "public", "incorporated", "LLC",
+        "partnership", "proprietorship", "franchise", "subsidiary", "parent company",
+        "headquarters", "office", "facility", "plant", "factory", "warehouse",
+        "retail", "wholesale", "ecommerce", "brick-and-mortar", "chain", "outlet",
+        
+        # Business roles and management
+        "executive", "CEO", "CFO", "CTO", "COO", "CMO", "CIO", "CHRO", "chief",
+        "director", "board", "chairman", "chairwoman", "chairperson", "president",
+        "vice president", "senior", "junior", "manager", "management", "supervisor",
+        "founder", "entrepreneur", "owner", "shareholder", "stakeholder",
+        "employee", "staff", "workforce", "personnel", "human resources", "HR",
+        "recruit", "hire", "layoff", "downsizing", "restructuring", "reorganization",
+        "leadership",
+        
+        # Financial terms
+        "profit", "revenue", "sales", "income", "earnings", "EBITDA", "turnover", 
+        "loss", "deficit", "expense", "cost", "overhead", "margin", "markup",
+        "budget", "forecast", "projection", "estimate", "actual", "variance",
+        "balance sheet", "income statement", "cash flow", "P&L", "liquidity",
+        "solvency", "asset", "liability", "equity", "debt", "leverage", "capital",
+        "working capital", "cash", "funds", "money", "payment", "transaction",
+        
+        # Markets and trading
+        "market", "stock", "share", "bond", "security", "commodity", "futures",
+        "option", "derivative", "forex", "foreign exchange", "currency", "crypto",
+        "trader", "trading", "buy", "sell", "long", "short", "position", "portfolio",
+        "diversification", "hedge", "risk", "return", "yield", "dividend", "interest",
+        "bull market", "bear market", "correction", "crash", "rally", "volatile",
+        "volatility", "index", "benchmark", "Dow Jones", "NASDAQ", "S&P 500", "NYSE",
+        
+        # Investment and funding
+        "investor", "investment", "fund", "mutual fund", "ETF", "hedge fund", 
+        "private equity", "venture", "venture capital", "VC", "angel investor",
+        "seed", "Series A", "Series B", "Series C", "funding", "financing",
+        "loan", "credit", "debt", "equity", "fundraising", "crowdfunding",
+        "IPO", "initial public offering", "going public", "listed", "delisted",
+        "merger", "acquisition", "M&A", "takeover", "buyout", "divestiture",
+        "valuation", "billion", "million", "trillion", "unicorn", "decacorn",
+        
+        # Economic terms
+        "economy", "economic", "economics", "macro", "micro", "fiscal", "monetary",
+        "supply", "demand", "market forces", "competition", "competitive", "monopoly",
+        "oligopoly", "antitrust", "deregulation", "growth", "decline",
+        "recession", "depression", "recovery", "expansion", "contraction", "cycle",
+        "inflation", "deflation", "stagflation", "hyperinflation", "CPI", "price",
+        "GDP", "gross domestic product", "GNP", "productivity", "output", "input",
+        
+        # Banking and finance
+        "finance", "financial", "bank", "banking", "commercial bank", "investment bank",
+        "central bank", "Federal Reserve", "Fed", "ECB", "Bank of England", "BOJ",
+        "interest rate", "prime rate", "discount rate", "basis point", "monetary policy",
+        "quantitative easing", "tightening", "loosening", "credit", "lending",
+        "borrowing", "loan", "mortgage", "consumer credit", "credit card", "debit card",
+        "checking", "savings", "deposit", "withdrawal", "ATM", "branch", "online banking",
+        
+        # Currencies and payments
+        "dollar", "euro", "pound", "yen", "yuan", "rupee", "ruble", "real", "peso",
+        "currency", "money", "fiat", "exchange rate", "remittance", "transfer",
+        "payment", "transaction", "wire", "ACH", "SWIFT", "clearing", "settlement",
+        "cryptocurrency", "bitcoin", "ethereum", "blockchain", "fintech", "paytech",
+        
+        # Business operations
+        "product", "service", "solution", "offering", "launch", "rollout", "release",
+        "operation", "production", "manufacturing", "supply chain", "logistics",
+        "procurement", "inventory", "distribution", "shipping", "delivery",
+        "quality", "control", "assurance", "standard", "certification",
+
+        # Marketing and sales
+        "marketing", "advertise", "advertising", "campaign", "promotion", "publicity",
+        "PR", "public relations", "brand", "branding", "identity", "image", "reputation",
+        "sales", "selling", "deal", "transaction", "pipeline", "lead", "prospect",
+        "customer", "client", "consumer", "buyer", "purchaser", "target market",
+        "segment", "demographic", "psychographic", "B2B", "B2C", "retail", "wholesale",
+        "price", "pricing", "discount", "premium", "luxury", "value", "bargain"
+    ],
+    
+    "world": [
+        # General international terms
+        "country", "nation", "state", "republic", "kingdom", "global", "international", 
+        "foreign", "world", "worldwide", "domestic", "abroad", "overseas",
+        "developed", "developing", "industrialized", "emerging", "third world",
+        "global south", "global north", "east", "west", "western", "eastern",
+        "bilateral", "multilateral", "transnational", "multinational", "sovereignty",
+        
+        # Regions and continents
+        "Europe", "European", "Asia", "Asian", "Africa", "African", "North America",
+        "South America", "Latin America", "Australia", "Oceania", "Antarctica",
+        "Middle East", "Central Asia", "Southeast Asia", "East Asia", "South Asia",
+        "Eastern Europe", "Western Europe", "Northern Europe", "Southern Europe",
+        "Mediterranean", "Scandinavia", "Nordic", "Baltic", "Balkans", "Caucasus",
+        "Caribbean", "Central America", "South Pacific", "Polynesia", "Micronesia",
+        
+        # Major countries and regions
+        "China", "Chinese", "Russia", "Russian", "India", "Indian", "Japan", "Japanese", 
+        "UK", "British", "England", "English", "Scotland", "Scottish", "Wales", "Welsh",
+        "Germany", "German", "France", "French", "Italy", "Italian", "Spain", "Spanish",
+        "Canada", "Canadian", "Brazil", "Brazilian", "Mexico", "Mexican", "Turkey", "Turkish",
+        "United States", "US", "USA", "American", "Britain", "Korea", "Korean",
+        "North Korea", "South Korea", "Saudi", "Saudi Arabia", "Saudi Arabian",
+        "Iran", "Iranian", "Iraq", "Iraqi", "Israel", "Israeli", "Palestine", "Palestinian",
+        "Egypt", "Egyptian", "Pakistan", "Pakistani", "Indonesia", "Indonesian",
+        "Australia", "Australian", "New Zealand", "Nigeria", "Nigerian", "South Africa",
+        "Argentina", "Argentinian", "Colombia", "Colombian", "Venezuela", "Venezuelan",
+        "Ukraine", "Ukrainian", "Poland", "Polish", "Switzerland", "Swiss",
+        "Netherlands", "Dutch", "Belgium", "Belgian", "Sweden", "Swedish", "Norway", "Norwegian",
+        
+        # International issues and topics
+        "war", "conflict", "crisis", "tension", "dispute", "hostility", "peace",
+        "peacekeeping", "ceasefire", "truce", "armistice", "treaty", "agreement",
+        "compromise", "negotiation", "mediation", "resolution", "settlement",
+        "refugee", "migrant", "asylum seeker", "displacement", "humanitarian",
+        "border", "frontier", "territory", "territorial", "sovereignty", "jurisdiction",
+        "terror", "terrorism", "extremism", "radicalism", "insurgency", "militant",
+        "sanction", "embargo", "restriction", "isolation", "blockade",
+        
+        # International trade and economy
+        "trade", "import", "export", "tariff", "duty", "quota", "subsidy",
+        "protectionism", "free trade", "fair trade", "globalization", "trade war",
+        "trade agreement", "trade deal", "trade deficit", "trade surplus",
+        "supply chain", "outsourcing", "offshoring", "reshoring", "nearshoring",
+        
+        # Diplomacy and international relations
+        "embassy", "consulate", "diplomatic", "diplomacy", "diplomat", "ambassador",
+        "consul", "attaché", "envoy", "emissary", "delegation", "mission",
+        "foreign policy", "international relations", "geopolitics", "geopolitical",
+        "influence", "power", "superpower", "hegemony", "alliance", "coalition",
+        "bloc", "axis", "sphere of influence", "buffer state", "proxy",
+        
+        # International organizations
+        "UN", "United Nations", "EU", "European Union", "NATO", "NAFTA", "USMCA",
+        "ASEAN", "OPEC", "Commonwealth", "Arab League", "African Union", "AU",
+        "BRICS", "G7", "G20", "IMF", "World Bank", "WTO", "WHO", "UNESCO",
+        "Security Council", "General Assembly", "International Court of Justice",
+        
+        # Travel and cultural exchange
+        "visa", "passport", "immigration", "emigration", "migration", "travel",
+        "tourism", "tourist", "visitor", "foreigner", "expatriate", "expat",
+        "citizenship", "nationality", "dual citizen", "naturalization",
+        "cultural", "tradition", "heritage", "indigenous", "native", "local",
+        "language", "dialect", "translation", "interpreter", "cross-cultural",
+
+        # Other
+        "event"
+    ],
+    
+    "sports": [
+        # General sports terms
+        "game", "match", "tournament", "championship", "league", "cup", "Olympics", 
+        "olympic", "world cup", "competition", "contest",
+        "sport", "sporting", "athletics", "physical", "play", "compete", "competition",
+        "amateur", "professional", "pro", "preseason", "regular season",
+        "postseason", "playoff", "final", "semifinal", "quarterfinal", "qualifying",
+        
+        # Team sports
+        "football", "soccer", "American football", "rugby", "basketball", "baseball", 
+        "cricket", "hockey", "ice hockey", "field hockey", "volleyball", "handball",
+        "water polo", "lacrosse", "ultimate frisbee", "netball", "kabaddi",
+        "team", "club", "franchise", "squad", "roster", "lineup", "formation",
+        "player", "coach", "manager", "trainer", "captain", "starter", "substitute",
+        "bench", "draft", "trade", "free agent", "contract", "transfer", "loan",
+        
+        # Individual sports
+        "tennis", "golf", "boxing", "wrestling", "martial arts", "MMA", "UFC",
+        "athletics", "track and field", "swimming", "diving", "gymnastics",
+        "skiing", "snowboarding", "skating", "figure skating", "speed skating",
+        "cycling", "mountain biking", "BMX", "motorsport", "F1", "Formula 1",
+        "NASCAR", "IndyCar", "MotoGP", "rally", "marathon", "triathlon", "decathlon",
+        "archery", "shooting", "fencing", "equestrian", "rowing", "canoeing", "kayaking",
+        "surfing", "skateboarding", "climbing", "bouldering", "weightlifting",
+        
+        # Scoring and results
+        "score", "point", "goal", "touchdown", "basket", "run", "wicket", "try",
+        "win", "lose", "draw", "tie", "defeat", "victory", "champion", "winner",
+        "loser", "runner-up", "finalist", "semifinalist", "eliminated", "advance",
+        "qualify", "record", "personal best", "world record", "Olympic record",
+        "streak", "undefeated", "unbeaten", "perfect season", "comeback",
+        
+        # Performance and training
+        "fitness", "training", "practice", "drill", "workout", "exercise", "regime",
+        "conditioning", "strength", "endurance", "speed", "agility", "flexibility",
+        "skill", "technique", "form", "style", "strategy", "tactic", "playbook",
+        "offense", "defense", "attack", "counter", "press", "formation",
+        "injury", "rehabilitation", "recovery", "physiotherapy", "sports medicine",
+        
+        # Sports infrastructure
+        "stadium", "arena", "court", "field", "pitch", "rink", "pool", "track",
+        "course", "gymnasium", "gym", "complex", "venue", "facility", "locker room",
+        "dugout", "bench", "sideline", "grandstand", "spectator", "fan", "supporter",
+        
+        # Sports organizations and competitions
+        "medal", "gold", "silver", "bronze", "podium", "Olympics", "Paralympic",
+        "commonwealth games", "Asian games", "Pan American games", "world championship",
+        "grand slam", "masters", "open", "invitational", "classic", "tour", "circuit",
+        "IPL", "Indian Premier League", "MLB", "Major League Baseball", 
+        "NBA", "National Basketball Association", "NFL", "National Football League", 
+        "NHL", "National Hockey League", "FIFA", "UEFA", "ATP", "WTA", "ICC",
+        "Premier League", "La Liga", "Bundesliga", "Serie A", "Ligue 1", "MLS",
+        "Champions League", "Europa League", "Super Bowl", "World Series", "Stanley Cup",
+        "NCAA", "collegiate", "college", "university", "varsity", "intramural",
+        
+        # Sports media and business
+        "broadcast", "coverage", "commentator", "announcer", "pundit", "analyst",
+        "highlight", "replay", "sports network", "ESPN", "Sky Sports", "Fox Sports",
+        "sponsorship", "endorsement", "advertisement", "merchandise", "jersey", "kit",
+        "ticket", "season ticket", "box seat", "premium", "concession", "vendor",
+        # Sports media and business (continued)
+        "broadcast", "coverage", "commentator", "announcer", "pundit", "analyst",
+        "highlight", "replay", "sports network", "ESPN", "Sky Sports", "Fox Sports",
+        "sponsorship", "endorsement", "advertisement", "merchandise", "jersey", "kit",
+        "ticket", "season ticket", "box seat", "premium", "concession", "vendor"
+    ],
+    
+    "entertainment": [
+        # Film and cinema
+        "movie", "film", "cinema", "feature", "short film", "documentary", "animation",
+        "blockbuster", "indie", "independent film", "foreign film", "box office",
+        "screening", "premiere", "release", "theatrical", "stream", "streaming",
+        "director", "producer", "screenwriter", "script", "screenplay", "adaptation",
+        "cinematography", "cinematographer", "editing", "editor", "visual effects",
+        "special effects", "CGI", "motion capture", "sound design", "soundtrack",
+        "score", "composer", "scene", "shot", "take", "cut", "sequel", "prequel",
+        "trilogy", "franchise", "universe", "reboot", "remake", "spin-off",
+        "genre", "action", "comedy", "drama", "thriller", "horror", "sci-fi",
+        "science fiction", "fantasy", "romance", "romantic comedy", "rom-com",
+        "mystery", "crime", "western", "historical", "biographical", "biopic",
+        
+        # Television
+        "TV", "television", "show", "episode",
+        "finale", "midseason", "sitcom", "drama series", "miniseries", "limited series",
+        "anthology", "reality TV", "game show", "talk show", "variety show",
+        "network", "cable", "premium cable", "broadcast", "channel", "program",
+        "primetime", "daytime", "syndication", "rerun", "renewed", "cancelled",
+        "showrunner", "creator", "writer", "TV writer", "episode writer", "staff writer",
+        
+        # Performing arts
+        "actor", "actress", "performer", "cast", "casting", "star", "co-star",
+        "supporting", "lead", "protagonist", "antagonist", "villain", "hero", "anti-hero",
+        "character", "role", "portrayal", "acting", "dialogue",
+        "monologue", "line", "script", "improv", "improvisation", "stand-up",
+        "comedian", "comic", "sketch", "theater", "theatre", "stage", "Broadway",
+        "West End", "play", "musical", "opera", "ballet", "dance", "choreography",
+        "production", "rehearsal", "audition", "understudy", "troupe", "ensemble",
+        
+        # Music
+        "music", "song", "track", "single", "album", "EP", "LP", "record",
+        "release", "drop", "artist", "musician", "singer", "vocalist", "band",
+        "group", "duo", "trio", "soloist", "frontman", "frontwoman", "lead singer",
+        "songwriter", "composer", "producer", "DJ", "rapper", "MC", "beatmaker",
+        "guitarist", "bassist", "drummer", "pianist", "keyboardist", "violinist",
+        "instrumentalist", "orchestra", "symphony", "philharmonic", "conductor",
+        "genre", "rock", "pop", "hip-hop", "rap", "R&B", "soul", "funk", "jazz",
+        "blues", "country", "folk", "electronic", "EDM", "dance", "techno", "house",
+        "metal", "punk", "alternative", "indie", "classical", "reggae", "latin",
+        "hit", "chart", "Billboard", "Grammy", "award-winning", "platinum", "gold",
+        "concert", "tour", "gig", "show", "venue", "arena",
+        "stadium", "festival", "Coachella", "Glastonbury", "Lollapalooza", "Bonnaroo",
+        
+        # Celebrity culture
+        "celebrity", "star", "fame", "famous", "A-list", "B-list", "icon", "iconic",
+        "superstar", "public figure", "household name", "stardom", "limelight",
+        "popular", "popularity", "fan", "fanbase", "followers", "stan", "groupie",
+        "paparazzi", "tabloid", "gossip", "rumor", "scandal", "controversy",
+        "interview", "press conference", "red carpet", "premiere", "gala", "award show",
+        
+        # Awards and recognition
+        "award", "nominee", "nomination", "winner", "recipient", "honor", "accolade",
+        "Oscar", "Academy Award", "Emmy", "Grammy", "Tony", "Golden Globe", "BAFTA",
+        "MTV Award", "People's Choice", "Critics' Choice", "SAG Award", "Billboard Award",
+        "best actor", "best actress", "best director", "best picture", "best film",
+        "best album", "best song", "hall of fame", "lifetime achievement", "legacy",
+        
+        # Media and publishing
+        "book", "novel", "fiction", "non-fiction", "memoir", "biography", "autobiography",
+        "bestseller", "bestselling", "author", "writer", "novelist", "literary",
+        "literature", "publisher", "publishing", "imprint", "edition", "volume",
+        "chapter", "page", "paragraph", "prose", "narrative", "plot", "storyline",
+        "character", "protagonist", "antagonist", "setting", "theme", "genre",
+        "mystery", "thriller", "romance", "sci-fi", "fantasy", "young adult", "YA",
+        "comic", "comic book", "graphic novel", "manga", "anime", "cartoon",
+        
+        # Digital entertainment
+        "streaming", "stream", "subscription", "platform", "service", "content",
+        "Netflix", "Disney+", "Amazon Prime", "Hulu", "HBO", "HBO Max", "Apple TV+",
+        "Peacock", "Paramount+", "YouTube", "YouTube Premium", "TikTok", "Instagram",
+        "influencer", "content creator", "vlogger", "blogger", "podcaster", "podcast",
+        "episode", "download", "subscriber", "follower", "like", "share", "viral",
+        "trending", "binge-watch", "marathon", "spoiler", "recap", "review", "trailer",
+        "teaser", "behind the scenes", "BTS", "exclusive", "original"
+    ]
+}
+
+# Add domain-specific RSS feeds for different categories
+CATEGORY_SPECIFIC_FEEDS = {
+    "ai": [
+        "https://www.artificialintelligence-news.com/feed/",
+        "https://www.deeplearningweekly.com/feed",
+        "https://openai.com/news/rss.xml",
+        "https://aiweekly.co/issues.rss",
+        "https://news.mit.edu/topic/mitartificial-intelligence2-rss.xml",
+        "https://ai.stanford.edu/blog/feed.xml",
+        "https://feeds.feedburner.com/blogspot/gJZg",
+        "https://blog.google/technology/ai/rss/",
+        "https://deepmind.google/blog/rss.xml",
+        "https://blog.tensorflow.org/feeds/posts/default",
+        "https://aws.amazon.com/blogs/machine-learning/feed/",
+        "https://machinelearning.apple.com/rss.xml",
+        "https://msrc.microsoft.com/blog/feed",
+        "https://learn.microsoft.com/en-us/archive/blogs/machinelearning/feed.xml",
+        "https://rss.arxiv.org/rss/cs.LG"
+    ],
+    "science": [
+        "https://www.science.org/rss/news_current.xml",
+        "https://www.nature.com/nature.rss",
+        "http://rss.sciam.com/basic-science",
+        "http://rss.sciam.com/ScientificAmerican-Global",
+        "https://www.newscientist.com/feed/home/?cmpid=RSS|NSNS-Home",
+        "https://phys.org/rss-feed/"
+    ],
+    "technology": [
+        "https://www.wired.com/feed/category/business/latest/rss",
+        "https://techcrunch.com/feed/",
+        "https://www.technologyreview.com/feed/",
+        "https://arstechnica.com/feed/",
+        "https://www.theverge.com/rss/index.xml",
+        "https://news.ycombinator.com/rss"
+    ],
+    "politics": [
+        "https://feeds.washingtonpost.com/rss/politics",
+        "https://rss.nytimes.com/services/xml/rss/nyt/Politics.xml",
+        "https://feeds.bbci.co.uk/news/politics/rss.xml",
+        "https://www.politico.com/rss/politicopicks.xml",
+        "https://www.realclearpolitics.com/index.xml"
+    ],
+    "business": [
+        "https://www.ft.com/rss/home",
+        "https://feeds.bloomberg.com/markets/news.rss",
+        "https://rss.nytimes.com/services/xml/rss/nyt/Business.xml",
+        "https://feeds.washingtonpost.com/rss/business",
+        "https://www.entrepreneur.com/latest.rss",
+        "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=10001147",
+        "https://feeds.content.dowjones.io/public/rss/WSJcomUSBusiness",
+        "https://feeds.a.dj.com/rss/RSSMarketsMain.xml"
+    ],
+    "world": [
+        "https://feeds.bbci.co.uk/news/world/rss.xml",
+        "https://rss.nytimes.com/services/xml/rss/nyt/World.xml",
+        "https://www.aljazeera.com/xml/rss/all.xml",
+        "https://feeds.washingtonpost.com/rss/world",
+        "http://rss.cnn.com/rss/cnn_world.rss"
+    ],
+    "sports": [
+        "https://www.espn.com/espn/rss/news",
+        "https://www.cbssports.com/rss/headlines/",
+        "https://www.espncricinfo.com/rss/content/story/feeds/0.xml",
+        "https://api.foxsports.com/v1/rss",
+        "https://www.sportingnews.com/us/rss",
+        "https://www.theguardian.com/sport/rss",
+    ],
+    "entertainment": [
+        "https://www.hollywoodreporter.com/feed/",
+        "https://variety.com/feed/",
+        "https://www.eonline.com/syndication/feeds/rssfeeds/topstories.xml",
+        "https://www.rollingstone.com/feed/",
+        "https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml"
+    ],
+    "fact_checking": [
+        "https://www.snopes.com/feed/",
+        "https://www.politifact.com/rss/all/",
+        "https://www.factcheck.org/feed/",
+        "https://leadstories.com/atom.xml",
+        "https://fullfact.org/feed/all/",
+        "https://www.truthorfiction.com/feed/"
+    ]
+}
+
+def detect_claim_category(claim: str) -> Tuple[str, float]:
+    """
+    Detect the most likely category of a claim and its confidence score
+    
+    This function analyzes the claim text and matches it against category-specific keywords
+    to determine the most likely category for the claim (AI, science, politics, etc.).
+    
+    Args:
+        claim (str): The claim text
+        
+    Returns:
+        tuple: (category_name, confidence_score)
+    """
+    if not claim:
+        return "general", 0.3
+    
+    # Lowercase for better matching
+    claim_lower = claim.lower()
+    
+    # Count matches for each category
+    category_scores = {}
+    
+    for category, keywords in CLAIM_CATEGORIES.items():
+        # Count how many keywords from this category appear in the claim
+        matches = sum(1 for keyword in keywords if keyword.lower() in claim_lower)
+        
+        # Calculate a simple score based on matches
+        if matches > 0:
+            # Calculate a more significant score based on number of matches
+            score = min(0.9, 0.3 + (matches * 0.1))  # Base 0.3 + 0.1 per match, max 0.9
+            category_scores[category] = score
+    
+    # Find category with highest score
+    if not category_scores:
+        return "general", 0.3
+    
+    top_category = max(category_scores.items(), key=lambda x: x[1])
+    category_name, confidence = top_category
+    
+    # If the top score is too low, return general
+    if confidence < 0.3:
+        return "general", 0.3
+    
+    return category_name, confidence
+
+def get_category_specific_rss_feeds(category: str, max_feeds: int = 5) -> List[str]:
+    """
+    Get a list of RSS feeds specific to a category
+    
+    This function returns a subset of category-specific RSS feeds to use
+    for evidence gathering.
+    
+    Args:
+        category (str): The claim category
+        max_feeds (int): Maximum number of feeds to return
+        
+    Returns:
+        list: List of RSS feed URLs
+    """
+    # Get category-specific feeds
+    category_feeds = CATEGORY_SPECIFIC_FEEDS.get(category, [])
+    
+    # Limit to max_feeds
+    return category_feeds[:min(max_feeds, len(category_feeds))]
+
+def get_fallback_category(category: str) -> Optional[str]:
+    """
+    Get a fallback category for a given category when insufficient evidence is found
+    
+    This function determines which alternative category to use when the
+    primary category doesn't yield sufficient evidence. For example,
+    AI claims fall back to technology sources.
+    
+    Args:
+        category (str): The primary category to find a fallback for
+        
+    Returns:
+        str or None: Fallback category name or None if no fallback exists
+    """
+    # Define fallback categories for specific categories
+    fallbacks = {
+        "ai": "technology",  # For AI claims, use technology as fallback
+        # Other categories fall back to default RSS feeds, handled in retrieve_combined_evidence
+    }
+    
+    return fallbacks.get(category)

modules/claim_extraction.py

@@ -0,0 +1,241 @@
+import logging
+import time
+import re
+from langdetect import detect
+import spacy
+
+from utils.performance import PerformanceTracker
+from utils.models import get_nlp_model, get_llm_model
+from modules.classification import normalize_tense
+
+logger = logging.getLogger("misinformation_detector")
+
+performance_tracker = PerformanceTracker()
+
+def extract_claims(text):
+    """
+    Extract the main factual claim from the provided text.
+    For concise claims (<30 words), preserves them exactly.
+    For longer text, uses OpenAI to extract the claim.
+    """
+    logger.info(f"Extracting claims from: {text}")
+    start_time = time.time()
+
+    # First, check if the input already appears to be a concise claim
+    if len(text.split()) < 30:
+        logger.info("Input appears to be a concise claim already, preserving as-is")
+        performance_tracker.log_processing_time(start_time)
+        performance_tracker.log_claim_processed()
+        return text
+
+    try:
+        # For longer text, use OpenAI for extraction
+        extracted_claim = extract_with_openai(text)
+        
+        # Log processing time
+        performance_tracker.log_processing_time(start_time)
+        performance_tracker.log_claim_processed()
+        
+        logger.info(f"Extracted claim: {extracted_claim}")
+        return extracted_claim
+    except Exception as e:
+        logger.error(f"Error extracting claims: {str(e)}")
+        # Fallback to original text on error
+        return text
+
+def extract_with_openai(text):
+    """
+    Use OpenAI model for claim extraction
+    """
+    try:
+        # Get LLM model
+        llm_model = get_llm_model()
+        
+        # Create a very explicit prompt to avoid hallucination
+        prompt = f"""
+        Extract the main factual claim from the following text. 
+        DO NOT add any information not present in the original text.
+        DO NOT add locations, dates, or other details.
+        ONLY extract what is explicitly stated.
+        
+        Text: {text}
+        
+        Main factual claim:
+        """
+        
+        # Call OpenAI with temperature=0 for deterministic output
+        response = llm_model.invoke(prompt, temperature=0)
+        extracted_claim = response.content.strip()
+        
+        # Further clean up any explanations or extra text
+        if ":" in extracted_claim:
+            parts = extracted_claim.split(":")
+            if len(parts) > 1:
+                extracted_claim = parts[-1].strip()
+        
+        logger.info(f"OpenAI extraction: {extracted_claim}")
+        
+        # Validate that we're not adding info not in the original
+        nlp = get_nlp_model()
+        extracted_claim = validate_extraction(text, extracted_claim, nlp)
+        
+        return extracted_claim
+    except Exception as e:
+        logger.error(f"Error in OpenAI claim extraction: {str(e)}")
+        return text  # Fallback to original
+
+def validate_extraction(original_text, extracted_claim, nlp):
+    """
+    Validate that the extracted claim doesn't add information not present in the original text
+    """
+    # If extraction fails or is empty, return original
+    if not extracted_claim or extracted_claim.strip() == "":
+        logger.warning("Empty extraction result, using original text")
+        return original_text
+    
+    # Check for added location information
+    location_terms = ["united states", "america", "u.s.", "usa", "china", "india", "europe", 
+                      "russia", "japan", "uk", "germany", "france", "australia"]
+    for term in location_terms:
+        if term in extracted_claim.lower() and term not in original_text.lower():
+            logger.warning(f"Extraction added location '{term}' not in original, using original text")
+            return original_text
+    
+    # Check for entity preservation/addition using spaCy
+    try:
+        # Get entities from extracted text
+        extracted_doc = nlp(extracted_claim)
+        extracted_entities = [ent.text.lower() for ent in extracted_doc.ents]
+        
+        # Get entities from original text
+        original_doc = nlp(original_text)
+        original_entities = [ent.text.lower() for ent in original_doc.ents]
+        
+        # Check for new entities that don't exist in original
+        for entity in extracted_entities:
+            if not any(entity in orig_entity or orig_entity in entity for orig_entity in original_entities):
+                logger.warning(f"Extraction added new entity '{entity}', using original text")
+                return original_text
+        
+        return extracted_claim
+    except Exception as e:
+        logger.error(f"Error in extraction validation: {str(e)}")
+        return original_text  # On error, safer to return original
+
+def shorten_claim_for_evidence(claim):
+    """
+    Shorten a claim to use for evidence retrieval by preserving important entities,
+    verbs, and keywords while maintaining claim context
+    
+    Args:
+        claim (str): The original claim
+        
+    Returns:
+        str: A shortened version of the claim optimized for evidence retrieval
+    """
+    try:
+        normalized_claim = normalize_tense(claim)
+        # Get NLP model
+        nlp = get_nlp_model()
+        
+        # Process claim with NLP
+        doc = nlp(claim)
+        
+        # Components to extract
+        important_components = []
+        
+        # 1. Extract all named entities as highest priority
+        entities = [ent.text for ent in doc.ents]
+        important_components.extend(entities)
+        
+        # 2. Extract key proper nouns if not already captured in entities
+        for token in doc:
+            if token.pos_ == "PROPN" and token.text not in important_components:
+                important_components.append(token.text)
+        
+        # 3. Extract main verbs (actions)
+        verbs = []
+        for token in doc:
+            if token.pos_ == "VERB" and not token.is_stop:
+                verbs.append(token.text)
+        
+        # 4. Check for important title terms like "president", "prime minister"
+        title_terms = ["president", "prime minister", "minister", "chancellor", "premier", 
+                      "governor", "mayor", "senator", "CEO", "founder", "director"]
+        
+        for term in title_terms:
+            if term in claim.lower():
+                # Find the full phrase (e.g., "Canadian Prime Minister")
+                matches = re.finditer(r'(?i)(?:\w+\s+)*\b' + re.escape(term) + r'\b(?:\s+\w+)*', claim)
+                for match in matches:
+                    phrase = match.group(0)
+                    if phrase not in important_components:
+                        important_components.append(phrase)
+        
+        # 5. Add important temporal indicators
+        temporal_terms = ["today", "yesterday", "recently", "just", "now",
+                               "current", "currently", "latest", "new", "week",
+                               "month", "year", "announces", "announced", "introduces",
+                               "introduced", "launches", "launched", "releases",
+                               "released", "rolls out", "rolled out", "presents", "presented", "unveils", "unveiled", 
+                               "starts", "started", "begins", "began", "initiates", "initiated", "anymore"
+        ]
+        
+        # Add significant temporal context
+        temporal_context = []
+        for term in temporal_terms:
+            if term in claim.lower():
+                temporal_matches = re.finditer(r'(?i)(?:\w+\s+){0,2}\b' + re.escape(term) + r'\b(?:\s+\w+){0,2}', claim)
+                for match in temporal_matches:
+                    temporal_context.append(match.group(0))
+        
+        # 6. Always include negation words as they're critical for meaning
+        negation_terms = ["not", "no longer", "former", "ex-", "isn't", "aren't", "doesn't", "don't"]
+        
+        negation_context = []
+        for term in negation_terms:
+            if term in claim.lower():
+                # Find the context around the negation (3 words before and after)
+                neg_matches = re.finditer(r'(?i)(?:\w+\s+){0,3}\b' + re.escape(term) + r'\b(?:\s+\w+){0,3}', claim)
+                for match in neg_matches:
+                    negation_context.append(match.group(0))
+        
+        # Combine all components
+        all_components = important_components + verbs + temporal_context + negation_context
+        
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_components = []
+        for component in all_components:
+            if component.lower() not in seen:
+                seen.add(component.lower())
+                unique_components.append(component)
+        
+        # If we have too few components (< 2), use the original claim
+        if len(unique_components) < 2:
+            # If the claim is already short (< 10 words), use as is
+            if len(claim.split()) < 10:
+                return claim
+            
+            # Otherwise, use the first 8 words
+            words = claim.split()
+            return " ".join(words[:min(8, len(words))])
+        
+        # Join components to create shortened claim
+        # Sort components to maintain approximate original word order
+        def get_position(comp):
+            return claim.lower().find(comp.lower())
+        
+        unique_components.sort(key=get_position)
+        shortened_claim = " ".join(unique_components)
+        
+        # If the shortened claim is still too long, limit to first 10 words
+        if len(shortened_claim.split()) > 10:
+            return " ".join(shortened_claim.split()[:10])
+            
+        return shortened_claim
+        
+    except Exception as e:
+        logger.error(f"Error in shortening claim: {str(e)}")
+        # Return original claim on error
+        return claim

modules/classification.py

@@ -0,0 +1,472 @@
+import logging
+import re
+from utils.models import get_llm_model, get_nlp_model
+from utils.performance import PerformanceTracker
+
+logger = logging.getLogger("misinformation_detector")
+
+performance_tracker = PerformanceTracker()
+
+def classify_with_llm(query, evidence):
+    """
+    Classification function that evaluates evidence against a claim
+    to determine support, contradiction, or insufficient evidence.
+    
+    This function analyzes the provided evidence to evaluate if it supports, 
+    contradicts, or is insufficient to verify the claim. It implements:
+    - Strict output formatting requirements
+    - Evidence source validation
+    - Confidence scoring based on confirmation strength
+    - Flexible regex pattern matching
+    - Detailed debug logging
+    - Fallback parsing for non-standard responses
+    
+    Args:
+        query (str): The factual claim being verified
+        evidence (list): Evidence items to evaluate against the claim
+        
+    Returns:
+        list: Classification results with labels and confidence scores
+    """
+    logger.info(f"Classifying evidence for claim: {query}")
+    
+    # Get the LLM model
+    llm_model = get_llm_model()
+    
+    # Skip if no evidence
+    if not evidence:
+        logger.warning("No evidence provided for classification")
+        return []
+
+    # Normalize evidence to a list
+    if not isinstance(evidence, list):
+        if evidence:
+            try:
+                evidence = [evidence]
+            except Exception as e:
+                logger.error(f"Could not convert evidence to list: {e}")
+                return []
+        else:
+            return []
+
+    # Extract essential claim components for improved keyword detection
+    claim_components = extract_claim_keywords(query)
+    essential_keywords = claim_components.get("keywords", [])
+    essential_entities = claim_components.get("entities", [])
+    
+    # Ensure processing is limited to top 10 evidence items to reduce token usage
+    evidence = evidence[:10]
+
+    # Validate evidence for verifiable sources
+    validated_evidence = []
+    for idx, chunk in enumerate(evidence):
+        # Basic evidence validation
+        if not isinstance(chunk, str) or not chunk.strip():
+            continue
+            
+        # Check if evidence contains source information
+        has_valid_source = False
+        if "URL:" in chunk and ("http://" in chunk or "https://" in chunk):
+            has_valid_source = True
+        elif "Source:" in chunk and len(chunk.split("Source:")[1].strip()) > 3:
+            has_valid_source = True
+            
+        # Add validation flag to evidence
+        validated_evidence.append({
+            "text": chunk,
+            "index": idx + 1,
+            "has_valid_source": has_valid_source
+        })
+    
+    # If no valid evidence remains, return early
+    if not validated_evidence:
+        logger.warning("No valid evidence items to classify")
+        return []
+
+    try:
+        # Format evidence items with validation information
+        evidence_text = ""
+        for item in validated_evidence:
+            # Truncate long evidence
+            chunk_text = item["text"]
+            if len(chunk_text) > 1000:
+                chunk_text = chunk_text[:1000] + "..."
+            
+            # Include validation status in the prompt
+            source_status = "WITH VALID SOURCE" if item["has_valid_source"] else "WARNING: NO CLEAR SOURCE"
+            evidence_text += f"EVIDENCE {item['index']}:\n{chunk_text}\n[{source_status}]\n\n"
+
+        # Create a structured prompt with explicit format instructions and validation requirements
+        prompt = f"""
+        CLAIM: {query}
+
+        EVIDENCE:
+        {evidence_text}
+
+        TASK: Evaluate if each evidence supports, contradicts, or is insufficient/irrelevant to the claim.
+        
+        INSTRUCTIONS:
+        1. For each evidence, provide your analysis in EXACTLY this format:
+        
+        EVIDENCE [number] ANALYSIS:
+        Classification: [Choose exactly one: support/contradict/insufficient]
+        Confidence: [number between 0-100]
+        Reason: [brief explanation]
+        
+        2. Support = Evidence EXPLICITLY confirms ALL parts of the claim are true
+        3. Contradict = Evidence EXPLICITLY confirms the claim is false
+        4. Insufficient = Evidence is irrelevant, ambiguous, or doesn't provide enough information
+        
+        CRITICAL VALIDATION RULES:
+        - Mark as "support" ONLY when evidence EXPLICITLY mentions ALL key entities AND actions from the claim
+        - Do not label evidence as "support" if it only discusses the same topic without confirming the specific claim
+        - Do not make inferential leaps - if the evidence doesn't explicitly state the claim, mark as "insufficient"
+        - Assign LOW confidence (0-50) when evidence doesn't explicitly mention all claim elements
+        - Assign ZERO confidence (0) to evidence without valid sources
+        - If evidence describes similar but different events, mark as "insufficient", not "support" or "contradict"
+        - If evidence describes the same topic as the claim but does not confirm or contradict the claim, mark as "insufficient", not "support" or "contradict"
+        - If evidence is in a different language or unrelated topic, mark as "insufficient" with 0 confidence
+        - Check that all entities (names, places, dates, numbers) in the claim are explicitly confirmed
+        
+        FOCUS ON THE EXACT CLAIM ONLY.
+        ESSENTIAL KEYWORDS TO LOOK FOR: {', '.join(essential_keywords)}
+        ESSENTIAL ENTITIES TO VERIFY: {', '.join(essential_entities)}
+
+        IMPORTANT NOTE ABOUT VERB TENSES: When analyzing this claim, treat present tense verbs (like "unveils") 
+        and perfect form verbs (like "has unveiled") as equivalent to their simple past tense forms 
+        (like "unveiled"). The tense variation should not affect your classification decision.
+        """
+
+        # Get response with temperature=0 for consistency
+        result = llm_model.invoke(prompt, temperature=0)
+        result_text = result.content.strip()
+        
+        # Log the raw LLM response for debugging
+        logger.debug(f"Raw LLM classification response:\n{result_text}")
+        
+        # Define a more flexible regex pattern matching the requested format
+        # This pattern accommodates variations in whitespace and formatting
+        analysis_pattern = r'EVIDENCE\s+(\d+)\s+ANALYSIS:[\s\n]*Classification:[\s\n]*(support|contradict|insufficient)[\s\n]*Confidence:[\s\n]*(\d+)[\s\n]*Reason:[\s\n]*(.*?)(?=[\s\n]*EVIDENCE\s+\d+\s+ANALYSIS:|[\s\n]*$)'
+        
+        # Parse each evidence analysis
+        classification_results = []
+        
+        # Try matching with our pattern
+        matches = list(re.finditer(analysis_pattern, result_text, re.IGNORECASE | re.DOTALL))
+        
+        # Log match information for debugging
+        logger.debug(f"Found {len(matches)} structured evidence analyses in response")
+        
+        # Process matches
+        for match in matches:
+            try:
+                evidence_idx = int(match.group(1)) - 1
+                classification = match.group(2).lower()
+                confidence = int(match.group(3)) / 100.0  # Convert to 0-1 scale
+                reason = match.group(4).strip()
+                
+                # Check if this evidence item exists in our original list
+                if 0 <= evidence_idx < len(evidence):
+                    # Get the original evidence text
+                    evidence_text = evidence[evidence_idx]
+                    
+                    # Check for valid source
+                    source_valid = False
+                    if "URL:" in evidence_text and ("http://" in evidence_text or "https://" in evidence_text):
+                        source_valid = True
+                    elif "Source:" in evidence_text:
+                        source_valid = True
+                        
+                    # Reduce confidence for evidence without valid sources
+                    if not source_valid and confidence > 0.3:
+                        confidence = 0.3
+                        reason += " (Confidence reduced due to lack of verifiable source)"
+                        
+                    # Create result entry
+                    classification_results.append({
+                        "label": classification,
+                        "confidence": confidence,
+                        "evidence": evidence_text,
+                        "reason": reason
+                    })
+            except (ValueError, IndexError) as e:
+                logger.error(f"Error parsing evidence analysis: {e}")
+        
+        # If no structured matches were found, try using a simpler approach
+        if not classification_results:
+            logger.warning("No structured evidence analysis found, using fallback method")
+            
+            # Log detailed information about the failure
+            logger.warning(f"Expected format not found in response. Response excerpt: {result_text[:200]}...")
+            
+            # Simple fallback parsing based on keywords
+            for idx, ev in enumerate(evidence):
+                # Check for keywords in the LLM response
+                ev_mention = f"EVIDENCE {idx+1}"
+                if ev_mention in result_text:
+                    # Find the section for this evidence
+                    parts = result_text.split(ev_mention)
+                    if len(parts) > 1:
+                        analysis_text = parts[1].split("EVIDENCE")[0] if "EVIDENCE" in parts[1] else parts[1]
+                        
+                        # Determine classification
+                        label = "insufficient"  # Default
+                        confidence = 0.0  # Default - zero confidence for fallback parsing
+
+                        # Check for support indicators
+                        if "support" in analysis_text.lower() or "confirms" in analysis_text.lower():
+                            label = "support"
+                            confidence = 0.4  # Lower confidence for fallback support
+                            
+                        # Check for contradict indicators    
+                        elif "contradict" in analysis_text.lower() or "false" in analysis_text.lower():
+                            label = "contradict"
+                            confidence = 0.4  # Lower confidence for fallback contradict
+                        
+                        # Check for valid source to adjust confidence
+                        source_valid = False
+                        if "URL:" in ev and ("http://" in ev or "https://" in ev):
+                            source_valid = True
+                        elif "Source:" in ev:
+                            source_valid = True
+                            
+                        if not source_valid:
+                            confidence = min(confidence, 0.3)
+                            
+                        # Create basic result
+                        classification_results.append({
+                            "label": label,
+                            "confidence": confidence,
+                            "evidence": ev,
+                            "reason": f"Determined via fallback parsing. {'Valid source found.' if source_valid else 'Warning: No clear source identified.'}"
+                        })
+                        
+                        logger.debug(f"Fallback parsing for evidence {idx+1}: {label} with confidence {confidence}")
+        
+        logger.info(f"Classified {len(classification_results)} evidence items")
+        return classification_results
+
+    except Exception as e:
+        logger.error(f"Error in evidence classification: {str(e)}")
+        # Provide a basic fallback
+        fallback_results = []
+        for ev in evidence:
+            fallback_results.append({
+                "label": "insufficient",
+                "confidence": 0.5,
+                "evidence": ev,
+                "reason": "Classification failed with error, using fallback"
+            })
+        return fallback_results
+
+def normalize_tense(claim):
+    """
+    Normalize verb tenses in claims to ensure consistent classification.
+    
+    This function standardizes verb forms by converting present simple tense 
+    verbs (e.g., "unveils") and perfect forms (e.g., "has unveiled") to their 
+    past tense equivalents (e.g., "unveiled"). This ensures that semantically 
+    equivalent claims are processed consistently regardless of verb tense 
+    variations.
+    
+    Args:
+        claim (str): The original claim text to normalize
+        
+    Returns:
+        str: The normalized claim with consistent tense handling
+        
+    Note:
+        This function specifically targets present simple and perfect forms,
+        preserving the semantic differences of continuous forms (is unveiling)
+        and future tense (will unveil).
+    """
+    # Define patterns to normalize common verb forms.
+    # Each tuple contains (regex_pattern, replacement_text)
+    tense_patterns = [
+        # Present simple to past tense conversions
+        (r'\bunveils\b', r'unveiled'),
+        (r'\blaunches\b', r'launched'),
+        (r'\breleases\b', r'released'),
+        (r'\bannounces\b', r'announced'),
+        (r'\binvites\b', r'invited'),
+        (r'\bretaliates\b', r'retaliated'),
+        (r'\bends\b', r'ended'),
+        (r'\bbegins\b', r'began'),
+        (r'\bstarts\b', r'started'),
+        (r'\bcompletes\b', r'completed'),
+        (r'\bfinishes\b', r'finished'),
+        (r'\bintroduces\b', r'introduced'),
+        (r'\bcreates\b', r'created'),
+        (r'\bdevelops\b', r'developed'),
+        (r'\bpublishes\b', r'published'),
+        (r'\bacquires\b', r'acquired'),
+        (r'\bbuys\b', r'bought'),
+        (r'\bsells\b', r'sold'),
+        
+        # Perfect forms (has/have/had + past participle) to simple past
+        (r'\b(has|have|had)\s+unveiled\b', r'unveiled'),
+        (r'\b(has|have|had)\s+launched\b', r'launched'),
+        (r'\b(has|have|had)\s+released\b', r'released'),
+        (r'\b(has|have|had)\s+announced\b', r'announced'),
+        (r'\b(has|have|had)\s+invited\b', r'invited'),
+        (r'\b(has|have|had)\s+retaliated\b', r'retaliated'),
+        (r'\b(has|have|had)\s+ended\b', r'ended'),
+        (r'\b(has|have|had)\s+begun\b', r'began'),
+        (r'\b(has|have|had)\s+started\b', r'started'),
+        (r'\b(has|have|had)\s+introduced\b', r'introduced'),
+        (r'\b(has|have|had)\s+created\b', r'created'),
+        (r'\b(has|have|had)\s+developed\b', r'developed'),
+        (r'\b(has|have|had)\s+published\b', r'published'),
+        (r'\b(has|have|had)\s+acquired\b', r'acquired'),
+        (r'\b(has|have|had)\s+bought\b', r'bought'),
+        (r'\b(has|have|had)\s+sold\b', r'sold')
+    ]
+    
+    # Apply normalization patterns
+    normalized = claim
+    for pattern, replacement in tense_patterns:
+        normalized = re.sub(pattern, replacement, normalized, flags=re.IGNORECASE)
+    
+    # Log if normalization occurred for debugging purposes
+    if normalized != claim:
+        logger.info(f"Normalized claim from: '{claim}' to: '{normalized}'")
+        
+    return normalized 
+
+def aggregate_evidence(classification_results):
+    """
+    Aggregate evidence classifications to determine overall verdict
+    using a weighted scoring system of evidence count and quality.
+    
+    Args:
+        classification_results (list): List of evidence classification results
+        
+    Returns:
+        tuple: (verdict, confidence) - The final verdict and confidence score
+    """
+    logger.info(f"Aggregating evidence from {len(classification_results) if classification_results else 0} results")
+    
+    if not classification_results:
+        logger.warning("No classification results to aggregate")
+        return "Uncertain", 0.0  # Default with zero confidence
+    
+    # Only consider support and contradict evidence items
+    support_items = [item for item in classification_results if item.get("label") == "support"]
+    contradict_items = [item for item in classification_results if item.get("label") == "contradict"]
+    
+    # Count number of support and contradict items
+    support_count = len(support_items)
+    contradict_count = len(contradict_items)
+    
+    # Calculate confidence scores for support and contradict items
+    support_confidence_sum = sum(item.get("confidence", 0) for item in support_items)
+    contradict_confidence_sum = sum(item.get("confidence", 0) for item in contradict_items)
+    
+    # Apply weights: 55% for count, 45% for quality (confidence)
+    # Normalize counts to avoid division by zero
+    max_count = max(1, max(support_count, contradict_count))
+    
+    # Calculate weighted scores
+    count_support_score = (support_count / max_count) * 0.55
+    count_contradict_score = (contradict_count / max_count) * 0.55
+    
+    # Normalize confidence scores to avoid division by zero
+    max_confidence_sum = max(1, max(support_confidence_sum, contradict_confidence_sum))
+    
+    quality_support_score = (support_confidence_sum / max_confidence_sum) * 0.45
+    quality_contradict_score = (contradict_confidence_sum / max_confidence_sum) * 0.45
+    
+    # Total scores
+    total_support = count_support_score + quality_support_score
+    total_contradict = count_contradict_score + quality_contradict_score
+    
+    # Check if all evidence is irrelevant/insufficient
+    if support_count == 0 and contradict_count == 0:
+        logger.info("All evidence items are irrelevant/insufficient")
+        return "Uncertain", 0.0
+    
+    # Determine verdict based on higher total score
+    if total_support > total_contradict:
+        verdict = "True (Based on Evidence)"
+        min_score = total_contradict
+        max_score = total_support
+    else:
+        verdict = "False (Based on Evidence)"
+        min_score = total_support
+        max_score = total_contradict
+    
+    # Calculate final confidence using the formula:
+    # (1 - min_score/max_score) * 100%
+    if max_score > 0:
+        final_confidence = 1.0 - (min_score / max_score)
+    else:
+        final_confidence = 0.0
+    
+    # Handle cases where confidence is very low
+    if final_confidence == 0.0:
+        return "Uncertain", 0.0
+    elif final_confidence < 0.1:  # Less than 10%
+        # Keep the verdict but with very low confidence
+        logger.info(f"Very low confidence verdict: {verdict} with {final_confidence:.2f} confidence")
+    
+    logger.info(f"Final verdict: {verdict}, confidence: {final_confidence:.2f}")
+    
+    return verdict, final_confidence
+
+def extract_claim_keywords(claim):
+    """
+    Extract important keywords from claim using NLP processing
+    
+    Args:
+        claim (str): The claim text
+        
+    Returns:
+        dict: Dictionary containing keywords and other claim components
+    """
+    try:
+        # Get NLP model
+        nlp = get_nlp_model()
+        
+        # Process claim with NLP
+        doc = nlp(claim)
+        
+        # Extract entities
+        entities = [ent.text for ent in doc.ents]
+        
+        # Extract important keywords (non-stopword nouns, adjectives, and verbs longer than 3 chars)
+        keywords = []
+        for token in doc:
+            # Keep all important parts of speech, longer than 3 characters
+            if token.pos_ in ["NOUN", "PROPN", "ADJ", "VERB"] and not token.is_stop and len(token.text) > 3:
+                keywords.append(token.text.lower())
+            # Also include some important modifiers and quantifiers
+            elif token.pos_ in ["NUM", "ADV"] and not token.is_stop and len(token.text) > 1:
+                keywords.append(token.text.lower())
+        
+        # Extract verbs separately
+        verbs = [token.lemma_.lower() for token in doc if token.pos_ == "VERB" and not token.is_stop]
+        
+        # Also extract multi-word phrases that might be important
+        noun_phrases = []
+        for chunk in doc.noun_chunks:
+            if len(chunk.text) > 3 and not all(token.is_stop for token in chunk):
+                noun_phrases.append(chunk.text.lower())
+        
+        # Add phrases to keywords if not already included
+        for phrase in noun_phrases:
+            if phrase not in keywords and phrase.lower() not in [k.lower() for k in keywords]:
+                keywords.append(phrase.lower())
+        
+        # Return all components
+        return {
+            "entities": entities,
+            "keywords": keywords,
+            "verbs": verbs,
+            "noun_phrases": noun_phrases
+        }
+        
+    except Exception as e:
+        logger.error(f"Error extracting claim keywords: {e}")
+        # Return basic fallback using simple word extraction
+        words = [word.lower() for word in claim.split() if len(word) > 3]
+        return {"keywords": words, "entities": [], "verbs": [], "noun_phrases": []}

modules/evidence_retrieval.py

@@ -0,0 +1,816 @@
+"""
+Evidence retrieval module for the Fake News Detector application.
+
+This module provides functions for retrieving evidence from various sources,
+analyzing relevance using entity extraction and verb matching, and
+combining evidence to support fact-checking operations.
+"""
+
+import logging
+import time
+import re
+import random
+import requests
+import json
+import ssl
+from urllib.parse import urlencode
+from bs4 import BeautifulSoup
+from SPARQLWrapper import SPARQLWrapper, JSON
+from datetime import datetime, timedelta
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from utils.api_utils import api_error_handler, safe_json_parse
+from utils.models import get_nlp_model
+from modules.claim_extraction import shorten_claim_for_evidence
+from modules.rss_feed import retrieve_evidence_from_rss
+from config import NEWS_API_KEY, FACTCHECK_API_KEY
+# Import the performance tracker
+from utils.performance import PerformanceTracker
+performance_tracker = PerformanceTracker()
+
+logger = logging.getLogger("misinformation_detector")
+
+def extract_claim_components(claim):
+    """
+    Extract key components from a claim using NER and dependency parsing.
+    
+    Args:
+        claim (str): The claim text
+        
+    Returns:
+        dict: Dictionary containing entities, verbs, and important keywords
+    """
+    if not claim:
+        return {"entities": [], "verbs": [], "keywords": []}
+    
+    try:
+        # Get NLP model
+        nlp = get_nlp_model()
+        doc = nlp(claim)
+        
+        # Extract named entities - keep original case for better matching
+        entities = []
+        for ent in doc.ents:
+            entities.append(ent.text)
+        
+        # Also extract any capitalized words as potential entities not caught by NER
+        words = claim.split()
+        for word in words:
+            clean_word = word.strip('.,;:!?()[]{}""\'')
+            # Check if word starts with capital letter and isn't already in entities
+            if clean_word and clean_word[0].isupper() and clean_word not in entities:
+                entities.append(clean_word)
+        
+        # Extract main verbs
+        verbs = []
+        for token in doc:
+            if token.pos_ == "VERB" and not token.is_stop:
+                # Get the lemma to handle different verb forms
+                verbs.append(token.lemma_.lower())
+        
+        # Extract important keywords (non-stopword nouns, adjectives)
+        keywords = []
+        for token in doc:
+            if token.pos_ in ["NOUN", "ADJ"] and not token.is_stop and len(token.text) > 2:
+                keywords.append(token.text.lower())
+        
+        # Extract temporal indicators
+        temporal_words = []
+        temporal_indicators = ["today", "yesterday", "recently", "just", "now",
+                               "current", "currently", "latest", "new", "week",
+                               "month", "year", "announces", "announced", "introduces",
+                               "introduced", "launches", "launched", "releases",
+                               "released", "rolls out", "rolled out", "presents", "presented", "unveils", "unveiled", 
+                               "starts", "started", "begins", "began", "initiates", "initiated", "anymore"
+        ]
+
+        for token in doc:
+            if token.text.lower() in temporal_indicators:
+                temporal_words.append(token.text.lower())
+        
+        return {
+            "entities": entities,
+            "verbs": verbs,
+            "keywords": keywords,
+            "temporal_words": temporal_words
+        }
+    except Exception as e:
+        logger.error(f"Error extracting claim components: {e}")
+        return {"entities": [], "verbs": [], "keywords": [], "temporal_words": []}
+
+def analyze_evidence_relevance(evidence_items, claim_components):
+    """
+    Analyze evidence relevance based on entity match, verb match and keyword match.
+    
+    Args:
+        evidence_items (list): List of evidence text strings
+        claim_components (dict): Components extracted from the claim
+        
+    Returns:
+        list: List of (evidence, score) tuples sorted by relevance score
+    """
+    if not evidence_items or not claim_components:
+        return []
+    
+    scored_evidence = []
+    
+    # Extract components for easier access
+    claim_entities = claim_components.get("entities", [])
+    claim_verbs = claim_components.get("verbs", [])
+    claim_keywords = claim_components.get("keywords", [])
+    
+    for evidence in evidence_items:
+        if not isinstance(evidence, str):
+            continue
+        
+        evidence_lower = evidence.lower()
+        
+        # 1. Count entity matches - try both case-sensitive and case-insensitive matching
+        entity_matches = 0
+        for entity in claim_entities:
+            # Try exact match first (preserves case)
+            if entity in evidence:
+                entity_matches += 1
+            # Then try lowercase match
+            elif entity.lower() in evidence_lower:
+                entity_matches += 1
+        
+        # 2. Count verb matches (always lowercase)
+        verb_matches = sum(1 for verb in claim_verbs if verb in evidence_lower)
+        
+        # 3. Calculate entity and verb weighted score
+        entity_verb_score = (entity_matches * 3.0) + (verb_matches * 2.0)
+        
+        # 4. Count keyword matches (always lowercase)
+        keyword_matches = sum(1 for keyword in claim_keywords if keyword in evidence_lower)
+        
+        # 5. Determine final score based on entity and verb matches
+        if entity_verb_score > 0:
+            final_score = entity_verb_score
+        else:
+            final_score = keyword_matches * 1.0  # Use keyword matches if no entity/verb matches
+        
+        scored_evidence.append((evidence, final_score))
+    
+    # Sort by score (descending)
+    scored_evidence.sort(key=lambda x: x[1], reverse=True)
+    
+    return scored_evidence
+
+def get_recent_date_range(claim=None):
+    """
+    Return date range for news filtering based on temporal indicators in the claim.
+    
+    Args:
+        claim (str, optional): The claim text to analyze for temporal indicators
+        
+    Returns:
+        tuple: (from_date, to_date) as formatted strings 'YYYY-MM-DD'
+    """
+    today = datetime.now()
+    
+    # Default to 3 days for no claim or claims without temporal indicators
+    default_days = 3
+    extended_days = 15  # For 'recently', 'this week', etc.
+    
+    if claim:
+        # Specific day indicators get 3 days
+        specific_day_terms = ["today", "yesterday", "day before yesterday"]
+        
+        # Extended time terms get 15 days
+        extended_time_terms = [
+            "recently", "currently", "freshly", "this week", "few days", 
+            "couple of days", "last week", "past week", "several days",
+            "anymore"
+        ]
+        
+        claim_lower = claim.lower()
+        
+        # Check for extended time terms first, then specific day terms
+        if any(term in claim_lower for term in extended_time_terms):
+            from_date = (today - timedelta(days=extended_days)).strftime('%Y-%m-%d')
+            to_date = today.strftime('%Y-%m-%d')
+            logger.info(f"Using extended time range of {extended_days} days based on temporal indicators")
+            return from_date, to_date
+        elif any(term in claim_lower for term in specific_day_terms):
+            from_date = (today - timedelta(days=default_days)).strftime('%Y-%m-%d')
+            to_date = today.strftime('%Y-%m-%d')
+            logger.info(f"Using specific day range of {default_days} days based on temporal indicators")
+            return from_date, to_date
+    
+    # Default case - use standard 3-day window
+    from_date = (today - timedelta(days=default_days)).strftime('%Y-%m-%d')
+    to_date = today.strftime('%Y-%m-%d')
+    return from_date, to_date
+
+@api_error_handler("wikipedia")
+def retrieve_evidence_from_wikipedia(claim):
+    """Retrieve evidence from Wikipedia for a given claim"""
+    logger.info(f"Retrieving evidence from Wikipedia for: {claim}")
+
+    # Ensure shortened_claim is a string
+    try:
+        shortened_claim = shorten_claim_for_evidence(claim)
+    except Exception as e:
+        logger.error(f"Error in claim shortening: {e}")
+        shortened_claim = claim  # Fallback to original claim
+
+    # Ensure query_parts is a list of strings
+    query_parts = str(shortened_claim).split()
+    evidence = []
+    source_count = {"wikipedia": 0}
+
+    for i in range(len(query_parts), 0, -1):  # Start with full query, shorten iteratively
+        try:
+            # Safely join and encode query
+            current_query = "+".join(query_parts[:i])
+            search_url = f"https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={current_query}&format=json"
+            logger.info(f"Wikipedia search URL: {search_url}")
+
+            headers = {
+                "User-Agent": "MisinformationDetectionResearchBot/1.0 (Research Project)"
+            }
+
+            # Make the search request with reduced timeout
+            response = requests.get(search_url, headers=headers, timeout=7)
+            response.raise_for_status()
+
+            # Safely parse JSON
+            search_data = safe_json_parse(response, "wikipedia")
+
+            # Safely extract search results
+            search_results = search_data.get("query", {}).get("search", [])
+
+            # Ensure search_results is a list
+            if not isinstance(search_results, list):
+                logger.warning(f"Unexpected search results type: {type(search_results)}")
+                search_results = []
+
+            # Use ThreadPoolExecutor to fetch page content in parallel
+            with ThreadPoolExecutor(max_workers=3) as executor:
+                # Submit up to 3 page requests in parallel
+                futures = []
+                for idx, result in enumerate(search_results[:3]):
+                    # Ensure result is a dictionary
+                    if not isinstance(result, dict):
+                        logger.warning(f"Skipping non-dictionary result: {type(result)}")
+                        continue
+
+                    # Safely extract title
+                    page_title = result.get("title", "")
+                    if not page_title:
+                        continue
+
+                    page_url = f"https://en.wikipedia.org/wiki/{page_title.replace(' ', '_')}"
+                    
+                    # Submit the page request task to executor
+                    futures.append(executor.submit(
+                        fetch_wikipedia_page_content, 
+                        page_url, 
+                        page_title, 
+                        headers
+                    ))
+                
+                # Process completed futures as they finish
+                for future in as_completed(futures):
+                    try:
+                        page_result = future.result()
+                        if page_result:
+                            evidence.append(page_result)
+                            source_count["wikipedia"] += 1
+                    except Exception as e:
+                        logger.error(f"Error processing Wikipedia page: {e}")
+
+            # Stop if we found any evidence
+            if evidence:
+                break
+
+        except Exception as e:
+            logger.error(f"Error retrieving from Wikipedia: {str(e)}")
+            continue
+
+    # Ensure success is a boolean
+    success = bool(evidence)
+
+    # Safely log evidence retrieval
+    try:
+        performance_tracker.log_evidence_retrieval(success, source_count)
+    except Exception as e:
+        logger.error(f"Error logging evidence retrieval: {e}")
+
+    if not evidence:
+        logger.warning("No evidence found from Wikipedia.")
+
+    return evidence
+
+def fetch_wikipedia_page_content(page_url, page_title, headers):
+    """Helper function to fetch and parse Wikipedia page content"""
+    try:
+        # Get page content with reduced timeout
+        page_response = requests.get(page_url, headers=headers, timeout=5)
+        page_response.raise_for_status()
+
+        # Extract relevant sections using BeautifulSoup
+        soup = BeautifulSoup(page_response.text, 'html.parser')
+        paragraphs = soup.find_all('p', limit=3)  # Limit to first 3 paragraphs
+        content = " ".join([para.get_text(strip=True) for para in paragraphs])
+        
+        # Truncate content to reduce token usage earlier in the pipeline
+        if len(content) > 1000:
+            content = content[:1000] + "..."
+
+        if content.strip():  # Ensure content is not empty
+            return f"Title: {page_title}, URL: {page_url}, Content: {content}"
+        return None
+    except Exception as e:
+        logger.error(f"Error fetching Wikipedia page {page_url}: {e}")
+        return None
+
+@api_error_handler("wikidata")
+def retrieve_evidence_from_wikidata(claim):
+    """Retrieve evidence from Wikidata for a given claim"""
+    logger.info(f"Retrieving evidence from Wikidata for: {claim}")
+
+    # Prepare entities for SPARQL query
+    shortened_claim = shorten_claim_for_evidence(claim)
+    query_terms = shortened_claim.split()
+
+    # Initialize SPARQLWrapper for Wikidata
+    sparql = SPARQLWrapper("https://query.wikidata.org/sparql")
+
+    # Use a more conservative user agent to avoid blocks
+    sparql.addCustomHttpHeader("User-Agent", "MisinformationDetectionResearchBot/1.0")
+    
+    # Fix SSL issues by disabling SSL verification for this specific request
+    try:
+        # Create a context where we don't verify SSL certs
+        import ssl
+        import urllib.request
+        
+        # Create a context that doesn't verify certificates
+        ssl_context = ssl._create_unverified_context()
+        
+        # Monkey patch the opener for SPARQLWrapper
+        opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=ssl_context))
+        urllib.request.install_opener(opener)
+    except Exception as e:
+        logger.error(f"Error setting up SSL context: {str(e)}")
+
+    # Construct basic SPARQL query for relevant entities
+    query = """
+    SELECT ?item ?itemLabel ?description ?article WHERE {
+      SERVICE wikibase:mwapi {
+        bd:serviceParam wikibase:api "EntitySearch" .
+        bd:serviceParam wikibase:endpoint "www.wikidata.org" .
+        bd:serviceParam mwapi:search "%s" .
+        bd:serviceParam mwapi:language "en" .
+        ?item wikibase:apiOutputItem mwapi:item .
+      }
+      ?item schema:description ?description .
+      FILTER(LANG(?description) = "en")
+      OPTIONAL {
+        ?article schema:about ?item .
+        ?article schema:isPartOf <https://en.wikipedia.org/> .
+      }
+      SERVICE wikibase:label { bd:serviceParam wikibase:language "en" . }
+    }
+    LIMIT 5
+    """ % " ".join(query_terms)
+
+    sparql.setQuery(query)
+    sparql.setReturnFormat(JSON)
+
+    try:
+        results = sparql.query().convert()
+
+        wikidata_evidence = []
+
+        for result in results["results"]["bindings"]:
+            entity_label = result.get("itemLabel", {}).get("value", "Unknown")
+            description = result.get("description", {}).get("value", "No description")
+            article_url = result.get("article", {}).get("value", "")
+            
+            # Truncate description to reduce token usage
+            if len(description) > 1000:
+                description = description[:1000] + "..."
+
+            evidence_text = f"Entity: {entity_label}, Description: {description}"
+            if article_url:
+                evidence_text += f", URL: {article_url}"
+
+            wikidata_evidence.append(evidence_text)
+
+        logger.info(f"Retrieved {len(wikidata_evidence)} Wikidata entities")
+        return wikidata_evidence
+
+    except Exception as e:
+        logger.error(f"Error retrieving from Wikidata: {str(e)}")
+        return []
+
+@api_error_handler("openalex")
+def retrieve_evidence_from_openalex(claim):
+    """Retrieve evidence from OpenAlex for a given claim (replacement for Semantic Scholar)"""
+    logger.info(f"Retrieving evidence from OpenAlex for: {claim}")
+
+    try:
+        shortened_claim = shorten_claim_for_evidence(claim)
+        query = shortened_claim.replace(" ", "+")
+        
+        # OpenAlex API endpoint
+        api_url = f"https://api.openalex.org/works?search={query}&filter=is_paratext:false&per_page=3"
+        
+        headers = {
+            "Accept": "application/json",
+            "User-Agent": "MisinformationDetectionResearchBot/1.0 ([email protected])",
+        }
+        
+        scholarly_evidence = []
+        
+        try:
+            # Request with reduced timeout
+            response = requests.get(api_url, headers=headers, timeout=8)
+            
+            # Check response status
+            if response.status_code == 200:
+                # Successfully retrieved data
+                data = safe_json_parse(response, "openalex")
+                papers = data.get("results", [])
+
+                for paper in papers:
+                    title = paper.get("title", "Unknown Title")
+                    abstract = paper.get("abstract_inverted_index", None)
+                    
+                    # OpenAlex stores abstracts in an inverted index format, so we need to reconstruct it
+                    abstract_text = "No abstract available"
+                    if abstract:
+                        try:
+                            # Simple approach to reconstruct from inverted index
+                            # For a production app, implement a proper reconstruction algorithm
+                            words = list(abstract.keys())
+                            abstract_text = " ".join(words[:30]) + "..."
+                        except Exception as e:
+                            logger.error(f"Error reconstructing abstract: {e}")
+                    
+                    url = paper.get("doi", "")
+                    if url and not url.startswith("http"):
+                        url = f"https://doi.org/{url}"
+                    
+                    year = ""
+                    publication_date = paper.get("publication_date", "")
+                    if publication_date:
+                        year = publication_date.split("-")[0]
+                    
+                    # Truncate abstract to reasonable length
+                    if len(abstract_text) > 1000:
+                        abstract_text = abstract_text[:1000] + "..."
+
+                    evidence_text = f"Title: {title}, Year: {year}, Abstract: {abstract_text}, URL: {url}"
+                    scholarly_evidence.append(evidence_text)
+
+            else:
+                logger.error(f"OpenAlex API error: {response.status_code}")
+
+        except requests.exceptions.Timeout:
+            logger.warning("OpenAlex request timed out")
+        except requests.exceptions.ConnectionError:
+            logger.warning("OpenAlex connection error")
+        except Exception as e:
+            logger.error(f"Unexpected error in OpenAlex request: {str(e)}")
+
+        logger.info(f"Retrieved {len(scholarly_evidence)} scholarly papers from OpenAlex")
+        return scholarly_evidence
+
+    except Exception as e:
+        logger.error(f"Fatal error in OpenAlex retrieval: {str(e)}")
+        return []
+
+@api_error_handler("factcheck")
+def retrieve_evidence_from_factcheck(claim):
+    """Retrieve evidence from Google's Fact Check Tools API for a given claim"""
+    logger.info(f"Retrieving evidence from Google's Fact Check Tools API for: {claim}")
+    factcheck_api_key = FACTCHECK_API_KEY
+
+    # Safely shorten claim
+    try:
+        shortened_claim = shorten_claim_for_evidence(claim)
+    except Exception as e:
+        logger.error(f"Error shortening claim: {e}")
+        shortened_claim = claim
+
+    query_parts = str(shortened_claim).split()
+    factcheck_results = []
+    source_count = {"factcheck": 0}
+
+    for i in range(len(query_parts), 0, -1):  # Iteratively try shorter queries
+        try:
+            current_query = " ".join(query_parts[:i])
+            encoded_query = urlencode({"query": current_query})
+            factcheck_url = f"https://factchecktools.googleapis.com/v1alpha1/claims:search?{encoded_query}&key={factcheck_api_key}"
+            logger.info(f"Factcheck URL: {factcheck_url}")
+
+            # Make request with reduced timeout
+            response = requests.get(factcheck_url, timeout=7)
+            response.raise_for_status()
+            data = safe_json_parse(response, "factcheck")
+
+            # Safely extract claims
+            claims = data.get("claims", [])
+            if not isinstance(claims, list):
+                logger.warning(f"Unexpected claims type: {type(claims)}")
+                claims = []
+
+            if claims:  # If results found
+                logger.info(f"Results found for query '{current_query}'.")
+                for item in claims:
+                    try:
+                        # Ensure item is a dictionary
+                        if not isinstance(item, dict):
+                            logger.warning(f"Skipping non-dictionary item: {type(item)}")
+                            continue
+
+                        claim_text = str(item.get("text", ""))
+                        # Truncate claim text
+                        if len(claim_text) > 1000:
+                            claim_text = claim_text[:1000] + "..."
+                            
+                        reviews = item.get("claimReview", [])
+
+                        # Ensure reviews is a list
+                        if not isinstance(reviews, list):
+                            logger.warning(f"Unexpected reviews type: {type(reviews)}")
+                            reviews = []
+
+                        for review in reviews:
+                            # Ensure review is a dictionary
+                            if not isinstance(review, dict):
+                                logger.warning(f"Skipping non-dictionary review: {type(review)}")
+                                continue
+
+                            publisher = str(review.get("publisher", {}).get("name", "Unknown Source"))
+                            rating = str(review.get("textualRating", "Unknown"))
+                            review_url = str(review.get("url", ""))
+
+                            if claim_text:
+                                factcheck_results.append(
+                                    f"Claim: {claim_text}, Rating: {rating}, " +
+                                    f"Source: {publisher}, URL: {review_url}"
+                                )
+                                source_count["factcheck"] += 1
+
+                    except Exception as e:
+                        logger.error(f"Error processing FactCheck result: {e}")
+
+                break  # Break once we have results
+            else:
+                logger.info(f"No results for query '{current_query}', trying shorter version.")
+
+        except Exception as e:
+            logger.error(f"Error in FactCheck retrieval: {e}")
+
+    # Safely log evidence retrieval
+    try:
+        success = bool(factcheck_results)
+        performance_tracker.log_evidence_retrieval(success, source_count)
+    except Exception as e:
+        logger.error(f"Error logging evidence retrieval: {e}")
+
+    if not factcheck_results:
+        logger.warning("No factcheck evidence found after trying all query variants.")
+
+    return factcheck_results
+
+@api_error_handler("newsapi")
+def retrieve_news_articles(claim, requires_recent=False):
+    """Retrieve evidence from News API for a given claim with improved single request approach"""
+    logger.info(f"Retrieving evidence from News API for: {claim}")
+
+    # Get API key
+    news_api_key = NEWS_API_KEY
+    if not news_api_key:
+        logger.error("No News API key available")
+        return []
+
+    news_results = []
+    source_count = {"news": 0}
+
+    # Get date range for recent news
+    from_date, to_date = get_recent_date_range()
+    logger.info(f"Filtering for news from {from_date} to {to_date}")
+
+    try:
+        # Extract a simplified claim for better matching
+        shortened_claim = shorten_claim_for_evidence(claim)
+        
+        # Use a single endpoint with proper parameters
+        encoded_query = urlencode({"q": shortened_claim})
+        
+        # Use the 'everything' endpoint as it's more comprehensive
+        news_api_url = f"https://newsapi.org/v2/everything?{encoded_query}&apiKey={news_api_key}&language=en&pageSize=5&sortBy=publishedAt"
+        
+        # Only apply date filtering if the claim requires recency
+        if requires_recent:
+            news_api_url += f"&from={from_date}&to={to_date}"
+        
+        log_url = news_api_url.replace(news_api_key, "API_KEY_REDACTED")
+        logger.info(f"Requesting: {log_url}")
+
+        # Make a single request with proper headers and reduced timeout
+        headers = {
+            "User-Agent": "MisinformationDetectionResearchBot/1.0",
+            "X-Api-Key": news_api_key,
+            "Accept": "application/json"
+        }
+
+        response = requests.get(
+            news_api_url,
+            headers=headers,
+            timeout=8
+        )
+
+        logger.info(f"Response status: {response.status_code}")
+
+        if response.status_code == 200:
+            data = safe_json_parse(response, "newsapi")
+            
+            if data.get("status") == "ok":
+                articles = data.get("articles", [])
+                logger.info(f"Found {len(articles)} articles")
+                
+                for article in articles:
+                    try:
+                        # Robust article parsing
+                        title = str(article.get("title", ""))
+                        description = str(article.get("description", ""))
+                        content = str(article.get("content", ""))
+                        source_name = str(article.get("source", {}).get("name", "Unknown"))
+                        url = str(article.get("url", ""))
+                        published_at = str(article.get("publishedAt", ""))
+                        
+                        # Parse date to prioritize recent content
+                        article_date = None
+                        try:
+                            if published_at:
+                                article_date = datetime.strptime(published_at.split('T')[0], '%Y-%m-%d')
+                        except Exception as date_error:
+                            logger.warning(f"Could not parse date: {published_at}")
+                        
+                        # Calculate recency score (higher = more recent)
+                        recency_score = 1.0  # Default
+                        if article_date:
+                            days_old = (datetime.now() - article_date).days
+                            if days_old == 0:  # Today
+                                recency_score = 3.0
+                            elif days_old == 1:  # Yesterday
+                                recency_score = 2.0
+                        
+                        # Use description if content is empty or too short
+                        if not content or len(content) < 50:
+                            content = description
+                        
+                        # Truncate content to reduce token usage
+                        if len(content) > 1000:
+                            content = content[:1000] + "..."
+
+                        # Ensure meaningful content
+                        if title and (content or description):
+                            news_item = {
+                                "text": (
+                                    f"Title: {title}, " +
+                                    f"Source: {source_name}, " +
+                                    f"Date: {published_at}, " +
+                                    f"URL: {url}, " +
+                                    f"Content: {content}"
+                                ),
+                                "recency_score": recency_score,
+                                "date": article_date
+                            }
+                            news_results.append(news_item)
+                            source_count["news"] += 1
+                            logger.info(f"Added article: {title}")
+
+                    except Exception as article_error:
+                        logger.error(f"Error processing article: {article_error}")
+                
+                # Sort results by recency
+                if news_results:
+                    news_results.sort(key=lambda x: x.get('recency_score', 0), reverse=True)
+    
+    except Exception as query_error:
+        logger.error(f"Error processing query: {query_error}")
+
+    # Convert to plain text list for compatibility with existing code
+    news_texts = [item["text"] for item in news_results]
+
+    # Log evidence retrieval
+    try:
+        success = bool(news_texts)
+        performance_tracker.log_evidence_retrieval(success, source_count)
+    except Exception as log_error:
+        logger.error(f"Error logging evidence retrieval: {log_error}")
+
+    # Log results
+    if news_texts:
+        logger.info(f"Retrieved {len(news_texts)} news articles")
+    else:
+        logger.warning("No news articles found")
+
+    return news_texts
+
+def retrieve_combined_evidence(claim):
+    """
+    Retrieve evidence from multiple sources in parallel and analyze relevance.
+    
+    This function:
+    1. Extracts claim components (entities, verbs, keywords)
+    2. Determines if the claim is temporal
+    3. Retrieves evidence from all sources in parallel
+    4. Analyzes relevance based on entity and verb matching
+    5. Returns the most relevant evidence items for claim verification
+    
+    Args:
+        claim (str): The factual claim to gather evidence for
+        
+    Returns:
+        list: List of the most relevant evidence items (max 5) for claim verification
+    """
+    logger.info(f"Starting evidence retrieval for: {claim}")
+    start_time = time.time()
+
+    # Use the category detector to identify the claim category
+    from modules.category_detection import get_category_specific_rss_feeds, get_fallback_category, detect_claim_category
+
+    # Extract key claim components for relevance matching
+    claim_components = extract_claim_components(claim)
+    logger.info(f"Extracted claim components: entities={claim_components['entities']}, verbs={claim_components['verbs']}")
+    
+    # Determine if claim has temporal attributes
+    requires_recent_evidence = bool(claim_components.get("temporal_words", []))
+    logger.info(f"Claim requires recent evidence: {requires_recent_evidence}")
+    
+    # Determine the claim category
+    category, confidence = detect_claim_category(claim)
+    logger.info(f"Detected claim category: {category} (confidence: {confidence:.2f})")
+    
+    # Initialize results container
+    all_evidence = []
+    source_counts = {}
+    
+    # Define all evidence sources to query in parallel
+    evidence_sources = [
+        ("wikipedia", retrieve_evidence_from_wikipedia, [claim]),
+        ("wikidata", retrieve_evidence_from_wikidata, [claim]),
+        ("scholarly", retrieve_evidence_from_openalex, [claim]),
+        ("claimreview", retrieve_evidence_from_factcheck, [claim]),
+        ("news", retrieve_news_articles, [claim, requires_recent_evidence])
+    ]
+    
+    # Add RSS feeds based on category with appropriate fallback
+    if category == "ai":
+        # For AI category, add AI-specific RSS feeds
+        category_feeds = get_category_specific_rss_feeds(category)
+        evidence_sources.append(("rss_ai", retrieve_evidence_from_rss, [claim, 10, category_feeds]))
+        
+        # Add technology fallback feeds for AI
+        fallback_category = get_fallback_category(category)  # Should be "technology"
+        if fallback_category:
+            fallback_feeds = get_category_specific_rss_feeds(fallback_category)
+            evidence_sources.append(("rss_tech", retrieve_evidence_from_rss, [claim, 10, fallback_feeds]))
+    else:
+        # For other categories, add their specific RSS feeds
+        category_feeds = get_category_specific_rss_feeds(category)
+        if category_feeds:
+            evidence_sources.append(("rss_category", retrieve_evidence_from_rss, [claim, 10, category_feeds]))
+        
+        # Add default RSS feeds as fallback for all non-AI categories
+        evidence_sources.append(("rss_default", retrieve_evidence_from_rss, [claim, 10]))
+    
+    # Execute all evidence gathering in parallel
+    with ThreadPoolExecutor(max_workers=len(evidence_sources)) as executor:
+        # Create a mapping of futures to source names for easier tracking
+        futures = {}
+        for source_name, func, args in evidence_sources:
+            future = executor.submit(func, *args)
+            futures[future] = source_name
+        
+        # Process results as they complete
+        for future in as_completed(futures):
+            source_name = futures[future]
+            try:
+                evidence_items = future.result()
+                if evidence_items:
+                    all_evidence.extend(evidence_items)
+                    source_counts[source_name] = len(evidence_items)
+                    logger.info(f"Retrieved {len(evidence_items)} items from {source_name}")
+            except Exception as e:
+                logger.error(f"Error retrieving from {source_name}: {str(e)}")
+    
+    # If no evidence was found at all, create a minimal placeholder
+    if not all_evidence:
+        logger.warning("No evidence found from any source")
+        return [f"No specific evidence found for the claim: '{claim}'. This may be due to the claim being very recent, niche, or involving private information."]
+    
+    # Analyze evidence relevance
+    scored_evidence = analyze_evidence_relevance(all_evidence, claim_components)
+    
+    # Return top 10 most relevant evidence items
+    return [evidence for evidence, score in scored_evidence[:10]]

modules/explanation.py

@@ -0,0 +1,282 @@
+import logging
+import re
+import ast
+from utils.models import get_llm_model
+
+logger = logging.getLogger("misinformation_detector")
+
+def extract_most_relevant_evidence(evidence_results):
+    """
+    Intelligently extract the most relevant piece of evidence
+    
+    Args:
+        evidence_results (list): List of evidence items
+    
+    Returns:
+        str: Most relevant evidence piece
+    """
+    if not evidence_results:
+        return None
+
+    # If evidence is a dictionary with 'evidence' key
+    if isinstance(evidence_results[0], dict):
+        # Sort by confidence if available
+        sorted_evidence = sorted(
+            evidence_results, 
+            key=lambda x: x.get('confidence', 0), 
+            reverse=True
+        )
+        
+        # Return the evidence from the highest confidence item
+        for item in sorted_evidence:
+            evidence = item.get('evidence')
+            if evidence:
+                return evidence
+
+    # If plain list of evidence
+    return next((ev for ev in evidence_results if ev and isinstance(ev, str)), None)
+
+def generate_explanation(claim, evidence_results, truth_label, confidence=None):
+    """
+    Generate an explanation for the claim's classification based on evidence.
+    
+    This function creates a human-readable explanation of why a claim was classified
+    as true, false, or uncertain. It handles different truth label formats through
+    normalization and provides robust fallback mechanisms for error cases.
+    
+    Args:
+        claim (str): The original factual claim being verified
+        evidence_results (list/str): Evidence supporting the classification, can be
+                                    a list of evidence items or structured results
+        truth_label (str): Classification of the claim (True/False/Uncertain),
+                           which may come in various formats
+        confidence (float, optional): Confidence level between 0 and 1
+                                     
+    Returns:
+        str: Natural language explanation of the verdict with appropriate
+             confidence framing and evidence citations
+    """
+    logger.info(f"Generating explanation for claim with verdict: {truth_label}")
+    
+    try:
+        # Normalize truth_label to handle different formats consistently
+        normalized_label = normalize_truth_label(truth_label)
+        
+        # Normalize evidence_results to a list
+        if not isinstance(evidence_results, list):
+            try:
+                evidence_results = ast.literal_eval(str(evidence_results)) if evidence_results else []
+            except:
+                evidence_results = [evidence_results] if evidence_results else []
+
+        # Get the LLM model
+        explanation_model = get_llm_model()
+
+        # Extract most relevant evidence
+        most_relevant_evidence = extract_most_relevant_evidence(evidence_results)
+
+        # Prepare evidence text for prompt
+        evidence_text = "\n".join([
+            f"Evidence {i+1}: {str(ev)[:200] + '...' if len(str(ev)) > 200 else str(ev)}" 
+            for i, ev in enumerate(evidence_results[:5])
+        ])
+
+        # Filter only supporting and contradicting evidence for clarity
+        support_items = [item for item in evidence_results if isinstance(item, dict) and item.get("label") == "support"]
+        contradict_items = [item for item in evidence_results if isinstance(item, dict) and item.get("label") == "contradict"]
+        
+        # Convert confidence to percentage and description
+        confidence_desc = ""
+        very_low_confidence = False
+        
+        # For Uncertain verdicts, always use 0% confidence regardless of evidence confidence values
+        if "uncertain" in normalized_label.lower():
+            confidence = 0.0
+            confidence_desc = "no confidence (0%)"
+        elif confidence is not None:
+            confidence_pct = int(confidence * 100)
+            
+            if confidence == 0.0:
+                confidence_desc = "no confidence (0%)"
+            elif confidence < 0.1:
+                confidence_desc = f"very low confidence ({confidence_pct}%)"
+                very_low_confidence = True
+            elif confidence < 0.3:
+                confidence_desc = f"low confidence ({confidence_pct}%)"
+            elif confidence < 0.7:
+                confidence_desc = f"moderate confidence ({confidence_pct}%)"
+            elif confidence < 0.9:
+                confidence_desc = f"high confidence ({confidence_pct}%)"
+            else:
+                confidence_desc = f"very high confidence ({confidence_pct}%)"
+        else:
+            # Default if no confidence provided
+            confidence_desc = "uncertain confidence"
+
+        # Create prompt with specific instructions based on the type of claim
+        has_negation = any(neg in claim.lower() for neg in ["not", "no longer", "isn't", "doesn't", "won't", "cannot"])
+        
+        # For claims with "True" verdict
+        if "true" in normalized_label.lower():
+            # Special case for very low confidence (but not zero)
+            if very_low_confidence:
+                prompt = f"""
+                Claim: "{claim}"
+                
+                Verdict: {normalized_label} (with {confidence_desc})
+
+                Available Evidence:
+                {evidence_text}
+
+                Task: Generate a clear explanation that:
+                1. States that the claim appears to be true based on the available evidence
+                2. EMPHASIZES that the confidence level is VERY LOW ({confidence_pct}%)
+                3. Explains that this means the evidence slightly favors the claim but is not strong enough to be certain
+                4. STRONGLY recommends that the user verify this with other authoritative sources
+                5. Is factual and precise
+                """
+            else:
+                prompt = f"""
+                Claim: "{claim}"
+                
+                Verdict: {normalized_label} (with {confidence_desc})
+
+                Available Evidence:
+                {evidence_text}
+
+                Task: Generate a clear explanation that:
+                1. Clearly states that the claim IS TRUE based on the evidence
+                2. {"Pay special attention to the logical relationship since the claim contains negation" if has_negation else "Explains why the evidence supports the claim"}
+                3. Uses confidence level of {confidence_desc}
+                4. Highlights the most relevant supporting evidence
+                5. Is factual and precise
+                """
+
+        # For claims with "False" verdict
+        elif "false" in normalized_label.lower():
+            # Special case for very low confidence (but not zero)
+            if very_low_confidence:
+                prompt = f"""
+                Claim: "{claim}"
+                
+                Verdict: {normalized_label} (with {confidence_desc})
+
+                Available Evidence:
+                {evidence_text}
+
+                Task: Generate a clear explanation that:
+                1. States that the claim appears to be false based on the available evidence
+                2. EMPHASIZES that the confidence level is VERY LOW ({confidence_pct}%)
+                3. Explains that this means the evidence slightly contradicts the claim but is not strong enough to be certain
+                4. STRONGLY recommends that the user verify this with other authoritative sources
+                5. Is factual and precise
+                """
+            else:
+                prompt = f"""
+                Claim: "{claim}"
+                
+                Verdict: {normalized_label} (with {confidence_desc})
+
+                Available Evidence:
+                {evidence_text}
+
+                Task: Generate a clear explanation that:
+                1. Clearly states that the claim IS FALSE based on the evidence
+                2. {"Pay special attention to the logical relationship since the claim contains negation" if has_negation else "Explains why the evidence contradicts the claim"}
+                3. Uses confidence level of {confidence_desc}
+                4. Highlights the contradicting evidence
+                5. Is factual and precise
+                """
+
+        # For uncertain claims
+        else:
+            prompt = f"""
+            Claim: "{claim}"
+            
+            Verdict: {normalized_label} (with {confidence_desc})
+
+            Available Evidence:
+            {evidence_text}
+
+            Task: Generate a clear explanation that:
+            1. Clearly states that there is insufficient evidence to determine if the claim is true or false
+            2. Explains what information is missing or why the available evidence is insufficient
+            3. Uses confidence level of {confidence_desc}
+            4. Makes NO speculation about whether the claim might be true or false
+            5. Explicitly mentions that the user should seek information from other reliable sources
+            """
+
+        # Generate explanation with multiple attempts for reliability
+        max_attempts = 3
+        for attempt in range(max_attempts):
+            try:
+                # Invoke the model
+                response = explanation_model.invoke(prompt)
+                explanation = response.content.strip()
+
+                # Validate explanation length
+                if explanation and len(explanation.split()) >= 5:
+                    return explanation
+
+            except Exception as attempt_error:
+                logger.error(f"Explanation generation attempt {attempt+1} failed: {str(attempt_error)}")
+
+        # Ultimate fallback explanations if all attempts fail
+        if "uncertain" in normalized_label.lower():
+            return f"The claim '{claim}' cannot be verified due to insufficient evidence. The available information does not provide clear support for or against this claim. Consider consulting reliable sources for verification."
+        elif very_low_confidence:
+            return f"The claim '{claim}' appears to be {'supported' if 'true' in normalized_label.lower() else 'contradicted'} by the evidence, but with very low confidence ({confidence_pct}%). The evidence is not strong enough to make a definitive determination. It is strongly recommended to verify this information with other authoritative sources."
+        elif "true" in normalized_label.lower():
+            return f"The claim '{claim}' is supported by the evidence with {confidence_desc}. {most_relevant_evidence or 'The evidence indicates this claim is accurate.'}"
+        else:
+            return f"The claim '{claim}' is contradicted by the evidence with {confidence_desc}. {most_relevant_evidence or 'The evidence indicates this claim is not accurate.'}"
+
+    except Exception as e:
+        logger.error(f"Comprehensive error in explanation generation: {str(e)}")
+        # Final fallback with minimal but useful information
+        normalized_label = normalize_truth_label(truth_label)
+        return f"The claim is classified as {normalized_label} based on the available evidence."
+
+def normalize_truth_label(truth_label):
+    """
+    Normalize truth label to handle different formats consistently.
+    
+    This function extracts the core truth classification (True/False/Uncertain) from
+    potentially complex or inconsistently formatted truth labels. It preserves
+    contextual information like "(Based on Evidence)" when present.
+    
+    Args:
+        truth_label (str): The truth label to normalize, which may contain 
+                           additional descriptive text or formatting
+        
+    Returns:
+        str: Normalized truth label that preserves the core classification and
+             important context while eliminating inconsistencies
+             
+    Examples:
+        >>> normalize_truth_label("True (Based on Evidence)")
+        "True (Based on Evidence)"
+        >>> normalize_truth_label("false (Based on Evidence)")
+        "False (Based on Evidence)"
+        >>> normalize_truth_label("The evidence shows this claim is False")
+        "False"
+    """
+    if not truth_label:
+        return "Uncertain"
+    
+    # Convert to string if not already
+    label_str = str(truth_label)
+    
+    # Extract the core label if it contains additional text like "(Based on Evidence)"
+    base_label_match = re.search(r'(True|False|Uncertain|Error)', label_str, re.IGNORECASE)
+    if base_label_match:
+        # Get the core label and capitalize it for consistency
+        base_label = base_label_match.group(1).capitalize()
+        
+        # Add back the context if it was present
+        if "(Based on Evidence)" in label_str:
+            return f"{base_label} (Based on Evidence)"
+        return base_label
+    
+    # Return the original if we couldn't normalize it
+    return label_str

modules/rss_feed.py

@@ -0,0 +1,408 @@
+import feedparser
+import time
+import logging
+import re
+import ssl
+import requests
+from datetime import datetime, timedelta
+from threading import Timer
+from urllib.parse import urlparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+logger = logging.getLogger("misinformation_detector")
+
+# Disable SSL certificate verification for feeds with self-signed certs
+ssl._create_default_https_context = ssl._create_unverified_context
+
+# List of RSS feeds to check for news
+# These are popular news sources with reliable and frequently updated RSS feeds
+RSS_FEEDS = [
+# --------------------
+# 🌐 General World News
+# --------------------
+"http://rss.cnn.com/rss/cnn_world.rss",                         # CNN World News
+"https://rss.nytimes.com/services/xml/rss/nyt/World.xml",      # NYT World News
+"https://feeds.washingtonpost.com/rss/world",                  # The Washington Post World News
+"https://feeds.bbci.co.uk/news/world/rss.xml",                 # BBC News - World
+
+# --------------------
+# 🧠 Tech & Startup News (Global)
+# --------------------
+"https://techcrunch.com/feed/",                                # TechCrunch - Startup and Technology News
+"https://venturebeat.com/feed/",                               # VentureBeat - Tech News
+"https://www.wired.com/feed/rss",                              # Wired - Technology News
+"https://www.cnet.com/rss/news/",                              # CNET - Technology News
+"https://news.google.com/rss?gl=IN&ceid=IN:en&topic=t&hl=en-IN",  # Google News India - Technology
+"https://news.google.com/rss?gl=US&ceid=US:en&topic=t&hl=en-US",  # Google News US - Technology
+
+# --------------------
+# 💼 Startup & VC Focused
+# --------------------
+"https://news.crunchbase.com/feed/",                           # Crunchbase News - Startup Funding
+"https://techstartups.com/feed/",                              # Tech Startups - Startup News
+
+# --------------------
+# 📰 Global Business & Corporate Feeds
+# --------------------
+"https://feeds.bloomberg.com/technology/news.rss",             # Bloomberg Technology News
+"https://www.ft.com/technology?format=rss",                    # Financial Times Technology News
+"https://news.google.com/rss?gl=IN&ceid=IN:en&topic=b&hl=en-IN",  # Google News India - Business
+
+# --------------------
+# 🇮🇳 India-specific News
+# --------------------
+"https://inc42.com/feed/",                                     # Inc42 - Indian Startups and Technology
+"https://timesofindia.indiatimes.com/rssfeedstopstories.cms",           # TOI - Top Stories
+"https://timesofindia.indiatimes.com/rssfeedmostrecent.cms",            # TOI - Most Recent Stories
+"https://timesofindia.indiatimes.com/rssfeeds/-2128936835.cms",         # TOI - India News
+"https://timesofindia.indiatimes.com/rssfeeds/296589292.cms",           # TOI - World News
+"https://timesofindia.indiatimes.com/rssfeeds/1898055.cms",             # TOI - Business News
+"https://timesofindia.indiatimes.com/rssfeeds/54829575.cms",            # TOI - Cricket News
+"https://timesofindia.indiatimes.com/rssfeeds/4719148.cms",             # TOI - Sports News
+"https://timesofindia.indiatimes.com/rssfeeds/-2128672765.cms",         # TOI - Science News
+
+# --------------------
+# 🏏 Sports News (Global + Cricket)
+# --------------------
+"https://www.espn.com/espn/rss/news",                          # ESPN - Top Sports News
+"https://feeds.skynews.com/feeds/rss/sports.xml",              # Sky News - Sports
+"https://sports.ndtv.com/rss/all",                                 # NDTV Sports
+"https://www.espncricinfo.com/rss/content/story/feeds/0.xml",  # ESPN Cricinfo - Cricket News
+
+# --------------------
+# ✅ Fact-Checking Sources
+# --------------------
+"https://www.snopes.com/feed/",                                # Snopes - Fact Checking
+"https://www.politifact.com/rss/all/",                         # PolitiFact - Fact Checking
+"https://www.factcheck.org/feed/",                             # FactCheck - Fact Checking
+"https://leadstories.com/atom.xml",                            # Lead Stories - Fact Checking
+"https://fullfact.org/feed/all/",                              # Full Fact - Fact Checking
+"https://www.truthorfiction.com/feed/",                         # TruthOrFiction - Fact Checking
+
+# --------------------
+# 🗳️ Politics & Policy (General)
+# --------------------
+"https://feeds.bbci.co.uk/news/politics/rss.xml",              # BBC News - Politics
+"https://feeds.bbci.co.uk/news/science_and_environment/rss.xml",  # BBC - Science & Environment
+
+# --------------------
+# 🗳️ Science
+# --------------------
+"https://www.nature.com/nature.rss",                              # Nature science
+"https://feeds.science.org/rss/science-advances.xml"              # science.org
+]
+
+def clean_html(raw_html):
+    """Remove HTML tags from text"""
+    if not raw_html:
+        return ""
+    clean_regex = re.compile('<.*?>')
+    clean_text = re.sub(clean_regex, '', raw_html)
+    # Remove extra whitespace
+    clean_text = re.sub(r'\s+', ' ', clean_text).strip()
+    return clean_text
+
+def parse_feed(feed_url, timeout=5):
+    """
+    Parse a single RSS feed with proper timeout handling
+    Uses requests with timeout first, then passes content to feedparser
+    """
+    try:
+        # Use requests with timeout to fetch the RSS content
+        response = requests.get(feed_url, timeout=timeout)
+        response.raise_for_status()
+        
+        # Then parse the content with feedparser (which doesn't support timeout)
+        feed = feedparser.parse(response.content)
+        
+        # Basic validation of the feed
+        if hasattr(feed, 'entries') and feed.entries:
+            return feed
+        else:
+            logger.warning(f"Feed {feed_url} parsed but contains no entries")
+            return None
+            
+    except requests.exceptions.Timeout:
+        logger.warning(f"Timeout while fetching feed {feed_url}")
+        return None
+    except requests.exceptions.RequestException as e:
+        logger.error(f"Request error fetching feed {feed_url}: {str(e)}")
+        return None
+    except Exception as e:
+        logger.error(f"Error parsing feed {feed_url}: {str(e)}")
+        return None
+
+def fetch_all_feeds(feeds_list=None, max_workers=5, timeout=5):
+    """
+    Fetch multiple RSS feeds with proper timeout handling
+    Returns a list of (domain, feed) tuples for successfully fetched feeds
+    """
+    # Use default RSS_FEEDS list if none provided
+    if feeds_list is None:
+        feeds_list = RSS_FEEDS
+    
+    results = []
+    
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_to_url = {executor.submit(parse_feed, url, timeout): url for url in feeds_list}
+        for future in as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                feed = future.result()
+                if feed and hasattr(feed, 'entries') and feed.entries:
+                    # Extract domain for source attribution
+                    domain = urlparse(url).netloc
+                    results.append((domain, feed))
+                    logger.info(f"Successfully fetched {domain} with {len(feed.entries)} entries")
+            except Exception as e:
+                logger.error(f"Error processing {url}: {str(e)}")
+    
+    return results
+
+def extract_date(entry):
+    """Extract and normalize publication date from entry"""
+    for date_field in ['published_parsed', 'updated_parsed', 'created_parsed']:
+        if hasattr(entry, date_field) and getattr(entry, date_field):
+            try:
+                # Convert time tuple to datetime
+                time_tuple = getattr(entry, date_field)
+                return datetime(time_tuple[0], time_tuple[1], time_tuple[2], 
+                               time_tuple[3], time_tuple[4], time_tuple[5])
+            except Exception as e:
+                logger.debug(f"Error parsing {date_field}: {e}")
+                continue
+    
+    # Try string dates
+    for date_field in ['published', 'updated', 'pubDate']:
+        if hasattr(entry, date_field) and getattr(entry, date_field):
+            try:
+                date_str = getattr(entry, date_field)
+                # Try various formats
+                for fmt in ['%a, %d %b %Y %H:%M:%S %z', '%a, %d %b %Y %H:%M:%S %Z', 
+                           '%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S%z']:
+                    try:
+                        return datetime.strptime(date_str, fmt)
+                    except:
+                        continue
+            except Exception as e:
+                logger.debug(f"Error parsing date string {date_field}: {e}")
+                continue
+    
+    # Default to current time if parsing fails
+    return datetime.now()
+
+def is_recent(entry_date, claim=None, max_days=3):
+    """
+    Check if an entry is recent based on temporal indicators in the claim.
+    
+    Args:
+        entry_date (datetime): The date of the entry to check
+        claim (str, optional): The claim text to analyze for temporal indicators
+        max_days (int, optional): Default maximum age in days
+        
+    Returns:
+        bool: True if entry is considered recent, False otherwise
+    """
+    if not entry_date:
+        return False
+    
+    # Default max days if no claim is provided
+    default_days = max_days
+    extended_days = 15  # For 'recently', 'this week', etc.
+    
+    if claim:
+        # Specific day indicators get default days
+        specific_day_terms = ["today", "yesterday", "day before yesterday"]
+        
+        # Extended time terms get extended days
+        extended_time_terms = [
+            "recently", "currently", "freshly", "this week", "few days", 
+            "couple of days", "last week", "past week", "several days",
+            "anymore"
+        ]
+        
+        claim_lower = claim.lower()
+        
+        # Check for extended time terms first, then specific day terms
+        if any(term in claim_lower for term in extended_time_terms):
+            cutoff = datetime.now() - timedelta(days=extended_days)
+            return entry_date > cutoff
+        elif any(term in claim_lower for term in specific_day_terms):
+            cutoff = datetime.now() - timedelta(days=default_days)
+            return entry_date > cutoff
+    
+    # Default case - use standard window
+    cutoff = datetime.now() - timedelta(days=default_days)
+    return entry_date > cutoff
+
+def get_entry_relevance(entry, query_terms, domain):
+    """Calculate relevance score for an entry based on query match and recency"""
+    if not hasattr(entry, 'title') or not entry.title:
+        return 0
+    
+    # Extract text content
+    title = entry.title or ""
+    description = clean_html(entry.description) if hasattr(entry, 'description') else ""
+    content = ""
+    if hasattr(entry, 'content'):
+        for content_item in entry.content:
+            if 'value' in content_item:
+                content += clean_html(content_item['value']) + " "
+    
+    # Extract published date
+    pub_date = extract_date(entry)
+    
+    # Calculate recency score (0-1)
+    recency_score = 0
+    if pub_date:
+        days_old = (datetime.now() - pub_date).days
+        if days_old <= 1:  # Today or yesterday
+            recency_score = 1.0
+        elif days_old <= 2:
+            recency_score = 0.8
+        elif days_old <= 3:
+            recency_score = 0.5
+        else:
+            recency_score = 0.2
+    
+    # Calculate relevance score based on keyword matches
+    text = f"{title} {description} {content}".lower()
+    
+    # Count how many query terms appear in the content
+    query_terms_lower = [term.lower() for term in query_terms]
+    matches = sum(1 for term in query_terms_lower if term in text)
+    
+    # Calculate match score (0-1)
+    match_score = min(1.0, matches / max(1, len(query_terms) * 0.7))
+    
+    # Boost score for exact phrase matches
+    query_phrase = " ".join(query_terms_lower)
+    if query_phrase in text:
+        match_score += 0.5
+    
+    # Additional boost for title matches (they're more relevant)
+    title_matches = sum(1 for term in query_terms_lower if term in title.lower())
+    if title_matches > 0:
+        match_score += 0.2 * (title_matches / len(query_terms_lower))
+    
+    # Source quality factor (can be adjusted based on source reliability)
+    source_factor = 1.0
+    high_quality_domains = ['bbc.co.uk', 'nytimes.com', 'reuters.com', 'washingtonpost.com', 
+                           'espncricinfo.com', 'cricbuzz.com', 'snopes.com']
+    if any(quality_domain in domain for quality_domain in high_quality_domains):
+        source_factor = 1.2
+    
+    # Calculate final score
+    final_score = (match_score * 0.6) + (recency_score * 0.4) * source_factor
+    
+    return min(1.0, final_score)  # Cap at 1.0
+
+def retrieve_evidence_from_rss(claim, max_results=10, category_feeds=None):
+    """
+    Retrieve evidence from RSS feeds for a given claim
+    
+    Args:
+        claim (str): The claim to verify
+        max_results (int): Maximum number of results to return
+        category_feeds (list, optional): List of category-specific RSS feeds to check
+        
+    Returns:
+        list: List of relevant evidence items
+    """
+    start_time = time.time()
+    logger.info(f"Retrieving evidence from RSS feeds for: {claim}")
+    
+    # Extract key terms from claim
+    terms = [term.strip() for term in re.findall(r'\b\w+\b', claim) if len(term.strip()) > 2]
+    
+    try:
+        # Use category-specific feeds if provided
+        feeds_to_use = category_feeds if category_feeds else RSS_FEEDS
+        
+        # Log which feeds we're using
+        if category_feeds:
+            logger.info(f"Using {len(category_feeds)} category-specific RSS feeds")
+        else:
+            logger.info(f"Using {len(RSS_FEEDS)} default RSS feeds")
+        
+        # Limit the number of feeds to process for efficiency
+        if len(feeds_to_use) > 10:
+            # If we have too many feeds, select a subset
+            # Prioritize fact-checking sources
+            fact_check_feeds = [feed for feed in feeds_to_use if "fact" in feed.lower() or "snopes" in feed.lower() or "politifact" in feed.lower()]
+            other_feeds = [feed for feed in feeds_to_use if feed not in fact_check_feeds]
+            
+            # Take all fact-checking feeds plus a random selection of others
+            import random
+            selected_feeds = fact_check_feeds + random.sample(other_feeds, min(10 - len(fact_check_feeds), len(other_feeds)))
+        else:
+            selected_feeds = feeds_to_use
+            
+        # Fetch all feeds in parallel with the selected feeds
+        feeds = fetch_all_feeds(selected_feeds)
+        
+        if not feeds:
+            logger.warning("No RSS feeds could be fetched")
+            return []
+        
+        all_entries = []
+        
+        # Process all feed entries
+        for domain, feed in feeds:
+            for entry in feed.entries:
+                # Calculate relevance score
+                relevance = get_entry_relevance(entry, terms, domain)
+                
+                if relevance > 0.3:  # Only consider somewhat relevant entries
+                    # Extract entry details
+                    title = entry.title if hasattr(entry, 'title') else "No title"
+                    link = entry.link if hasattr(entry, 'link') else ""
+                    
+                    # Extract and clean description/content
+                    description = ""
+                    if hasattr(entry, 'description'):
+                        description = clean_html(entry.description)
+                    elif hasattr(entry, 'summary'):
+                        description = clean_html(entry.summary)
+                    elif hasattr(entry, 'content'):
+                        for content_item in entry.content:
+                            if 'value' in content_item:
+                                description += clean_html(content_item['value']) + " "
+                    
+                    # Truncate description if too long
+                    if len(description) > 1000:
+                        description = description[:1000] + "..."
+                    
+                    # Get publication date
+                    pub_date = extract_date(entry)
+                    date_str = pub_date.strftime('%Y-%m-%d') if pub_date else "Unknown date"
+                    
+                    # Format as evidence text
+                    evidence_text = (
+                        f"Title: {title}, "
+                        f"Source: {domain} (RSS), "
+                        f"Date: {date_str}, "
+                        f"URL: {link}, "
+                        f"Content: {description}"
+                    )
+                    
+                    all_entries.append({
+                        "text": evidence_text,
+                        "relevance": relevance,
+                        "date": pub_date or datetime.now()
+                    })
+        
+        # Sort entries by relevance
+        all_entries.sort(key=lambda x: x["relevance"], reverse=True)
+        
+        # Take top results
+        top_entries = all_entries[:max_results]
+        
+        logger.info(f"Retrieved {len(top_entries)} relevant RSS items from {len(feeds)} feeds in {time.time() - start_time:.2f}s")
+        
+        # Return just the text portion
+        return [entry["text"] for entry in top_entries]
+    
+    except Exception as e:
+        logger.error(f"Error in RSS retrieval: {str(e)}")
+        return []

utils/__init__.py

@@ -0,0 +1,20 @@
+"""
+Utils package initialization.
+
+This package provides utility functions for the AskVeracity fact-checking system.
+"""
+
+from .api_utils import api_error_handler, safe_json_parse, RateLimiter
+from .performance import PerformanceTracker
+from .models import initialize_models, get_nlp_model, get_llm_model
+
+
+__all__ = [
+    'api_error_handler',
+    'safe_json_parse',
+    'RateLimiter',
+    'PerformanceTracker',
+    'initialize_models',
+    'get_nlp_model',
+    'get_llm_model'
+]

utils/api_utils.py

@@ -0,0 +1,229 @@
+"""
+API utilities for the Fake News Detector application.
+
+This module provides utilities for handling API calls, rate limiting,
+error handling, and exponential backoff for retrying failed requests.
+"""
+
+import time
+import functools
+import random
+import logging
+import requests
+from datetime import datetime, timedelta
+from collections import deque
+
+from config import RATE_LIMITS, ERROR_BACKOFF
+
+logger = logging.getLogger("misinformation_detector")
+
+class RateLimiter:
+    """
+    Rate limiter for API calls with support for different APIs.
+    
+    This class implements a token bucket algorithm for rate limiting,
+    with support for different rate limits for different APIs.
+    It also provides exponential backoff for error handling.
+    """
+    
+    def __init__(self):
+        """Initialize the rate limiter with configuration from settings."""
+        # Store rate limits for different APIs
+        self.limits = {}
+        
+        # Initialize limits from config
+        for api_name, limit_info in RATE_LIMITS.items():
+            self.limits[api_name] = {
+                "requests": limit_info["requests"], 
+                "period": limit_info["period"], 
+                "timestamps": deque()
+            }
+
+        # Error backoff settings
+        self.max_retries = ERROR_BACKOFF["max_retries"]
+        self.initial_backoff = ERROR_BACKOFF["initial_backoff"]
+        self.backoff_factor = ERROR_BACKOFF["backoff_factor"]
+
+    def check_and_update(self, api_name):
+        """
+        Check if request is allowed and update timestamps.
+        
+        Args:
+            api_name (str): Name of the API to check
+            
+        Returns:
+            tuple: (allowed, wait_time)
+                - allowed (bool): Whether the request is allowed
+                - wait_time (float): Time to wait if not allowed
+        """
+        if api_name not in self.limits:
+            return True, 0  # Unknown API, allow by default
+
+        now = datetime.now()
+        limit_info = self.limits[api_name]
+
+        # Remove timestamps older than the period
+        cutoff = now - timedelta(seconds=limit_info["period"])
+        while limit_info["timestamps"] and limit_info["timestamps"][0] < cutoff:
+            limit_info["timestamps"].popleft()
+
+        # Check if we're at the rate limit
+        if len(limit_info["timestamps"]) >= limit_info["requests"]:
+            # Calculate wait time until oldest timestamp expires
+            wait_time = (limit_info["timestamps"][0] + timedelta(seconds=limit_info["period"]) - now).total_seconds()
+            return False, max(0, wait_time)
+
+        # Add current timestamp and allow request
+        limit_info["timestamps"].append(now)
+        return True, 0
+
+    def wait_if_needed(self, api_name):
+        """
+        Wait if rate limit is reached.
+        
+        Args:
+            api_name (str): Name of the API to check
+            
+        Returns:
+            bool: True if waited, False otherwise
+        """
+        allowed, wait_time = self.check_and_update(api_name)
+        if not allowed:
+            logger.info(f"Rate limit reached for {api_name}. Waiting {wait_time:.2f} seconds...")
+            time.sleep(wait_time + 0.1)  # Add a small buffer
+            return True
+        return False
+
+    def get_backoff_time(self, attempt):
+        """
+        Calculate exponential backoff time with jitter.
+        
+        Args:
+            attempt (int): Current attempt number (0-based)
+            
+        Returns:
+            float: Backoff time in seconds
+        """
+        backoff = self.initial_backoff * (self.backoff_factor ** attempt)
+        # Add jitter to prevent thundering herd problem
+        jitter = random.uniform(0, 0.1 * backoff)
+        return backoff + jitter
+
+
+# Create rate limiter instance
+rate_limiter = RateLimiter()
+
+# API Error Handler decorator
+def api_error_handler(api_name):
+    """
+    Decorator for API calls with error handling and rate limiting.
+    
+    This decorator handles rate limiting, retries with exponential
+    backoff, and error handling for API calls.
+    
+    Args:
+        api_name (str): Name of the API being called
+        
+    Returns:
+        callable: Decorated function
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                # Apply rate limiting - make sure rate_limiter exists and has the method
+                if hasattr(rate_limiter, 'wait_if_needed'):
+                    rate_limiter.wait_if_needed(api_name)
+
+                # Track retries
+                for attempt in range(rate_limiter.max_retries):
+                    try:
+                        return func(*args, **kwargs)
+                    except requests.exceptions.HTTPError as e:
+                        status_code = e.response.status_code if hasattr(e, 'response') else 0
+
+                        # Handle specific HTTP errors
+                        if status_code == 429:  # Too Many Requests
+                            logger.warning(f"{api_name} rate limit exceeded (429). Attempt {attempt+1}/{rate_limiter.max_retries}")
+                            # Get retry-after header or use exponential backoff
+                            retry_after = e.response.headers.get('Retry-After')
+                            if retry_after and retry_after.isdigit():
+                                wait_time = int(retry_after)
+                            else:
+                                wait_time = rate_limiter.get_backoff_time(attempt)
+                            logger.info(f"Waiting {wait_time} seconds before retry...")
+                            time.sleep(wait_time)
+                        elif status_code >= 500:  # Server errors
+                            logger.warning(f"{api_name} server error ({status_code}). Attempt {attempt+1}/{rate_limiter.max_retries}")
+                            time.sleep(rate_limiter.get_backoff_time(attempt))
+                        elif status_code == 403:  # Forbidden - likely API key issue
+                            logger.error(f"{api_name} access forbidden (403). Check API key.")
+                            return None  # Don't retry on auth errors
+                        elif status_code == 404:  # Not Found
+                            logger.warning(f"{api_name} resource not found (404).")
+                            return None  # Don't retry on resource not found
+                        else:
+                            logger.error(f"{api_name} HTTP error: {e}")
+                            if attempt < rate_limiter.max_retries - 1:
+                                wait_time = rate_limiter.get_backoff_time(attempt)
+                                logger.info(f"Waiting {wait_time} seconds before retry...")
+                                time.sleep(wait_time)
+                            else:
+                                return None
+
+                    except requests.exceptions.ConnectionError as e:
+                        logger.error(f"{api_name} connection error: {e}")
+                        if attempt < rate_limiter.max_retries - 1:
+                            wait_time = rate_limiter.get_backoff_time(attempt)
+                            logger.info(f"Waiting {wait_time} seconds before retry...")
+                            time.sleep(wait_time)
+                        else:
+                            return None
+
+                    except requests.exceptions.Timeout as e:
+                        logger.error(f"{api_name} timeout error: {e}")
+                        if attempt < rate_limiter.max_retries - 1:
+                            wait_time = rate_limiter.get_backoff_time(attempt)
+                            logger.info(f"Waiting {wait_time} seconds before retry...")
+                            time.sleep(wait_time)
+                        else:
+                            return None
+
+                    except Exception as e:
+                        logger.error(f"{api_name} unexpected error: {str(e)}")
+                        if attempt < rate_limiter.max_retries - 1:
+                            wait_time = rate_limiter.get_backoff_time(attempt)
+                            logger.info(f"Waiting {wait_time} seconds before retry...")
+                            time.sleep(wait_time)
+                        else:
+                            return None
+
+                # If we've exhausted all retries
+                logger.error(f"{api_name} call failed after {rate_limiter.max_retries} attempts")
+                return None
+
+            except Exception as e:
+                # Catch any unexpected errors in the decorator itself
+                logger.error(f"{api_name} decorator error: {str(e)}")
+                return None
+
+        return wrapper
+    return decorator
+
+def safe_json_parse(response, api_name):
+    """
+    Safely parse JSON response with error handling.
+    
+    Args:
+        response (requests.Response): Response object to parse
+        api_name (str): Name of the API for logging
+        
+    Returns:
+        dict: Parsed JSON or empty dict on error
+    """
+    try:
+        return response.json()
+    except ValueError as e:
+        logger.error(f"Error parsing {api_name} JSON response: {e}")
+        logger.debug(f"Response content: {response.text[:500]}...")
+        return {}

utils/models.py

@@ -0,0 +1,157 @@
+"""
+Model management utility for the Fake News Detector application.
+
+This module provides functions for initializing, caching, and
+retrieving language models used throughout the application.
+It ensures models are loaded efficiently and reused appropriately.
+"""
+
+import os
+import logging
+import functools
+from langchain_openai import ChatOpenAI
+import spacy
+
+logger = logging.getLogger("misinformation_detector")
+
+# Global variables for models
+nlp = None
+model = None
+models_initialized = False
+
+# Add caching decorator
+def cached_model(func):
+    """
+    Decorator to cache model loading for improved performance.
+    
+    This decorator ensures that models are only loaded once and
+    then reused for subsequent calls, improving performance by
+    avoiding redundant model loading.
+    
+    Args:
+        func (callable): Function that loads a model
+        
+    Returns:
+        callable: Wrapped function that returns a cached model
+    """
+    cache = {}
+    
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        # Use function name as cache key
+        key = func.__name__
+        if key not in cache:
+            logger.info(f"Model not in cache, calling {key}...")
+            cache[key] = func(*args, **kwargs)
+        return cache[key]
+    
+    return wrapper
+
+def initialize_models():
+    """
+    Initialize all required models.
+    
+    This function loads and initializes all the language models
+    needed by the application, including spaCy for NLP tasks and
+    OpenAI for LLM-based processing.
+    
+    Returns:
+        str: Initialization status message
+        
+    Raises:
+        ValueError: If OpenAI API key is not set
+    """
+    global nlp, model, models_initialized
+    
+    # Skip initialization if already done
+    if models_initialized:
+        logger.info("Models already initialized, skipping initialization")
+        return "Models already initialized"
+    
+    # Check OpenAI API key
+    if "OPENAI_API_KEY" not in os.environ or not os.environ["OPENAI_API_KEY"].strip():
+        logger.error("OPENAI_API_KEY environment variable not set or empty")
+        raise ValueError("OpenAI API key is required. Please set it in the Hugging Face Space secrets.")
+
+    try:
+        # Load NLP model
+        try:
+            logger.info("Loading spaCy NLP model...")
+            nlp = spacy.load("en_core_web_sm")
+            logger.info("Loaded spaCy NLP model")
+        except OSError as e:
+            # This handles the case if the model wasn't installed correctly
+            logger.warning(f"Could not load spaCy model: {str(e)}")
+            logger.info("Attempting to download spaCy model...")
+            try:
+                import subprocess
+                import sys
+                # This downloads the model if it's missing
+                subprocess.check_call([sys.executable, "-m", "spacy", "download", "en_core_web_sm"])
+                # Try loading again
+                nlp = spacy.load("en_core_web_sm")
+                logger.info("Successfully downloaded and loaded spaCy model")
+            except Exception as download_err:
+                logger.error(f"Failed to download spaCy model: {str(download_err)}")
+                # Continue with other initialization, we'll handle missing NLP model elsewhere
+
+        # Set up OpenAI model
+        logger.info("Initializing ChatOpenAI model...")
+        model = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
+        logger.info("Initialized ChatOpenAI model")
+        
+        # Mark initialization as complete
+        models_initialized = True
+        return "Models initialized successfully"
+    
+    except Exception as e:
+        logger.error(f"Error initializing models: {str(e)}")
+        raise e
+
+@cached_model
+def get_nlp_model():
+    """
+    Get the spaCy NLP model, initializing if needed.
+    
+    This function returns a cached spaCy model for NLP tasks.
+    If the model hasn't been loaded yet, it will be loaded.
+    
+    Returns:
+        spacy.Language: Loaded spaCy model
+    """
+    global nlp
+    if nlp is None:
+        try:
+            # Try to load just the spaCy model if not loaded yet
+            logger.info("Loading spaCy NLP model...")
+            nlp = spacy.load("en_core_web_sm")
+            logger.info("Loaded spaCy NLP model")
+        except Exception as e:
+            logger.error(f"Error loading spaCy model: {str(e)}")
+            # Fall back to full initialization
+            initialize_models()
+    return nlp
+
+@cached_model
+def get_llm_model():
+    """
+    Get the ChatOpenAI model, initializing if needed.
+    
+    This function returns a cached OpenAI LLM model.
+    If the model hasn't been loaded yet, it will be loaded.
+    
+    Returns:
+        ChatOpenAI: Loaded LLM model
+    """
+    global model
+    if model is None:
+        try:
+            # Try to load just the LLM model if not loaded yet
+            logger.info("Initializing ChatOpenAI model...")
+            model = ChatOpenAI(model="gpt-3.5-turbo", temperature=0)
+            logger.info("Initialized ChatOpenAI model")
+        except Exception as e:
+            logger.error(f"Error initializing ChatOpenAI model: {str(e)}")
+            # Fall back to full initialization
+            initialize_models()
+    return model

utils/performance.py

@@ -0,0 +1,135 @@
+"""
+Performance tracking utility for the Fake News Detector application.
+
+This module provides functionality to track and analyze the
+performance of the application, including processing times,
+success rates, and resource utilization.
+"""
+
+import time
+import logging
+
+logger = logging.getLogger("misinformation_detector")
+
+class PerformanceTracker:
+    """
+    Tracks and logs performance metrics for the fact-checking system.
+    
+    This class maintains counters and statistics for various performance
+    metrics, such as processing times, evidence retrieval success rates,
+    and confidence scores.
+    """
+    
+    def __init__(self):
+        """Initialize the performance tracker with empty metrics."""
+        self.metrics = {
+            "claims_processed": 0,
+            "evidence_retrieval_success_rate": [],
+            "processing_times": [],
+            "confidence_scores": [],
+            "source_types_used": {},
+            "temporal_relevance": []
+        }
+
+    def log_claim_processed(self):
+        """
+        Increment the counter for processed claims.
+        This should be called whenever a claim is processed successfully.
+        """
+        self.metrics["claims_processed"] += 1
+
+    def log_evidence_retrieval(self, success, sources_count):
+        """
+        Log the success or failure of evidence retrieval.
+        
+        Args:
+            success (bool): Whether evidence retrieval was successful
+            sources_count (dict): Count of evidence items by source type
+        """
+        # Ensure success is a boolean
+        success_value = 1 if success else 0
+        self.metrics["evidence_retrieval_success_rate"].append(success_value)
+
+        # Safely process source types
+        if isinstance(sources_count, dict):
+            for source_type, count in sources_count.items():
+                # Ensure source_type is a string and count is an integer
+                source_type = str(source_type)
+                try:
+                    count = int(count)
+                except (ValueError, TypeError):
+                    count = 1
+
+                # Update source types used
+                self.metrics["source_types_used"][source_type] = \
+                    self.metrics["source_types_used"].get(source_type, 0) + count
+
+    def log_processing_time(self, start_time):
+        """
+        Log the processing time for an operation.
+        
+        Args:
+            start_time (float): Start time obtained from time.time()
+        """
+        end_time = time.time()
+        processing_time = end_time - start_time
+        self.metrics["processing_times"].append(processing_time)
+
+    def log_confidence_score(self, score):
+        """
+        Log a confidence score.
+        
+        Args:
+            score (float): Confidence score between 0 and 1
+        """
+        # Ensure score is a float between 0 and 1
+        try:
+            score = float(score)
+            if 0 <= score <= 1:
+                self.metrics["confidence_scores"].append(score)
+        except (ValueError, TypeError):
+            logger.warning(f"Invalid confidence score: {score}")
+
+    def log_temporal_relevance(self, relevance_score):
+        """
+        Log a temporal relevance score.
+        
+        Args:
+            relevance_score (float): Temporal relevance score between 0 and 1
+        """
+        # Ensure relevance score is a float between 0 and 1
+        try:
+            relevance_score = float(relevance_score)
+            if 0 <= relevance_score <= 1:
+                self.metrics["temporal_relevance"].append(relevance_score)
+        except (ValueError, TypeError):
+            logger.warning(f"Invalid temporal relevance score: {relevance_score}")
+
+    def get_summary(self):
+        """
+        Get a summary of all performance metrics.
+        
+        Returns:
+            dict: Summary of performance metrics
+        """
+        # Safely calculate averages with error handling
+        def safe_avg(metric_list):
+            try:
+                return sum(metric_list) / max(len(metric_list), 1)
+            except (TypeError, ValueError):
+                return 0.0
+
+        return {
+            "claims_processed": self.metrics["claims_processed"],
+            "avg_evidence_retrieval_success_rate": safe_avg(self.metrics["evidence_retrieval_success_rate"]),
+            "avg_processing_time": safe_avg(self.metrics["processing_times"]),
+            "avg_confidence_score": safe_avg(self.metrics["confidence_scores"]),
+            "source_types_used": dict(self.metrics["source_types_used"]),
+            "avg_temporal_relevance": safe_avg(self.metrics["temporal_relevance"])
+        }
+
+    def reset(self):
+        """Reset all performance metrics."""
+        self.__init__()
+        logger.info("Performance metrics have been reset")
+        return "Performance metrics reset successfully"