update

.idea/workspace.xml

@@ -5,8 +5,12 @@
   </component>
   <component name="ChangeListManager">
     <list default="true" id="8e67814c-7f04-433c-ab7a-2b65a1106d4c" name="Changes" comment="">
+      <change afterPath="$PROJECT_DIR$/processors/ngram_analysis.py" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/visualization/ngram_visualizer.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/processors/bow_analysis.py" beforeDir="false" afterPath="$PROJECT_DIR$/processors/bow_analysis.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/ui/analysis_screen.py" beforeDir="false" afterPath="$PROJECT_DIR$/ui/analysis_screen.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/visualization/bow_visualizer.py" beforeDir="false" afterPath="$PROJECT_DIR$/visualization/bow_visualizer.py" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -56,7 +60,7 @@
       <option name="presentableId" value="Default" />
       <updated>1745170754325</updated>
       <workItem from="1745170755404" duration="245000" />
-      <workItem from="1745172030020" duration="4469000" />
+      <workItem from="1745172030020" duration="7284000" />
     </task>
     <servers />
   </component>

app.py

@@ -122,7 +122,7 @@ def create_app():
             status_message = gr.Markdown(visible=False)
             
             # Define a helper function to extract parameter values and run the analysis
-            def run_analysis(dataset, selected_analyses, bow_top):
+            def run_analysis(dataset, selected_analyses, bow_top, ngram_n, ngram_top):
                 try:
                     if not dataset or "entries" not in dataset or not dataset["entries"]:
                         return (
@@ -144,6 +144,8 @@ def create_app():
                     
                     parameters = {
                         "bow_top": bow_top,
+                        "ngram_n": ngram_n,
+                        "ngram_top": ngram_top
                     }
                     print("Running analysis with parameters:", parameters)
                     
@@ -324,7 +326,7 @@ def create_app():
             # Run analysis with proper parameters
             run_analysis_btn.click(
                 fn=run_analysis,
-                inputs=[dataset_state, analysis_options, bow_top_slider],
+                inputs=[dataset_state, analysis_options, bow_top_slider, ngram_n, ngram_top],
                 outputs=[
                     analysis_results_state,
                     analysis_output,

processors/ngram_analysis.py

@@ -0,0 +1,109 @@
+"""
+N-gram analysis for comparing text responses
+"""
+from sklearn.feature_extraction.text import CountVectorizer
+import numpy as np
+from collections import Counter
+import nltk
+from nltk.util import ngrams
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+
+
+def compare_ngrams(texts, model_names, n=2, top_n=25):
+    """
+    Compare n-gram representations across multiple texts.
+
+    Args:
+        texts (list): List of text responses to compare
+        model_names (list): Names of models corresponding to responses
+        n (int): Size of n-grams (1 for unigrams, 2 for bigrams, etc.)
+        top_n (int): Number of top n-grams to consider
+
+    Returns:
+        dict: N-gram analysis results
+    """
+    # Initialize the results dictionary
+    result = {
+        "models": model_names,
+        "ngram_size": n,
+        "important_ngrams": {},
+        "ngram_count_matrix": {},
+        "differential_ngrams": []
+    }
+
+    # Make sure we have texts to analyze
+    if not texts or len(texts) < 1:
+        return result
+
+    # Create n-gram representations using CountVectorizer
+    vectorizer = CountVectorizer(
+        ngram_range=(n, n),  # Use the specified n-gram size
+        max_features=1000,
+        stop_words='english'
+    )
+
+    X = vectorizer.fit_transform(texts)
+
+    # Get feature names (n-grams)
+    feature_names = vectorizer.get_feature_names_out()
+
+    # Create n-gram count matrix
+    ngram_counts = {}
+    for i, model in enumerate(model_names):
+        counts = X[i].toarray()[0]
+        ngram_counts[model] = {}
+
+        # Store n-gram frequencies for this model
+        for j, ngram in enumerate(feature_names):
+            if counts[j] > 0:  # Only store n-grams that appear
+                ngram_counts[model][ngram] = int(counts[j])
+
+                # Add to n-gram count matrix
+                if ngram not in result["ngram_count_matrix"]:
+                    result["ngram_count_matrix"][ngram] = {}
+                result["ngram_count_matrix"][ngram][model] = int(counts[j])
+
+    # Find important n-grams for each model
+    for model, ngram_freq in ngram_counts.items():
+        # Sort by frequency
+        sorted_ngrams = sorted(ngram_freq.items(), key=lambda x: x[1], reverse=True)
+
+        # Store top N n-grams
+        result["important_ngrams"][model] = [
+            {"ngram": ngram, "count": count}
+            for ngram, count in sorted_ngrams[:top_n]
+        ]
+
+    # Calculate differential n-grams (n-grams with biggest frequency difference between models)
+    if len(model_names) >= 2:
+        model1, model2 = model_names[0], model_names[1]
+
+        # Calculate differences
+        diff_scores = {}
+        for ngram in result["ngram_count_matrix"]:
+            count1 = result["ngram_count_matrix"][ngram].get(model1, 0)
+            count2 = result["ngram_count_matrix"][ngram].get(model2, 0)
+
+            # Absolute difference
+            diff_scores[ngram] = abs(count1 - count2)
+
+        # Sort by difference
+        sorted_diffs = sorted(diff_scores.items(), key=lambda x: x[1], reverse=True)
+        result["differential_ngrams"] = [ngram for ngram, _ in sorted_diffs[:top_n]]
+
+        # Calculate overlap statistics
+        model1_ngrams = set(ngram_counts.get(model1, {}).keys())
+        model2_ngrams = set(ngram_counts.get(model2, {}).keys())
+        common_ngrams = model1_ngrams.intersection(model2_ngrams)
+
+        # Initialize comparisons if needed
+        if "comparisons" not in result:
+            result["comparisons"] = {}
+
+        comparison_key = f"{model1} vs {model2}"
+        result["comparisons"][comparison_key] = {
+            "common_ngram_count": len(common_ngrams)
+        }
+
+    return result

ui/analysis_screen.py

@@ -10,6 +10,7 @@ from visualization.bow_visualizer import process_and_visualize_analysis
 from processors.bow_analysis import compare_bow
 # from processors.metrics import calculate_similarity
 # from processors.diff_highlighter import highlight_differences
+from processors.ngram_analysis import compare_ngrams
 
 def create_analysis_screen():
     """
@@ -105,56 +106,50 @@ def create_analysis_screen():
     
     # Return the bow_top_slider directly so app.py can access it
     # Note: Removed the visualization_container from return values since we'll pre-create it
-    return analysis_options, analysis_params, run_analysis_btn, analysis_output, bow_top_slider
+    return analysis_options, analysis_params, run_analysis_btn, analysis_output, bow_top_slider, ngram_n, ngram_top
 
+# function
 def process_analysis_request(dataset, selected_analyses, parameters):
     """
     Process the analysis request and run selected analyses
-    
-    Args:
-        dataset (dict): The dataset containing prompts and LLM responses
-        selected_analyses (list): List of selected analysis types
-        parameters (dict): Parameters for each analysis type
-        
-    Returns:
-        tuple: (analysis_results, analysis_output_display)
     """
     try:
         print(f"Processing analysis request with: {selected_analyses}")
         print(f"Parameters: {parameters}")
-        
+
         if not dataset or "entries" not in dataset or not dataset["entries"]:
-            return {}, gr.update(visible=True, value=json.dumps({"error": "No dataset provided or dataset is empty"}, indent=2))
-        
+            return {}, gr.update(visible=True,
+                                 value=json.dumps({"error": "No dataset provided or dataset is empty"}, indent=2))
+
         analysis_results = {"analyses": {}}
-        
+
         # Extract prompt and responses
         prompt = dataset["entries"][0]["prompt"]
         response_texts = [entry["response"] for entry in dataset["entries"]]
         model_names = [entry["model"] for entry in dataset["entries"]]
-        
+
         print(f"Analyzing prompt: '{prompt[:50]}...'")
         print(f"Models: {model_names}")
-        
+
         analysis_results["analyses"][prompt] = {}
-        
-        # Currently only implement Bag of Words since it's the most complete
+
+        # Run Bag of Words analysis if selected
         if "Bag of Words" in selected_analyses:
-            # Set a default value
-            top_words = 25
-            
-            # Try to get the parameter from the parameters dict
-            if parameters and isinstance(parameters, dict) and "bow_top" in parameters:
-                top_words = parameters["bow_top"]
-                
+            top_words = parameters.get("bow_top", 25)
             print(f"Running BOW analysis with top_words={top_words}")
-            
-            # Call the BOW comparison function
             bow_results = compare_bow(response_texts, model_names, top_words)
             analysis_results["analyses"][prompt]["bag_of_words"] = bow_results
-        
+
+        # Run N-gram analysis if selected
+        if "N-gram Analysis" in selected_analyses:
+            ngram_n = int(parameters.get("ngram_n", "2"))
+            ngram_top = parameters.get("ngram_top", 10)
+            print(f"Running N-gram analysis with n={ngram_n}, top_n={ngram_top}")
+            ngram_results = compare_ngrams(response_texts, model_names, ngram_n, ngram_top)
+            analysis_results["analyses"][prompt]["ngram_analysis"] = ngram_results
+
         print("Analysis complete - results:", analysis_results)
-        
+
         # Return results and update the output component
         return analysis_results, gr.update(visible=False, value=analysis_results)  # Hide the raw JSON
     except Exception as e:

visualization/bow_visualizer.py

@@ -7,6 +7,8 @@ from plotly.subplots import make_subplots
 import pandas as pd
 from difflib import SequenceMatcher
 
+from visualization.ngram_visualizer import create_ngram_visualization
+
 def create_bow_visualization(analysis_results):
     """
     Create visualizations for bag of words analysis results
@@ -122,78 +124,89 @@ def create_bow_visualization(analysis_results):
     
     return output_components
 
+
+# Then update the process_and_visualize_analysis function
 def process_and_visualize_analysis(analysis_results):
     """
     Process the analysis results and create visualization components
-    
+
     Args:
         analysis_results (dict): The analysis results
-        
+
     Returns:
         list: List of gradio components for visualization
     """
     try:
         print(f"Starting visualization of analysis results: {type(analysis_results)}")
         components = []
-        
+
         if not analysis_results or "analyses" not in analysis_results:
             print("Warning: Empty or invalid analysis results")
             components.append(gr.Markdown("No analysis results to visualize."))
             return components
-        
+
         # For each prompt in the analysis results
         for prompt, analyses in analysis_results.get("analyses", {}).items():
             print(f"Visualizing results for prompt: {prompt[:30]}...")
             components.append(gr.Markdown(f"## Analysis for Prompt:\n\"{prompt}\""))
-            
+
             # Check for Bag of Words analysis
             if "bag_of_words" in analyses:
                 print("Processing Bag of Words visualization")
                 components.append(gr.Markdown("### Bag of Words Analysis"))
                 bow_results = analyses["bag_of_words"]
-                
+
                 # Display models compared
                 if "models" in bow_results:
                     models = bow_results["models"]
                     components.append(gr.Markdown(f"**Models compared**: {', '.join(models)}"))
-                
+
                 # Display important words for each model
                 if "important_words" in bow_results:
                     components.append(gr.Markdown("#### Most Common Words by Model"))
-                    
+
                     for model, words in bow_results["important_words"].items():
                         print(f"Creating word list for model {model}")
                         word_list = [f"{item['word']} ({item['count']})" for item in words[:10]]
                         components.append(gr.Markdown(f"**{model}**: {', '.join(word_list)}"))
-                
+
                 # Add visualizations for word frequency differences
-                if "differential_words" in bow_results and "word_count_matrix" in bow_results and len(bow_results["models"]) >= 2:
+                if "differential_words" in bow_results and "word_count_matrix" in bow_results and len(
+                        bow_results["models"]) >= 2:
                     diff_words = bow_results["differential_words"]
                     word_matrix = bow_results["word_count_matrix"]
                     models = bow_results["models"]
-                    
+
                     if diff_words and word_matrix and len(diff_words) > 0:
                         components.append(gr.Markdown("### Words with Biggest Frequency Differences"))
-                        
+
                         # Create dataframe for plotting
                         model1, model2 = models[0], models[1]
                         diff_data = []
-                        
+
                         for word in diff_words[:10]:  # Limit to top 10 for readability
                             if word in word_matrix:
                                 counts = word_matrix[word]
                                 model1_count = counts.get(model1, 0)
                                 model2_count = counts.get(model2, 0)
-                                
+
                                 # Only include if there's a meaningful difference
                                 if abs(model1_count - model2_count) > 0:
                                     components.append(gr.Markdown(
                                         f"- **{word}**: {model1}: {model1_count}, {model2}: {model2_count}"
                                     ))
-        
+
+            # Check for N-gram analysis
+            if "ngram_analysis" in analyses:
+                print("Processing N-gram visualization")
+                # Use the dedicated n-gram visualization function
+                ngram_components = create_ngram_visualization(
+                    {"analyses": {prompt: {"ngram_analysis": analyses["ngram_analysis"]}}})
+                components.extend(ngram_components)
+
         if not components:
             components.append(gr.Markdown("No visualization components could be created from the analysis results."))
-        
+
         print(f"Visualization complete: generated {len(components)} components")
         return components
     except Exception as e:
@@ -201,3 +214,4 @@ def process_and_visualize_analysis(analysis_results):
         error_msg = f"Visualization error: {str(e)}\n{traceback.format_exc()}"
         print(error_msg)
         return [gr.Markdown(f"**Error during visualization:**\n\n```\n{error_msg}\n```")]
+

visualization/ngram_visualizer.py

@@ -0,0 +1,150 @@
+import gradio as gr
+import json
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+
+
+def create_ngram_visualization(analysis_results):
+    """
+    Create visualizations for n-gram analysis results
+
+    Args:
+        analysis_results (dict): Analysis results from the n-gram analysis
+
+    Returns:
+        list: List of gradio components with visualizations
+    """
+    output_components = []
+
+    # Check if we have valid results
+    if not analysis_results or "analyses" not in analysis_results:
+        return [gr.Markdown("No analysis results found.")]
+
+    # Process each prompt
+    for prompt, analyses in analysis_results["analyses"].items():
+        # Process N-gram analysis if available
+        if "ngram_analysis" in analyses:
+            ngram_results = analyses["ngram_analysis"]
+
+            # Show models being compared
+            models = ngram_results.get("models", [])
+            ngram_size = ngram_results.get("ngram_size", 2)
+            size_name = "Unigrams" if ngram_size == 1 else f"{ngram_size}-grams"
+
+            if len(models) >= 2:
+                output_components.append(
+                    gr.Markdown(f"### {size_name} Analysis: Comparing responses from {models[0]} and {models[1]}"))
+
+                # Get important n-grams for each model
+                important_ngrams = ngram_results.get("important_ngrams", {})
+
+                # Prepare data for plotting important n-grams
+                if important_ngrams:
+                    for model_name, ngrams in important_ngrams.items():
+                        df = pd.DataFrame(ngrams)
+
+                        # Create bar chart for top n-grams
+                        fig = px.bar(df, x='ngram', y='count',
+                                     title=f"Top {size_name} Used by {model_name}",
+                                     labels={'ngram': 'N-gram', 'count': 'Frequency'},
+                                     height=400)
+
+                        # Improve layout
+                        fig.update_layout(
+                            xaxis_title="N-gram",
+                            yaxis_title="Frequency",
+                            xaxis={'categoryorder': 'total descending'}
+                        )
+
+                        output_components.append(gr.Plot(value=fig))
+
+                # Visualize differential n-grams (n-grams with biggest frequency difference)
+                diff_ngrams = ngram_results.get("differential_ngrams", [])
+                ngram_matrix = ngram_results.get("ngram_count_matrix", {})
+
+                if diff_ngrams and ngram_matrix and len(diff_ngrams) > 0:
+                    output_components.append(gr.Markdown(f"### {size_name} with Biggest Frequency Differences"))
+
+                    # Create dataframe for plotting
+                    model1, model2 = models[0], models[1]
+                    diff_data = []
+
+                    for ngram in diff_ngrams[:15]:  # Limit to top 15 for readability
+                        if ngram in ngram_matrix:
+                            counts = ngram_matrix[ngram]
+                            diff_data.append({
+                                "ngram": ngram,
+                                model1: counts.get(model1, 0),
+                                model2: counts.get(model2, 0)
+                            })
+
+                    if diff_data:
+                        diff_df = pd.DataFrame(diff_data)
+
+                        # Create grouped bar chart
+                        fig = go.Figure()
+                        fig.add_trace(go.Bar(
+                            x=diff_df['ngram'],
+                            y=diff_df[model1],
+                            name=model1,
+                            marker_color='indianred'
+                        ))
+                        fig.add_trace(go.Bar(
+                            x=diff_df['ngram'],
+                            y=diff_df[model2],
+                            name=model2,
+                            marker_color='lightsalmon'
+                        ))
+
+                        fig.update_layout(
+                            title=f"{size_name} Frequency Comparison",
+                            xaxis_title="N-gram",
+                            yaxis_title="Frequency",
+                            barmode='group',
+                            height=500
+                        )
+
+                        output_components.append(gr.Plot(value=fig))
+
+                # Add similarity comparison if available
+                if "comparisons" in ngram_results:
+                    output_components.append(gr.Markdown("### N-gram Similarity Metrics"))
+                    comparison_key = f"{models[0]} vs {models[1]}"
+
+                    if comparison_key in ngram_results["comparisons"]:
+                        metrics = ngram_results["comparisons"][comparison_key]
+                        common_count = metrics.get("common_ngram_count", 0)
+
+                        metrics_text = f"""
+                        - **Common {size_name}**: {common_count} {size_name.lower()} appear in both responses
+                        """
+
+                        output_components.append(gr.Markdown(metrics_text))
+
+    # If no components were added other than header, show a message
+    if len(output_components) <= 1:
+        output_components.append(gr.Markdown(f"No detailed N-gram analysis found in results."))
+
+    return output_components
+
+
+def process_and_visualize_ngram_analysis(analysis_results):
+    """
+    Process the n-gram analysis results and create visualization components
+
+    Args:
+        analysis_results (dict): The analysis results
+
+    Returns:
+        list: List of gradio components for visualization
+    """
+    try:
+        print(f"Starting visualization of n-gram analysis results")
+        return create_ngram_visualization(analysis_results)
+    except Exception as e:
+        import traceback
+        error_msg = f"N-gram visualization error: {str(e)}\n{traceback.format_exc()}"
+        print(error_msg)
+        return [gr.Markdown(f"**Error during n-gram visualization:**\n\n```\n{error_msg}\n```")]