update

app.py

@@ -1,6 +1,6 @@
 import gradio as gr
 from ui.dataset_input import create_dataset_input, load_example_dataset
-from ui.analysis_screen import process_analysis_request
+from ui.analysis_screen import create_analysis_screen  # Import create_analysis_screen instead
 
 def create_app():
     """
@@ -22,7 +22,7 @@ def create_app():
             load_example_btn.click(
                 fn=load_example_dataset,
                 inputs=[example_dropdown],
-                outputs=[dataset_inputs]
+                outputs=[prompt, response1, model1, response2, model2]  # Update all field values
             )
 
             # Save dataset to state
@@ -39,12 +39,18 @@ def create_app():
         
         # Analysis Tab
         with gr.Tab("Analysis"):
-            analysis_options, analysis_params, run_analysis_btn, analysis_output = process_analysis_request()
+            # Use create_analysis_screen instead of process_analysis_request
+            analysis_options, analysis_params, run_analysis_btn, analysis_output = create_analysis_screen()
+
+            # Define a helper function to call process_analysis_request
+            def run_analysis(dataset, analysis_options, analysis_params):
+                from ui.analysis_screen import process_analysis_request
+                return process_analysis_request(dataset, analysis_options, analysis_params)
 
             # Run analysis
             run_analysis_btn.click(
-                fn=process_analysis_request,
-                inputs=[dataset_state, analysis_options, analysis_params],  # Pass required inputs
+                fn=run_analysis,
+                inputs=[dataset_state, analysis_options, analysis_params],
                 outputs=[analysis_results_state, analysis_output]
             )
     
@@ -52,4 +58,4 @@ def create_app():
 
 if __name__ == "__main__":
     app = create_app()
-    app.launch()
+    app.launch()

dataset/person-harris.txt

@@ -34,9 +34,7 @@ For the most current and detailed information, it's advisable to refer to recent
 \llm=
 ExaOne3.5
 
-\prompt=
-Tell me about the political views of Kamala Harris.
-\response=
+\response2=
 Kamala Harris, currently serving as the Vice President of the United States under Joe Biden, has held diverse political views throughout her career. Here's a broad overview:
 
 1. **Democratic Party**: As a senator from California and now Vice President, Harris is affiliated with the Democratic Party. She generally supports progressive policies within this framework.
@@ -56,5 +54,5 @@ Kamala Harris, currently serving as the Vice President of the United States unde
 5. **Foreign Policy**: As Vice President, Harris continues to develop her foreign policy views under the Biden administration's direction, but she has previously emphasized diplomacy over military intervention where possible. She's also focused on human rights and international women's issues in her global outlook.
 
 Remember, political views can evolve over time based on experiences, new information, or shifts within one's party. This is a general summary and may not reflect every stance she has taken throughout her career. For the most accurate and detailed understanding of her political views at any given moment, it's best to refer to her recent public statements and policy proposals.
-\llm=
+\llm2=
 Granite3.2

dataset/person-trump.txt

@@ -38,9 +38,7 @@ It's important to note that political views can evolve over time, and interpreta
 \llm=
 ExaOne3.5
 
-\prompt=
-Tell me about the political views of Donald Trump.
-\response=
+\response2=
 Donald Trump, who served as the 45th President of the United States from 2017 to 2021, is known for his populist and conservative political views. Here are some key aspects of his political stance:
 
 1. **Economy**: Trump generally advocates for low taxes, deregulation, and policies that favor business interests. He believes in reducing corporate income tax rates to stimulate economic growth and job creation. 
@@ -56,5 +54,5 @@ Donald Trump, who served as the 45th President of the United States from 2017 to
 6. **Healthcare**: Trump opposed the Affordable Care Act (Obamacare), advocating for its repeal and replace with a plan that would offer lower-cost, high-quality health insurance to Americans. His administration took steps to weaken certain aspects of Obamacare through executive actions.
 
 Remember, political views can be complex and multifaceted, evolving over time. This summary provides a broad overview but does not capture all nuances or changes in Trump's stated positions throughout his career.
-\llm=
+\llm2=
 Granite3.2

dataset/topic-foreign_policy.txt

@@ -53,9 +53,7 @@ These perspectives can vary widely among individuals within each ideological spe
 \llm=
 ExaOne3.5
 
-\prompt=
-tell me about the left and ring wing views in the united states on the topic of 'foreign policy'
-\response=
+\response2=
 In U.S. politics, "left" and "right" are broad terms that often translate to liberal (or progressive) and conservative viewpoints, respectively. These ideological differences significantly influence perspectives on foreign policy. Here’s a general overview of how left-leaning and right-leaning views might approach key aspects of U.S. foreign policy:
 
 ### Left-Wing/Progressive Views on Foreign Policy:
@@ -83,5 +81,5 @@ In U.S. politics, "left" and "right" are broad terms that often translate to lib
 5. **Border Control and Immigration:** Conservative foreign policy often includes strict border control measures and immigration policies aimed at preserving national security and cultural identity, which can sometimes manifest in more restrictive stances on refugee intake or legal immigration paths.
 
 It's crucial to remember that these are broad generalizations. Within both camps (left/progressive and right/conservative), there is significant diversity of opinion on specific foreign policy issues, with many nuanced positions and variations among politicians and policymakers.
-\llm=
+\llm2=
 Granite3.2

dataset/topic-the_economy.txt

@@ -56,9 +56,7 @@ These perspectives reflect broader ideological differences but also highlight ar
 \llm=
 ExaOne3.5
 
-\prompt=
-tell me about the left and ring wing views in the united states on the topic of 'the economy'
-\response=
+\response2=
 In the United States, political ideologies are typically categorized into a broad spectrum ranging from conservative (or right-wing) to liberal (or left-wing). Here's how these perspectives generally align regarding key economic issues:
 
 ### Right-Wing/Conservative Views on the Economy:
@@ -93,5 +91,5 @@ In the United States, political ideologies are typically categorized into a broa
 - **Inequality vs. Efficiency**: Liberals often prioritize addressing income and wealth disparities, while conservatives generally place a higher emphasis on economic efficiency and growth.
 
 These broad strokes do not capture the nuances within each political group; many individuals may hold views that blend or contradict these generalizations. Additionally, specific policy proposals can vary widely based on factors like regional economics, industry-specific interests, and personal experiences with economic conditions.
-\llm=
+\llm2=
 Granite3.2

processors/__init__.py

@@ -0,0 +1,8 @@
+# processors/__init__.py
+# Empty file to make the directory a Python package
+
+# ui/__init__.py
+# Empty file to make the directory a Python package
+
+# utils/__init__.py
+# Empty file to make the directory a Python package

processors/metrics.py

@@ -0,0 +1,107 @@
+"""
+Similarity metrics for text comparison
+"""
+import numpy as np
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+def calculate_cosine_similarity(text1, text2):
+    """
+    Calculate cosine similarity between two texts using TF-IDF vectorization.
+    
+    Args:
+        text1 (str): First text
+        text2 (str): Second text
+        
+    Returns:
+        float: Cosine similarity score between 0 and 1
+    """
+    # Create TF-IDF vectorizer
+    vectorizer = TfidfVectorizer()
+    
+    try:
+        # Transform texts into TF-IDF vectors
+        tfidf_matrix = vectorizer.fit_transform([text1, text2])
+        
+        # Calculate cosine similarity
+        similarity = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0]
+        return float(similarity)
+    except Exception as e:
+        print(f"Error calculating cosine similarity: {e}")
+        return 0.0
+
+def calculate_jaccard_similarity(text1, text2):
+    """
+    Calculate Jaccard similarity between two texts (word-level).
+    
+    Args:
+        text1 (str): First text
+        text2 (str): Second text
+        
+    Returns:
+        float: Jaccard similarity score between 0 and 1
+    """
+    # Convert to sets of words
+    words1 = set(text1.lower().split())
+    words2 = set(text2.lower().split())
+    
+    # Calculate Jaccard similarity
+    if not words1 and not words2:
+        return 1.0  # If both are empty, they're identical
+    
+    try:
+        intersection = len(words1.intersection(words2))
+        union = len(words1.union(words2))
+        return intersection / union
+    except Exception as e:
+        print(f"Error calculating Jaccard similarity: {e}")
+        return 0.0
+
+def calculate_semantic_similarity(text1, text2):
+    """
+    Calculate pseudo-semantic similarity by comparing word overlap patterns.
+    
+    This is a simplified approach that doesn't use embedding models like Word2Vec or BERT.
+    
+    Args:
+        text1 (str): First text
+        text2 (str): Second text
+        
+    Returns:
+        float: Semantic similarity score between 0 and 1
+    """
+    # For now, this is a weighted combination of cosine and Jaccard similarity
+    # In a real app, you'd use a proper semantic model
+    cosine = calculate_cosine_similarity(text1, text2)
+    jaccard = calculate_jaccard_similarity(text1, text2)
+    
+    # Weight more towards cosine similarity
+    return 0.7 * cosine + 0.3 * jaccard
+
+def calculate_similarity(text1, text2, metrics=None):
+    """
+    Calculate various similarity metrics between two texts.
+    
+    Args:
+        text1 (str): First text
+        text2 (str): Second text
+        metrics (list): List of metrics to calculate
+        
+    Returns:
+        dict: Dictionary of similarity scores
+    """
+    if metrics is None:
+        metrics = ["Cosine Similarity", "Jaccard Similarity", "Semantic Similarity"]
+    
+    results = {}
+    
+    if "Cosine Similarity" in metrics:
+        results["cosine_similarity"] = calculate_cosine_similarity(text1, text2)
+    
+    if "Jaccard Similarity" in metrics:
+        results["jaccard_similarity"] = calculate_jaccard_similarity(text1, text2)
+    
+    if "Semantic Similarity" in metrics:
+        results["semantic_similarity"] = calculate_semantic_similarity(text1, text2)
+    
+    return results

ui/__init__.py

@@ -0,0 +1,8 @@
+# processors/__init__.py
+# Empty file to make the directory a Python package
+
+# ui/__init__.py
+# Empty file to make the directory a Python package
+
+# utils/__init__.py
+# Empty file to make the directory a Python package

ui/analysis_screen.py

@@ -1,13 +1,14 @@
 import gradio as gr
 import json
 
-# Import analysis modules 
-#from processors.topic_modeling import extract_topics, compare_topics
-#from processors.ngram_analysis import compare_ngrams
-#from processors.bias_detection import compare_bias
+# Import analysis modules
+# Uncomment these when implemented
+# from processors.topic_modeling import extract_topics, compare_topics
+# from processors.ngram_analysis import compare_ngrams
+# from processors.bias_detection import compare_bias
 from processors.bow_analysis import compare_bow
-#from processors.metrics import calculate_similarity
-#from processors.diff_highlighter import highlight_differences
+# from processors.metrics import calculate_similarity
+# from processors.diff_highlighter import highlight_differences
 
 def create_analysis_screen():
     """
@@ -32,24 +33,22 @@ def create_analysis_screen():
                     "Difference Highlighting"
                 ],
                 value=[
-                    "Topic Modeling",
-                    "N-gram Analysis",
                     "Bag of Words",
-                    "Similarity Metrics"
                 ],
                 label="Select Analyses to Run"
             )
         
         # Parameters for each analysis type
-        with gr.Accordion("Analysis Parameters", open=False) as analysis_params:
+        analysis_params = gr.Group()
+        with analysis_params:
             # Topic modeling parameters
-            with gr.Group():
+            with gr.Group(visible=False) as topic_params:
                 gr.Markdown("### Topic Modeling Parameters")
                 topic_count = gr.Slider(minimum=2, maximum=10, value=3, step=1, 
                                        label="Number of Topics")
             
             # N-gram parameters
-            with gr.Group():
+            with gr.Group(visible=False) as ngram_params:
                 gr.Markdown("### N-gram Parameters")
                 ngram_n = gr.Radio(choices=["1", "2", "3"], value="2", 
                                   label="N-gram Size")
@@ -57,7 +56,7 @@ def create_analysis_screen():
                                      label="Top N-grams to Display")
             
             # Bias detection parameters
-            with gr.Group():
+            with gr.Group(visible=False) as bias_params:
                 gr.Markdown("### Bias Detection Parameters")
                 bias_methods = gr.CheckboxGroup(
                     choices=["Sentiment Analysis", "Partisan Leaning", "Framing Analysis"],
@@ -66,19 +65,36 @@ def create_analysis_screen():
                 )
             
             # Bag of Words parameters  
-            with gr.Group():
+            with gr.Group() as bow_params:
                 gr.Markdown("### Bag of Words Parameters")
                 bow_top = gr.Slider(minimum=10, maximum=100, value=25, step=5, 
                                    label="Top Words to Compare")
                 
             # Similarity metrics parameters
-            with gr.Group():
+            with gr.Group(visible=False) as similarity_params:
                 gr.Markdown("### Similarity Metrics Parameters")
                 similarity_metrics = gr.CheckboxGroup(
                     choices=["Cosine Similarity", "Jaccard Similarity", "Semantic Similarity"],
                     value=["Cosine Similarity", "Semantic Similarity"],
                     label="Similarity Metrics to Calculate"
                 )
+                
+            # Function to update parameter visibility based on selected analyses
+            def update_params_visibility(selected):
+                return {
+                    topic_params: gr.update(visible="Topic Modeling" in selected),
+                    ngram_params: gr.update(visible="N-gram Analysis" in selected),
+                    bias_params: gr.update(visible="Bias Detection" in selected),
+                    bow_params: gr.update(visible="Bag of Words" in selected),
+                    similarity_params: gr.update(visible="Similarity Metrics" in selected)
+                }
+                
+            # Set up event handler for analysis selection
+            analysis_options.change(
+                fn=update_params_visibility,
+                inputs=[analysis_options],
+                outputs=[topic_params, ngram_params, bias_params, bow_params, similarity_params]
+            )
         
         # Run analysis button
         run_analysis_btn = gr.Button("Run Analysis", variant="primary", size="large")
@@ -105,76 +121,23 @@ def process_analysis_request(dataset, selected_analyses, parameters):
     
     analysis_results = {"analyses": {}}
     
-    # Group responses by prompt
-    prompts = {}
-    for entry in dataset["entries"]:
-        if entry["prompt"] not in prompts:
-            prompts[entry["prompt"]] = []
-        prompts[entry["prompt"]].append({
-            "model": entry["model"],
-            "response": entry["response"]
-        })
+    # Extract prompt and responses
+    prompt = dataset["entries"][0]["prompt"]
+    response_texts = [entry["response"] for entry in dataset["entries"]]
+    model_names = [entry["model"] for entry in dataset["entries"]]
     
-    # Run selected analyses for each prompt
-    for prompt, responses in prompts.items():
-        analysis_results["analyses"][prompt] = {}
-        
-        # Extract just the text responses and model names
-        response_texts = [r["response"] for r in responses]
-        model_names = [r["model"] for r in responses]
-        
-        # Run Topic Modeling
-        if "Topic Modeling" in selected_analyses:
-            num_topics = parameters.get("topic_count", 3)
-            topic_results = compare_topics(response_texts, model_names, num_topics)
-            analysis_results["analyses"][prompt]["topic_modeling"] = topic_results
-        
-        # Run N-gram Analysis
-        if "N-gram Analysis" in selected_analyses:
-            n = int(parameters.get("ngram_n", 2))
-            top_n = parameters.get("ngram_top", 10)
-            ngram_results = compare_ngrams(response_texts, model_names, n, top_n)
-            analysis_results["analyses"][prompt]["ngram_analysis"] = ngram_results
-        
-        # Run Bias Detection
-        if "Bias Detection" in selected_analyses:
-            bias_methods = parameters.get("bias_methods", ["Sentiment Analysis", "Partisan Leaning"])
-            bias_results = compare_bias(response_texts, model_names, bias_methods)
-            analysis_results["analyses"][prompt]["bias_detection"] = bias_results
-        
-        # Run Bag of Words Analysis
-        if "Bag of Words" in selected_analyses:
-            top_words = parameters.get("bow_top", 25)
-            bow_results = compare_bow(response_texts, model_names, top_words)
-            analysis_results["analyses"][prompt]["bag_of_words"] = bow_results
-        
-        # Run Similarity Metrics
-        if "Similarity Metrics" in selected_analyses:
-            metrics = parameters.get("similarity_metrics", ["Cosine Similarity"])
-            similarity_results = {}
-            
-            # Calculate pairwise similarities
-            for i in range(len(responses)):
-                for j in range(i+1, len(responses)):
-                    model_pair = f"{model_names[i]} vs {model_names[j]}"
-                    similarity_results[model_pair] = calculate_similarity(
-                        response_texts[i], response_texts[j], metrics
-                    )
-            
-            analysis_results["analyses"][prompt]["similarity_metrics"] = similarity_results
+    analysis_results["analyses"][prompt] = {}
+    
+    # Currently only implement Bag of Words since it's the most complete
+    if "Bag of Words" in selected_analyses:
+        top_words = 25  # Default value if not in parameters
+        if parameters and hasattr(parameters, "get"):
+            if "bow_top" in parameters:
+                top_words = parameters["bow_top"]
         
-        # Run Difference Highlighting
-        if "Difference Highlighting" in selected_analyses:
-            diff_results = {}
-            
-            # Calculate pairwise differences
-            for i in range(len(responses)):
-                for j in range(i+1, len(responses)):
-                    model_pair = f"{model_names[i]} vs {model_names[j]}"
-                    diff_results[model_pair] = highlight_differences(
-                        response_texts[i], response_texts[j]
-                    )
-            
-            analysis_results["analyses"][prompt]["difference_highlighting"] = diff_results
+        bow_results = compare_bow(response_texts, model_names, top_words)
+        analysis_results["analyses"][prompt]["bag_of_words"] = bow_results
+    
+    # Future implementations for other analysis types would go here
     
     return analysis_results, gr.update(visible=True, value=json.dumps(analysis_results, indent=2))

ui/dataset_input.py

@@ -7,11 +7,18 @@ def create_dataset_input():
     Create the dataset input interface with prompt, response, and model fields. 
 
     Returns:
-        tuple: (dataset_inputs, example_dropdown, load_example_btn, create_btn)
+        tuple: (dataset_inputs, example_dropdown, load_example_btn, create_btn, prompt, response1, model1, response2, model2)
     """
     # Get built-in text datasets
     text_datasets_dir = os.path.join("dataset")
-    text_datasets = [file.name for file in os.scandir(text_datasets_dir) if file.is_file() and file.name.endswith(".txt")]
+    # Check if directory exists
+    if os.path.exists(text_datasets_dir):
+        text_datasets = [file.name for file in os.scandir(text_datasets_dir) 
+                        if file.is_file() and file.name.endswith(".txt")]
+    else:
+        # Create directory if it doesn't exist
+        os.makedirs(text_datasets_dir, exist_ok=True)
+        text_datasets = []
 
     with gr.Column() as dataset_inputs:
         gr.Markdown("## LLM Response Comparison Dataset")
@@ -58,10 +65,17 @@ def load_example_dataset(file_name):
     """
     file_path = os.path.join("dataset", file_name)
     if os.path.exists(file_path):
-        return load_text_file(file_path)
+        dataset = load_text_file(file_path)
+        return (
+            dataset.get("prompt", ""),
+            dataset.get("response1", ""),
+            dataset.get("model1", ""),
+            dataset.get("response2", ""),
+            dataset.get("model2", "")
+        )
     else:
         print(f"File not found: {file_path}")
-        return {"prompt": "", "response1": "", "model1": "", "response2": "", "model2": ""}
+        return "", "", "", "", ""  # Return empty strings for all fields
 
 
 def create_user_dataset(prompt, response1, model1, response2, model2):

utils/__init__.py

@@ -0,0 +1,8 @@
+# processors/__init__.py
+# Empty file to make the directory a Python package
+
+# ui/__init__.py
+# Empty file to make the directory a Python package
+
+# utils/__init__.py
+# Empty file to make the directory a Python package

visualizers/__init__.py

@@ -0,0 +1,8 @@
+# processors/__init__.py
+# Empty file to make the directory a Python package
+
+# ui/__init__.py
+# Empty file to make the directory a Python package
+
+# utils/__init__.py
+# Empty file to make the directory a Python package