Spaces:
Sleeping
Sleeping
Ryan
commited on
Commit
·
7731b47
1
Parent(s):
6334788
update
Browse files- app.py +79 -4
- bert_classifier_function.py +45 -0
- processors/roberta_processor.py +246 -0
- processors/text_classifiers.py +81 -0
- requirements.txt +2 -0
- visualization/__init__.py +3 -1
- visualization/roberta_visualizer.py +240 -0
app.py
CHANGED
@@ -1,7 +1,9 @@
|
|
1 |
import gradio as gr
|
2 |
from ui.dataset_input import create_dataset_input, load_example_dataset
|
3 |
from ui.analysis_screen import create_analysis_screen, process_analysis_request
|
|
|
4 |
from visualization.bow_visualizer import process_and_visualize_analysis
|
|
|
5 |
import nltk
|
6 |
import os
|
7 |
import json
|
@@ -51,7 +53,7 @@ def download_nltk_resources():
|
|
51 |
|
52 |
def create_app():
|
53 |
"""
|
54 |
-
Create a streamlined Gradio app for dataset input and
|
55 |
|
56 |
Returns:
|
57 |
gr.Blocks: The Gradio application
|
@@ -60,6 +62,7 @@ def create_app():
|
|
60 |
# Application state to share data between tabs
|
61 |
dataset_state = gr.State({})
|
62 |
analysis_results_state = gr.State({})
|
|
|
63 |
|
64 |
# Dataset Input Tab
|
65 |
with gr.Tab("Dataset Input"):
|
@@ -218,7 +221,7 @@ def create_app():
|
|
218 |
gr.update(visible=False),
|
219 |
gr.update(visible=False),
|
220 |
True,
|
221 |
-
gr.update(visible=True, value=f"ℹ️ **{analyses['message']}**")
|
222 |
)
|
223 |
|
224 |
# Process based on the selected analysis type
|
@@ -539,8 +542,7 @@ def create_app():
|
|
539 |
gr.update(visible=False),
|
540 |
gr.update(visible=False),
|
541 |
gr.update(visible=False),
|
542 |
-
|
543 |
-
True,
|
544 |
gr.update(visible=True, value="❌ **No visualization data found.** Make sure to select a valid analysis option.")
|
545 |
)
|
546 |
|
@@ -583,7 +585,80 @@ def create_app():
|
|
583 |
True, # status_message_visible
|
584 |
gr.update(visible=True, value=f"❌ **Error during analysis:**\n\n```\n{str(e)}\n```") # status_message
|
585 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
586 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
587 |
# Add a Summary tab
|
588 |
with gr.Tab("Summary"):
|
589 |
gr.Markdown("## Analysis Summaries")
|
|
|
1 |
import gradio as gr
|
2 |
from ui.dataset_input import create_dataset_input, load_example_dataset
|
3 |
from ui.analysis_screen import create_analysis_screen, process_analysis_request
|
4 |
+
from ui.roberta_screen import create_roberta_screen, process_roberta_request
|
5 |
from visualization.bow_visualizer import process_and_visualize_analysis
|
6 |
+
from visualization.roberta_visualizer import process_and_visualize_sentiment_analysis
|
7 |
import nltk
|
8 |
import os
|
9 |
import json
|
|
|
53 |
|
54 |
def create_app():
|
55 |
"""
|
56 |
+
Create a streamlined Gradio app for dataset input and analysis.
|
57 |
|
58 |
Returns:
|
59 |
gr.Blocks: The Gradio application
|
|
|
62 |
# Application state to share data between tabs
|
63 |
dataset_state = gr.State({})
|
64 |
analysis_results_state = gr.State({})
|
65 |
+
roberta_results_state = gr.State({})
|
66 |
|
67 |
# Dataset Input Tab
|
68 |
with gr.Tab("Dataset Input"):
|
|
|
221 |
gr.update(visible=False),
|
222 |
gr.update(visible=False),
|
223 |
True,
|
224 |
+
gr.update(visible=True, value=f"ℹ️ **{analyses['message']}**") # status_message
|
225 |
)
|
226 |
|
227 |
# Process based on the selected analysis type
|
|
|
542 |
gr.update(visible=False),
|
543 |
gr.update(visible=False),
|
544 |
gr.update(visible=False),
|
545 |
+
True, # status_message_visible
|
|
|
546 |
gr.update(visible=True, value="❌ **No visualization data found.** Make sure to select a valid analysis option.")
|
547 |
)
|
548 |
|
|
|
585 |
True, # status_message_visible
|
586 |
gr.update(visible=True, value=f"❌ **Error during analysis:**\n\n```\n{str(e)}\n```") # status_message
|
587 |
)
|
588 |
+
|
589 |
+
# RoBERTa Sentiment Analysis Tab (NEW)
|
590 |
+
with gr.Tab("RoBERTa Sentiment"):
|
591 |
+
# Create the RoBERTa analysis UI components
|
592 |
+
run_roberta_btn, roberta_output, sentence_level, visualization_style, visualization_container, roberta_status = create_roberta_screen()
|
593 |
+
|
594 |
+
# Container for visualization results
|
595 |
+
with gr.Column() as roberta_viz_container:
|
596 |
+
roberta_viz_components = []
|
597 |
+
|
598 |
+
# Function to run RoBERTa sentiment analysis
|
599 |
+
def run_roberta_analysis(dataset, sentence_level, visualization_style):
|
600 |
+
try:
|
601 |
+
if not dataset or "entries" not in dataset or not dataset["entries"]:
|
602 |
+
return (
|
603 |
+
{}, # roberta_results_state
|
604 |
+
True, # status_message_visible
|
605 |
+
gr.update(visible=True, value="❌ **Error:** No dataset loaded. Please create or load a dataset first."), # status_message
|
606 |
+
False, # roberta_output visibility
|
607 |
+
[] # empty visualization components
|
608 |
+
)
|
609 |
+
|
610 |
+
print(f"Running RoBERTa sentiment analysis with sentence-level={sentence_level}, style={visualization_style}")
|
611 |
+
|
612 |
+
# Process the analysis request
|
613 |
+
roberta_results = process_roberta_request(dataset, sentence_level, visualization_style)
|
614 |
+
|
615 |
+
# Check if we have results
|
616 |
+
if "error" in roberta_results:
|
617 |
+
return (
|
618 |
+
roberta_results, # Store in state anyway for debugging
|
619 |
+
True, # status_message_visible
|
620 |
+
gr.update(visible=True, value=f"❌ **Error:** {roberta_results['error']}"), # status_message
|
621 |
+
False, # Hide raw output
|
622 |
+
[] # empty visualization components
|
623 |
+
)
|
624 |
+
|
625 |
+
# Create visualization components
|
626 |
+
viz_components = process_and_visualize_sentiment_analysis(roberta_results)
|
627 |
+
|
628 |
+
return (
|
629 |
+
roberta_results, # roberta_results_state
|
630 |
+
False, # status_message_visible
|
631 |
+
gr.update(visible=False), # status_message
|
632 |
+
False, # roberta_output visibility (hide raw output)
|
633 |
+
viz_components # visualization components
|
634 |
+
)
|
635 |
+
|
636 |
+
except Exception as e:
|
637 |
+
import traceback
|
638 |
+
error_msg = f"Error in RoBERTa analysis: {str(e)}\n{traceback.format_exc()}"
|
639 |
+
print(error_msg)
|
640 |
|
641 |
+
return (
|
642 |
+
{"error": error_msg}, # roberta_results_state
|
643 |
+
True, # status_message_visible
|
644 |
+
gr.update(visible=True, value=f"❌ **Error during RoBERTa analysis:**\n\n```\n{str(e)}\n```"), # status_message
|
645 |
+
False, # Hide raw output
|
646 |
+
[] # empty visualization components
|
647 |
+
)
|
648 |
+
|
649 |
+
# Connect the run button to the analysis function
|
650 |
+
run_roberta_btn.click(
|
651 |
+
fn=run_roberta_analysis,
|
652 |
+
inputs=[dataset_state, sentence_level, visualization_style],
|
653 |
+
outputs=[
|
654 |
+
roberta_results_state,
|
655 |
+
gr.Checkbox(visible=False, value=False), # Hidden checkbox for status visibility
|
656 |
+
roberta_status,
|
657 |
+
roberta_output,
|
658 |
+
roberta_viz_container
|
659 |
+
]
|
660 |
+
)
|
661 |
+
|
662 |
# Add a Summary tab
|
663 |
with gr.Tab("Summary"):
|
664 |
gr.Markdown("## Analysis Summaries")
|
bert_classifier_function.py
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
def classify_with_transformer(text, task="sentiment", model_name="distilbert-base-uncased"):
|
2 |
+
"""
|
3 |
+
Classify text using a pre-trained transformer model (BERT, RoBERTa, etc.)
|
4 |
+
|
5 |
+
Args:
|
6 |
+
text (str): Text to analyze
|
7 |
+
task (str): Classification task ('sentiment', 'emotion', etc.)
|
8 |
+
model_name (str): Name of the pre-trained model to use
|
9 |
+
|
10 |
+
Returns:
|
11 |
+
dict: Classification results with labels and scores
|
12 |
+
"""
|
13 |
+
try:
|
14 |
+
from transformers import pipeline
|
15 |
+
|
16 |
+
# Map tasks to appropriate models if not specified
|
17 |
+
task_model_map = {
|
18 |
+
"sentiment": "distilbert-base-uncased-finetuned-sst-2-english",
|
19 |
+
"emotion": "j-hartmann/emotion-english-distilroberta-base",
|
20 |
+
"toxicity": "unitary/toxic-bert"
|
21 |
+
}
|
22 |
+
|
23 |
+
# Use mapped model if using default and task is in the map
|
24 |
+
if model_name == "distilbert-base-uncased" and task in task_model_map:
|
25 |
+
model_to_use = task_model_map[task]
|
26 |
+
else:
|
27 |
+
model_to_use = model_name
|
28 |
+
|
29 |
+
# Initialize the classification pipeline
|
30 |
+
classifier = pipeline(task, model=model_to_use)
|
31 |
+
|
32 |
+
# Get classification results
|
33 |
+
results = classifier(text)
|
34 |
+
|
35 |
+
# Format results based on return type (list or dict)
|
36 |
+
if isinstance(results, list):
|
37 |
+
if len(results) == 1:
|
38 |
+
return results[0]
|
39 |
+
return results
|
40 |
+
return results
|
41 |
+
|
42 |
+
except ImportError:
|
43 |
+
return {"error": "Required packages not installed. Please install transformers and torch."}
|
44 |
+
except Exception as e:
|
45 |
+
return {"error": f"Classification failed: {str(e)}"}
|
processors/roberta_processor.py
ADDED
@@ -0,0 +1,246 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
RoBERTa-based sentiment analysis for comparing LLM responses
|
3 |
+
"""
|
4 |
+
import torch
|
5 |
+
import numpy as np
|
6 |
+
from transformers import RobertaTokenizer, RobertaForSequenceClassification
|
7 |
+
import nltk
|
8 |
+
from nltk.tokenize import sent_tokenize
|
9 |
+
|
10 |
+
# Global variables to store models once loaded
|
11 |
+
ROBERTA_TOKENIZER = None
|
12 |
+
ROBERTA_MODEL = None
|
13 |
+
|
14 |
+
def ensure_nltk_resources():
|
15 |
+
"""Make sure necessary NLTK resources are downloaded"""
|
16 |
+
try:
|
17 |
+
nltk.data.find('tokenizers/punkt')
|
18 |
+
except LookupError:
|
19 |
+
nltk.download('punkt', quiet=True)
|
20 |
+
|
21 |
+
def load_roberta_model():
|
22 |
+
"""
|
23 |
+
Load the RoBERTa model and tokenizer for sentiment analysis
|
24 |
+
|
25 |
+
Returns:
|
26 |
+
tuple: (tokenizer, model) for RoBERTa sentiment analysis
|
27 |
+
"""
|
28 |
+
global ROBERTA_TOKENIZER, ROBERTA_MODEL
|
29 |
+
|
30 |
+
# Return cached model if already loaded
|
31 |
+
if ROBERTA_TOKENIZER is not None and ROBERTA_MODEL is not None:
|
32 |
+
return ROBERTA_TOKENIZER, ROBERTA_MODEL
|
33 |
+
|
34 |
+
print("Loading RoBERTa model and tokenizer...")
|
35 |
+
|
36 |
+
try:
|
37 |
+
# Load tokenizer and model for sentiment analysis
|
38 |
+
ROBERTA_TOKENIZER = RobertaTokenizer.from_pretrained('roberta-base')
|
39 |
+
ROBERTA_MODEL = RobertaForSequenceClassification.from_pretrained('roberta-large-mnli')
|
40 |
+
|
41 |
+
return ROBERTA_TOKENIZER, ROBERTA_MODEL
|
42 |
+
except Exception as e:
|
43 |
+
print(f"Error loading RoBERTa model: {str(e)}")
|
44 |
+
# Return None values if loading fails
|
45 |
+
return None, None
|
46 |
+
|
47 |
+
def analyze_sentiment_roberta(text):
|
48 |
+
"""
|
49 |
+
Analyze sentiment using RoBERTa model
|
50 |
+
|
51 |
+
Args:
|
52 |
+
text (str): Text to analyze
|
53 |
+
|
54 |
+
Returns:
|
55 |
+
dict: Sentiment analysis results with label and scores
|
56 |
+
"""
|
57 |
+
ensure_nltk_resources()
|
58 |
+
|
59 |
+
# Handle empty text
|
60 |
+
if not text or not text.strip():
|
61 |
+
return {
|
62 |
+
"label": "neutral",
|
63 |
+
"scores": {
|
64 |
+
"contradiction": 0.33,
|
65 |
+
"neutral": 0.34,
|
66 |
+
"entailment": 0.33
|
67 |
+
},
|
68 |
+
"sentiment_score": 0.0,
|
69 |
+
"sentence_scores": []
|
70 |
+
}
|
71 |
+
|
72 |
+
# Load model
|
73 |
+
tokenizer, model = load_roberta_model()
|
74 |
+
if tokenizer is None or model is None:
|
75 |
+
return {
|
76 |
+
"error": "Failed to load RoBERTa model",
|
77 |
+
"label": "neutral",
|
78 |
+
"scores": {
|
79 |
+
"contradiction": 0.33,
|
80 |
+
"neutral": 0.34,
|
81 |
+
"entailment": 0.33
|
82 |
+
},
|
83 |
+
"sentiment_score": 0.0
|
84 |
+
}
|
85 |
+
|
86 |
+
try:
|
87 |
+
# Set device
|
88 |
+
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
89 |
+
model.to(device)
|
90 |
+
|
91 |
+
# Process the whole text
|
92 |
+
encoded_text = tokenizer(text, return_tensors='pt', truncation=True, max_length=512)
|
93 |
+
encoded_text = {k: v.to(device) for k, v in encoded_text.items()}
|
94 |
+
|
95 |
+
with torch.no_grad():
|
96 |
+
outputs = model(**encoded_text)
|
97 |
+
predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
|
98 |
+
|
99 |
+
# Get prediction
|
100 |
+
contradiction_score = predictions[0, 0].item()
|
101 |
+
neutral_score = predictions[0, 1].item()
|
102 |
+
entailment_score = predictions[0, 2].item()
|
103 |
+
|
104 |
+
# Map to sentiment
|
105 |
+
# contradiction = negative, entailment = positive, with a scale
|
106 |
+
sentiment_score = (entailment_score - contradiction_score) * 2 # Scale from -2 to 2
|
107 |
+
|
108 |
+
# Determine sentiment label
|
109 |
+
if sentiment_score > 0.5:
|
110 |
+
label = "positive"
|
111 |
+
elif sentiment_score < -0.5:
|
112 |
+
label = "negative"
|
113 |
+
else:
|
114 |
+
label = "neutral"
|
115 |
+
|
116 |
+
# Analyze individual sentences if text is long enough
|
117 |
+
sentences = sent_tokenize(text)
|
118 |
+
sentence_scores = []
|
119 |
+
|
120 |
+
# Only process sentences if there are more than one and text is substantial
|
121 |
+
if len(sentences) > 1 and len(text) > 100:
|
122 |
+
for sentence in sentences:
|
123 |
+
if len(sentence.split()) >= 3: # Only analyze meaningful sentences
|
124 |
+
encoded_sentence = tokenizer(sentence, return_tensors='pt', truncation=True)
|
125 |
+
encoded_sentence = {k: v.to(device) for k, v in encoded_sentence.items()}
|
126 |
+
|
127 |
+
with torch.no_grad():
|
128 |
+
outputs = model(**encoded_sentence)
|
129 |
+
predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
|
130 |
+
|
131 |
+
# Calculate sentence sentiment score
|
132 |
+
sent_contradiction = predictions[0, 0].item()
|
133 |
+
sent_neutral = predictions[0, 1].item()
|
134 |
+
sent_entailment = predictions[0, 2].item()
|
135 |
+
sent_score = (sent_entailment - sent_contradiction) * 2
|
136 |
+
|
137 |
+
# Determine sentiment label for this sentence
|
138 |
+
if sent_score > 0.5:
|
139 |
+
sent_label = "positive"
|
140 |
+
elif sent_score < -0.5:
|
141 |
+
sent_label = "negative"
|
142 |
+
else:
|
143 |
+
sent_label = "neutral"
|
144 |
+
|
145 |
+
sentence_scores.append({
|
146 |
+
"text": sentence,
|
147 |
+
"score": sent_score,
|
148 |
+
"label": sent_label,
|
149 |
+
"scores": {
|
150 |
+
"contradiction": sent_contradiction,
|
151 |
+
"neutral": sent_neutral,
|
152 |
+
"entailment": sent_entailment
|
153 |
+
}
|
154 |
+
})
|
155 |
+
|
156 |
+
return {
|
157 |
+
"label": label,
|
158 |
+
"scores": {
|
159 |
+
"contradiction": contradiction_score,
|
160 |
+
"neutral": neutral_score,
|
161 |
+
"entailment": entailment_score
|
162 |
+
},
|
163 |
+
"sentiment_score": sentiment_score,
|
164 |
+
"sentence_scores": sentence_scores
|
165 |
+
}
|
166 |
+
|
167 |
+
except Exception as e:
|
168 |
+
import traceback
|
169 |
+
print(f"Error analyzing sentiment with RoBERTa: {str(e)}")
|
170 |
+
print(traceback.format_exc())
|
171 |
+
|
172 |
+
return {
|
173 |
+
"error": str(e),
|
174 |
+
"label": "neutral",
|
175 |
+
"scores": {
|
176 |
+
"contradiction": 0.33,
|
177 |
+
"neutral": 0.34,
|
178 |
+
"entailment": 0.33
|
179 |
+
},
|
180 |
+
"sentiment_score": 0.0
|
181 |
+
}
|
182 |
+
|
183 |
+
def compare_sentiment_roberta(texts, model_names=None):
|
184 |
+
"""
|
185 |
+
Compare sentiment between two texts using RoBERTa
|
186 |
+
|
187 |
+
Args:
|
188 |
+
texts (list): List of texts to compare
|
189 |
+
model_names (list): Names of models corresponding to texts
|
190 |
+
|
191 |
+
Returns:
|
192 |
+
dict: Comparative sentiment analysis results
|
193 |
+
"""
|
194 |
+
# Set default model names if not provided
|
195 |
+
if model_names is None or len(model_names) < 2:
|
196 |
+
model_names = ["Model 1", "Model 2"]
|
197 |
+
|
198 |
+
# Handle case with fewer than 2 texts
|
199 |
+
if len(texts) < 2:
|
200 |
+
return {
|
201 |
+
"error": "Need at least 2 texts to compare",
|
202 |
+
"models": model_names[:len(texts)]
|
203 |
+
}
|
204 |
+
|
205 |
+
# Get sentiment analysis for each text
|
206 |
+
sentiment_results = []
|
207 |
+
for text in texts:
|
208 |
+
sentiment_results.append(analyze_sentiment_roberta(text))
|
209 |
+
|
210 |
+
# Create result dictionary
|
211 |
+
result = {
|
212 |
+
"models": model_names[:len(texts)],
|
213 |
+
"sentiment_analysis": {}
|
214 |
+
}
|
215 |
+
|
216 |
+
# Add individual model results
|
217 |
+
for i, model_name in enumerate(model_names[:len(texts)]):
|
218 |
+
result["sentiment_analysis"][model_name] = sentiment_results[i]
|
219 |
+
|
220 |
+
# Compare sentiment scores
|
221 |
+
if len(sentiment_results) >= 2:
|
222 |
+
model1_name, model2_name = model_names[0], model_names[1]
|
223 |
+
score1 = sentiment_results[0]["sentiment_score"]
|
224 |
+
score2 = sentiment_results[1]["sentiment_score"]
|
225 |
+
|
226 |
+
# Calculate difference and determine which is more positive/negative
|
227 |
+
difference = abs(score1 - score2)
|
228 |
+
|
229 |
+
result["comparison"] = {
|
230 |
+
"sentiment_difference": difference,
|
231 |
+
"significant_difference": difference > 0.5, # Threshold for significant difference
|
232 |
+
}
|
233 |
+
|
234 |
+
if score1 > score2:
|
235 |
+
result["comparison"]["more_positive"] = model1_name
|
236 |
+
result["comparison"]["more_negative"] = model2_name
|
237 |
+
result["comparison"]["difference_direction"] = f"{model1_name} is more positive than {model2_name}"
|
238 |
+
elif score2 > score1:
|
239 |
+
result["comparison"]["more_positive"] = model2_name
|
240 |
+
result["comparison"]["more_negative"] = model1_name
|
241 |
+
result["comparison"]["difference_direction"] = f"{model2_name} is more positive than {model1_name}"
|
242 |
+
else:
|
243 |
+
result["comparison"]["equal_sentiment"] = True
|
244 |
+
result["comparison"]["difference_direction"] = f"{model1_name} and {model2_name} have similar sentiment"
|
245 |
+
|
246 |
+
return result
|
processors/text_classifiers.py
CHANGED
@@ -149,4 +149,85 @@ def compare_classifications(text1, text2):
|
|
149 |
if not results:
|
150 |
results["Summary"] = "Both responses have similar writing characteristics"
|
151 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
152 |
return results
|
|
|
149 |
if not results:
|
150 |
results["Summary"] = "Both responses have similar writing characteristics"
|
151 |
|
152 |
+
return results
|
153 |
+
|
154 |
+
def classify_with_roberta(text, task="sentiment", model_name=None):
|
155 |
+
"""
|
156 |
+
Classify text using a RoBERTa model from the dataset directory
|
157 |
+
|
158 |
+
Args:
|
159 |
+
text (str): Text to analyze
|
160 |
+
task (str): Classification task ('sentiment', 'toxicity', 'topic', 'person')
|
161 |
+
model_name (str, optional): Specific model to use, if None will use task-appropriate model
|
162 |
+
|
163 |
+
Returns:
|
164 |
+
dict: Classification results with labels and scores
|
165 |
+
"""
|
166 |
+
try:
|
167 |
+
import torch
|
168 |
+
from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
|
169 |
+
|
170 |
+
# Map tasks to appropriate pre-trained models
|
171 |
+
task_model_map = {
|
172 |
+
"sentiment": "cardiffnlp/twitter-roberta-base-sentiment",
|
173 |
+
"toxicity": "cardiffnlp/twitter-roberta-base-hate",
|
174 |
+
"topic": "facebook/bart-large-mnli", # Zero-shot classification for topics
|
175 |
+
"person": "roberta-base" # Default for person detection - could be fine-tuned
|
176 |
+
}
|
177 |
+
|
178 |
+
# Use mapped model if not specified
|
179 |
+
if model_name is None and task in task_model_map:
|
180 |
+
model_to_use = task_model_map[task]
|
181 |
+
elif model_name is not None:
|
182 |
+
model_to_use = model_name
|
183 |
+
else:
|
184 |
+
model_to_use = "roberta-base"
|
185 |
+
|
186 |
+
# Special handling for zero-shot topic classification
|
187 |
+
if task == "topic":
|
188 |
+
classifier = pipeline("zero-shot-classification", model=model_to_use)
|
189 |
+
topics = ["economy", "foreign policy", "healthcare", "environment", "immigration"]
|
190 |
+
results = classifier(text, topics, multi_label=False)
|
191 |
+
return {
|
192 |
+
"labels": results["labels"],
|
193 |
+
"scores": results["scores"]
|
194 |
+
}
|
195 |
+
else:
|
196 |
+
# Initialize the classification pipeline
|
197 |
+
classifier = pipeline("text-classification", model=model_to_use, return_all_scores=True)
|
198 |
+
|
199 |
+
# Get classification results
|
200 |
+
results = classifier(text)
|
201 |
+
|
202 |
+
# Format results for consistent output
|
203 |
+
if isinstance(results, list) and len(results) == 1:
|
204 |
+
results = results[0]
|
205 |
+
|
206 |
+
return {
|
207 |
+
"task": task,
|
208 |
+
"model": model_to_use,
|
209 |
+
"results": results
|
210 |
+
}
|
211 |
+
|
212 |
+
except ImportError:
|
213 |
+
return {"error": "Required packages not installed. Please install transformers and torch."}
|
214 |
+
except Exception as e:
|
215 |
+
return {"error": f"Classification failed: {str(e)}"}
|
216 |
+
|
217 |
+
def analyze_dataset_with_roberta(dataset_texts, task="topic"):
|
218 |
+
"""
|
219 |
+
Analyze a collection of dataset texts using RoBERTa models
|
220 |
+
|
221 |
+
Args:
|
222 |
+
dataset_texts (dict): Dictionary with keys as text identifiers and values as text content
|
223 |
+
task (str): Classification task to perform
|
224 |
+
|
225 |
+
Returns:
|
226 |
+
dict: Classification results keyed by text identifier
|
227 |
+
"""
|
228 |
+
results = {}
|
229 |
+
|
230 |
+
for text_id, text_content in dataset_texts.items():
|
231 |
+
results[text_id] = classify_with_roberta(text_content, task=task)
|
232 |
+
|
233 |
return results
|
requirements.txt
CHANGED
@@ -5,3 +5,5 @@ nltk>=3.6.0
|
|
5 |
pandas>=1.3.0
|
6 |
plotly>=5.3.0
|
7 |
matplotlib>=3.4.0
|
|
|
|
|
|
5 |
pandas>=1.3.0
|
6 |
plotly>=5.3.0
|
7 |
matplotlib>=3.4.0
|
8 |
+
transformers>=4.15.0
|
9 |
+
torch>=1.9.0
|
visualization/__init__.py
CHANGED
@@ -6,10 +6,12 @@ from .bow_visualizer import process_and_visualize_analysis
|
|
6 |
from .topic_visualizer import process_and_visualize_topic_analysis
|
7 |
from .ngram_visualizer import process_and_visualize_ngram_analysis
|
8 |
from .bias_visualizer import process_and_visualize_bias_analysis
|
|
|
9 |
|
10 |
__all__ = [
|
11 |
'process_and_visualize_analysis',
|
12 |
'process_and_visualize_topic_analysis',
|
13 |
'process_and_visualize_ngram_analysis',
|
14 |
-
'process_and_visualize_bias_analysis'
|
|
|
15 |
]
|
|
|
6 |
from .topic_visualizer import process_and_visualize_topic_analysis
|
7 |
from .ngram_visualizer import process_and_visualize_ngram_analysis
|
8 |
from .bias_visualizer import process_and_visualize_bias_analysis
|
9 |
+
from .roberta_visualizer import process_and_visualize_sentiment_analysis
|
10 |
|
11 |
__all__ = [
|
12 |
'process_and_visualize_analysis',
|
13 |
'process_and_visualize_topic_analysis',
|
14 |
'process_and_visualize_ngram_analysis',
|
15 |
+
'process_and_visualize_bias_analysis',
|
16 |
+
'process_and_visualize_sentiment_analysis'
|
17 |
]
|
visualization/roberta_visualizer.py
ADDED
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
Visualization components for RoBERTa sentiment analysis
|
3 |
+
"""
|
4 |
+
import gradio as gr
|
5 |
+
import pandas as pd
|
6 |
+
import plotly.express as px
|
7 |
+
import plotly.graph_objects as go
|
8 |
+
from plotly.subplots import make_subplots
|
9 |
+
import numpy as np
|
10 |
+
import json
|
11 |
+
|
12 |
+
def create_sentiment_visualization(analysis_results):
|
13 |
+
"""
|
14 |
+
Create visualizations for RoBERTa sentiment analysis results
|
15 |
+
|
16 |
+
Args:
|
17 |
+
analysis_results (dict): Analysis results from the sentiment analysis
|
18 |
+
|
19 |
+
Returns:
|
20 |
+
list: List of gradio components with visualizations
|
21 |
+
"""
|
22 |
+
output_components = []
|
23 |
+
|
24 |
+
# Check if we have valid results
|
25 |
+
if not analysis_results or "analyses" not in analysis_results:
|
26 |
+
return [gr.Markdown("No analysis results found.")]
|
27 |
+
|
28 |
+
# Process each prompt
|
29 |
+
for prompt, analyses in analysis_results["analyses"].items():
|
30 |
+
output_components.append(gr.Markdown(f"## Analysis of Prompt: \"{prompt[:100]}{'...' if len(prompt) > 100 else ''}\""))
|
31 |
+
|
32 |
+
# Process RoBERTa sentiment analysis if available
|
33 |
+
if "roberta_sentiment" in analyses:
|
34 |
+
sentiment_results = analyses["roberta_sentiment"]
|
35 |
+
|
36 |
+
# Check if there's an error
|
37 |
+
if "error" in sentiment_results:
|
38 |
+
output_components.append(gr.Markdown(f"**Error in sentiment analysis:** {sentiment_results['error']}"))
|
39 |
+
continue
|
40 |
+
|
41 |
+
# Show models being compared
|
42 |
+
models = sentiment_results.get("models", [])
|
43 |
+
if len(models) >= 2:
|
44 |
+
output_components.append(gr.Markdown(f"### RoBERTa Sentiment Analysis: Comparing {models[0]} and {models[1]}"))
|
45 |
+
|
46 |
+
# Create a sentiment comparison chart
|
47 |
+
sa_data = sentiment_results.get("sentiment_analysis", {})
|
48 |
+
if sa_data and len(models) >= 2:
|
49 |
+
# Extract sentiment scores and labels for comparison
|
50 |
+
model_data = []
|
51 |
+
|
52 |
+
for model_name in models:
|
53 |
+
if model_name in sa_data:
|
54 |
+
model_result = sa_data[model_name]
|
55 |
+
model_data.append({
|
56 |
+
"model": model_name,
|
57 |
+
"sentiment_score": model_result.get("sentiment_score", 0),
|
58 |
+
"label": model_result.get("label", "neutral"),
|
59 |
+
"contradiction": model_result.get("scores", {}).get("contradiction", 0),
|
60 |
+
"neutral": model_result.get("scores", {}).get("neutral", 0),
|
61 |
+
"entailment": model_result.get("scores", {}).get("entailment", 0)
|
62 |
+
})
|
63 |
+
|
64 |
+
if model_data:
|
65 |
+
df = pd.DataFrame(model_data)
|
66 |
+
|
67 |
+
# Create gauge chart for sentiment scores
|
68 |
+
fig = go.Figure()
|
69 |
+
|
70 |
+
# Add gauge for each model
|
71 |
+
for i, row in df.iterrows():
|
72 |
+
# Set color based on sentiment
|
73 |
+
color = "green" if row["sentiment_score"] > 0.5 else "red" if row["sentiment_score"] < -0.5 else "gray"
|
74 |
+
|
75 |
+
fig.add_trace(go.Indicator(
|
76 |
+
mode="gauge+number",
|
77 |
+
value=row["sentiment_score"],
|
78 |
+
title={"text": f"{row['model']}<br><span style='font-size:0.8em;color:{color}'>{row['label'].capitalize()}</span>"},
|
79 |
+
gauge={
|
80 |
+
"axis": {"range": [-2, 2], "tickmode": "array", "tickvals": [-2, -1, 0, 1, 2],
|
81 |
+
"ticktext": ["Very Negative", "Negative", "Neutral", "Positive", "Very Positive"]},
|
82 |
+
"bar": {"color": color},
|
83 |
+
"threshold": {
|
84 |
+
"line": {"color": "black", "width": 2},
|
85 |
+
"thickness": 0.75,
|
86 |
+
"value": row["sentiment_score"]
|
87 |
+
},
|
88 |
+
"steps": [
|
89 |
+
{"range": [-2, -0.5], "color": "rgba(255, 0, 0, 0.2)"},
|
90 |
+
{"range": [-0.5, 0.5], "color": "rgba(128, 128, 128, 0.2)"},
|
91 |
+
{"range": [0.5, 2], "color": "rgba(0, 128, 0, 0.2)"}
|
92 |
+
]
|
93 |
+
},
|
94 |
+
domain={"row": 0, "column": i}
|
95 |
+
))
|
96 |
+
|
97 |
+
# Layout adjustments
|
98 |
+
fig.update_layout(
|
99 |
+
title="Sentiment Score Comparison",
|
100 |
+
grid={"rows": 1, "columns": len(df), "pattern": "independent"},
|
101 |
+
height=300,
|
102 |
+
margin=dict(t=70, b=30, l=30, r=30)
|
103 |
+
)
|
104 |
+
|
105 |
+
output_components.append(gr.Plot(value=fig))
|
106 |
+
|
107 |
+
# Create detailed scores visualization
|
108 |
+
fig2 = make_subplots(rows=1, cols=len(df),
|
109 |
+
subplot_titles=[f"{row['model']} Detailed Scores" for i, row in df.iterrows()])
|
110 |
+
|
111 |
+
for i, row in df.iterrows():
|
112 |
+
fig2.add_trace(
|
113 |
+
go.Bar(
|
114 |
+
x=["Contradiction (Negative)", "Neutral", "Entailment (Positive)"],
|
115 |
+
y=[row["contradiction"], row["neutral"], row["entailment"]],
|
116 |
+
marker_color=["rgba(255, 0, 0, 0.6)", "rgba(128, 128, 128, 0.6)", "rgba(0, 128, 0, 0.6)"]
|
117 |
+
),
|
118 |
+
row=1, col=i+1
|
119 |
+
)
|
120 |
+
|
121 |
+
fig2.update_layout(
|
122 |
+
title="RoBERTa Classification Scores",
|
123 |
+
showlegend=False,
|
124 |
+
height=350,
|
125 |
+
margin=dict(t=70, b=30, l=30, r=30)
|
126 |
+
)
|
127 |
+
|
128 |
+
output_components.append(gr.Plot(value=fig2))
|
129 |
+
|
130 |
+
# Display comparison summary
|
131 |
+
if "comparison" in sentiment_results:
|
132 |
+
comparison = sentiment_results["comparison"]
|
133 |
+
|
134 |
+
summary_html = """
|
135 |
+
<div style="margin: 20px 0; padding: 15px; background-color: #f8f9fa; border-radius: 5px;">
|
136 |
+
<h4 style="margin-top: 0;">Sentiment Comparison Summary</h4>
|
137 |
+
"""
|
138 |
+
|
139 |
+
# Add difference direction
|
140 |
+
if "difference_direction" in comparison:
|
141 |
+
summary_html += f"""
|
142 |
+
<p style="font-weight: 500; margin-bottom: 10px;">
|
143 |
+
{comparison["difference_direction"]}
|
144 |
+
</p>
|
145 |
+
"""
|
146 |
+
|
147 |
+
# Add significance info
|
148 |
+
if "significant_difference" in comparison:
|
149 |
+
color = "red" if comparison["significant_difference"] else "green"
|
150 |
+
significance = "Significant" if comparison["significant_difference"] else "Minor"
|
151 |
+
|
152 |
+
summary_html += f"""
|
153 |
+
<p>
|
154 |
+
<span style="font-weight: bold; color: {color};">{significance} difference</span> in sentiment
|
155 |
+
(difference score: {comparison.get("sentiment_difference", 0):.2f})
|
156 |
+
</p>
|
157 |
+
"""
|
158 |
+
|
159 |
+
summary_html += "</div>"
|
160 |
+
output_components.append(gr.HTML(summary_html))
|
161 |
+
|
162 |
+
# Display sentence-level sentiment analysis for both responses
|
163 |
+
model_sentences = {}
|
164 |
+
|
165 |
+
for model_name in models:
|
166 |
+
if model_name in sa_data and "sentence_scores" in sa_data[model_name] and sa_data[model_name]["sentence_scores"]:
|
167 |
+
model_sentences[model_name] = sa_data[model_name]["sentence_scores"]
|
168 |
+
|
169 |
+
if model_sentences and any(len(sentences) > 0 for sentences in model_sentences.values()):
|
170 |
+
output_components.append(gr.Markdown("### Sentence-Level Sentiment Analysis"))
|
171 |
+
|
172 |
+
for model_name, sentences in model_sentences.items():
|
173 |
+
if sentences:
|
174 |
+
output_components.append(gr.Markdown(f"#### {model_name} Response Breakdown"))
|
175 |
+
|
176 |
+
# Create HTML visualization for sentences with sentiment
|
177 |
+
sentences_html = """
|
178 |
+
<div style="margin-bottom: 20px;">
|
179 |
+
"""
|
180 |
+
|
181 |
+
for i, sentence in enumerate(sentences):
|
182 |
+
score = sentence.get("score", 0)
|
183 |
+
label = sentence.get("label", "neutral")
|
184 |
+
text = sentence.get("text", "")
|
185 |
+
|
186 |
+
# Skip very short sentences or empty text
|
187 |
+
if len(text.split()) < 3:
|
188 |
+
continue
|
189 |
+
|
190 |
+
# Color based on sentiment
|
191 |
+
if label == "positive":
|
192 |
+
color = f"rgba(0, 128, 0, {min(1.0, abs(score) * 0.5)})"
|
193 |
+
border = "rgba(0, 128, 0, 0.3)"
|
194 |
+
elif label == "negative":
|
195 |
+
color = f"rgba(255, 0, 0, {min(1.0, abs(score) * 0.5)})"
|
196 |
+
border = "rgba(255, 0, 0, 0.3)"
|
197 |
+
else:
|
198 |
+
color = "rgba(128, 128, 128, 0.1)"
|
199 |
+
border = "rgba(128, 128, 128, 0.3)"
|
200 |
+
|
201 |
+
sentences_html += f"""
|
202 |
+
<div style="padding: 10px; margin-bottom: 10px; background-color: {color};
|
203 |
+
border-radius: 5px; border: 1px solid {border};">
|
204 |
+
<div style="display: flex; justify-content: space-between;">
|
205 |
+
<span>{text}</span>
|
206 |
+
<span style="margin-left: 10px; font-weight: bold;">
|
207 |
+
{score:.2f} ({label.capitalize()})
|
208 |
+
</span>
|
209 |
+
</div>
|
210 |
+
</div>
|
211 |
+
"""
|
212 |
+
|
213 |
+
sentences_html += "</div>"
|
214 |
+
output_components.append(gr.HTML(sentences_html))
|
215 |
+
|
216 |
+
# If no components were added, show a message
|
217 |
+
if len(output_components) <= 1:
|
218 |
+
output_components.append(gr.Markdown("No detailed sentiment analysis found in results."))
|
219 |
+
|
220 |
+
return output_components
|
221 |
+
|
222 |
+
def process_and_visualize_sentiment_analysis(analysis_results):
|
223 |
+
"""
|
224 |
+
Process the sentiment analysis results and create visualization components
|
225 |
+
|
226 |
+
Args:
|
227 |
+
analysis_results (dict): The analysis results
|
228 |
+
|
229 |
+
Returns:
|
230 |
+
list: List of gradio components for visualization
|
231 |
+
"""
|
232 |
+
try:
|
233 |
+
print(f"Starting visualization of sentiment analysis results")
|
234 |
+
components = create_sentiment_visualization(analysis_results)
|
235 |
+
return components
|
236 |
+
except Exception as e:
|
237 |
+
import traceback
|
238 |
+
error_msg = f"Sentiment visualization error: {str(e)}\n{traceback.format_exc()}"
|
239 |
+
print(error_msg)
|
240 |
+
return [gr.Markdown(f"**Error during sentiment visualization:**\n\n```\n{str(e)}\n```")]
|