Spaces:
Sleeping
Sleeping
Ryan
commited on
Commit
·
cc937dd
1
Parent(s):
564f1f2
update
Browse files- app.py +95 -0
- processors/bias_detection.py +274 -0
- ui/analysis_screen.py +45 -0
- visualization/__init__.py +4 -2
- visualization/bias_visualizer.py +233 -0
app.py
CHANGED
@@ -447,6 +447,101 @@ def create_app():
|
|
447 |
f"- **{category}**: {diff}"
|
448 |
for category, diff in differences.items()
|
449 |
])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
450 |
|
451 |
# If we don't have visualization data from any analysis
|
452 |
if not visualization_area_visible:
|
|
|
447 |
f"- **{category}**: {diff}"
|
448 |
for category, diff in differences.items()
|
449 |
])
|
450 |
+
|
451 |
+
# Check for Bias Detection analysis
|
452 |
+
elif selected_analysis == "Bias Detection" and "bias_detection" in analyses:
|
453 |
+
visualization_area_visible = True
|
454 |
+
bias_results = analyses["bias_detection"]
|
455 |
+
models = bias_results.get("models", [])
|
456 |
+
|
457 |
+
if len(models) >= 2:
|
458 |
+
prompt_title_visible = True
|
459 |
+
prompt_title_value = f"## Analysis of Prompt: \"{prompt[:100]}...\""
|
460 |
+
|
461 |
+
models_compared_visible = True
|
462 |
+
models_compared_value = f"### Bias Analysis: Comparing responses from {models[0]} and {models[1]}"
|
463 |
+
|
464 |
+
# Display comparative bias results
|
465 |
+
model1_name = models[0]
|
466 |
+
model2_name = models[1]
|
467 |
+
|
468 |
+
if "comparative" in bias_results:
|
469 |
+
comparative = bias_results["comparative"]
|
470 |
+
|
471 |
+
# Format summary for display
|
472 |
+
model1_title_visible = True
|
473 |
+
model1_title_value = "#### Bias Detection Summary"
|
474 |
+
model1_words_visible = True
|
475 |
+
|
476 |
+
summary_parts = []
|
477 |
+
|
478 |
+
# Add sentiment comparison
|
479 |
+
if "sentiment" in comparative:
|
480 |
+
sent = comparative["sentiment"]
|
481 |
+
is_significant = sent.get("significant", False)
|
482 |
+
summary_parts.append(
|
483 |
+
f"**Sentiment Bias**: {model1_name} shows {sent.get(model1_name, 'N/A')} sentiment, " +
|
484 |
+
f"while {model2_name} shows {sent.get(model2_name, 'N/A')} sentiment. " +
|
485 |
+
f"({'Significant' if is_significant else 'Minor'} difference)"
|
486 |
+
)
|
487 |
+
|
488 |
+
# Add partisan comparison
|
489 |
+
if "partisan" in comparative:
|
490 |
+
part = comparative["partisan"]
|
491 |
+
is_significant = part.get("significant", False)
|
492 |
+
summary_parts.append(
|
493 |
+
f"**Partisan Leaning**: {model1_name} appears {part.get(model1_name, 'N/A')}, " +
|
494 |
+
f"while {model2_name} appears {part.get(model2_name, 'N/A')}. " +
|
495 |
+
f"({'Significant' if is_significant else 'Minor'} difference)"
|
496 |
+
)
|
497 |
+
|
498 |
+
# Add framing comparison
|
499 |
+
if "framing" in comparative:
|
500 |
+
frame = comparative["framing"]
|
501 |
+
different_frames = frame.get("different_frames", False)
|
502 |
+
m1_frame = frame.get(model1_name, "N/A").replace('_', ' ').title()
|
503 |
+
m2_frame = frame.get(model2_name, "N/A").replace('_', ' ').title()
|
504 |
+
summary_parts.append(
|
505 |
+
f"**Issue Framing**: {model1_name} primarily frames issues in {m1_frame} terms, " +
|
506 |
+
f"while {model2_name} uses {m2_frame} framing. " +
|
507 |
+
f"({'Different' if different_frames else 'Similar'} approaches)"
|
508 |
+
)
|
509 |
+
|
510 |
+
# Add overall assessment
|
511 |
+
if "overall" in comparative:
|
512 |
+
overall = comparative["overall"]
|
513 |
+
significant = overall.get("significant_bias_difference", False)
|
514 |
+
summary_parts.append(
|
515 |
+
f"**Overall Assessment**: " +
|
516 |
+
f"Analysis shows a {overall.get('difference', 0):.2f}/1.0 difference in bias patterns. " +
|
517 |
+
f"({'Significant' if significant else 'Minor'} overall bias difference)"
|
518 |
+
)
|
519 |
+
|
520 |
+
# Combine all parts
|
521 |
+
model1_words_value = "\n\n".join(summary_parts)
|
522 |
+
|
523 |
+
# Format detailed term analysis
|
524 |
+
if (model1_name in bias_results and "partisan" in bias_results[model1_name] and
|
525 |
+
model2_name in bias_results and "partisan" in bias_results[model2_name]):
|
526 |
+
|
527 |
+
model2_title_visible = True
|
528 |
+
model2_title_value = "#### Partisan Term Analysis"
|
529 |
+
model2_words_visible = True
|
530 |
+
|
531 |
+
m1_lib = bias_results[model1_name]["partisan"].get("liberal_terms", [])
|
532 |
+
m1_con = bias_results[model1_name]["partisan"].get("conservative_terms", [])
|
533 |
+
m2_lib = bias_results[model2_name]["partisan"].get("liberal_terms", [])
|
534 |
+
m2_con = bias_results[model2_name]["partisan"].get("conservative_terms", [])
|
535 |
+
|
536 |
+
model2_words_value = f"""
|
537 |
+
**{model1_name}**:
|
538 |
+
- Liberal terms: {', '.join(m1_lib) if m1_lib else 'None detected'}
|
539 |
+
- Conservative terms: {', '.join(m1_con) if m1_con else 'None detected'}
|
540 |
+
|
541 |
+
**{model2_name}**:
|
542 |
+
- Liberal terms: {', '.join(m2_lib) if m2_lib else 'None detected'}
|
543 |
+
- Conservative terms: {', '.join(m2_con) if m2_con else 'None detected'}
|
544 |
+
"""
|
545 |
|
546 |
# If we don't have visualization data from any analysis
|
547 |
if not visualization_area_visible:
|
processors/bias_detection.py
ADDED
@@ -0,0 +1,274 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
Bias detection processor for analyzing political bias in text responses
|
3 |
+
"""
|
4 |
+
import nltk
|
5 |
+
from nltk.sentiment import SentimentIntensityAnalyzer
|
6 |
+
from sklearn.feature_extraction.text import CountVectorizer
|
7 |
+
import re
|
8 |
+
import json
|
9 |
+
import os
|
10 |
+
import numpy as np
|
11 |
+
|
12 |
+
# Ensure NLTK resources are available
|
13 |
+
def download_nltk_resources():
|
14 |
+
"""Download required NLTK resources if not already downloaded"""
|
15 |
+
try:
|
16 |
+
nltk.download('vader_lexicon', quiet=True)
|
17 |
+
except:
|
18 |
+
pass
|
19 |
+
|
20 |
+
download_nltk_resources()
|
21 |
+
|
22 |
+
# Dictionary of partisan-leaning words
|
23 |
+
# These are simplified examples; a real implementation would use a more comprehensive lexicon
|
24 |
+
PARTISAN_WORDS = {
|
25 |
+
"liberal": [
|
26 |
+
"progressive", "equity", "climate", "reform", "collective",
|
27 |
+
"diversity", "inclusive", "sustainable", "justice", "regulation"
|
28 |
+
],
|
29 |
+
"conservative": [
|
30 |
+
"traditional", "freedom", "liberty", "individual", "faith",
|
31 |
+
"values", "efficient", "deregulation", "patriot", "security"
|
32 |
+
]
|
33 |
+
}
|
34 |
+
|
35 |
+
# Dictionary of framing patterns
|
36 |
+
FRAMING_PATTERNS = {
|
37 |
+
"economic": [
|
38 |
+
r"econom(y|ic|ics)", r"tax(es|ation)", r"budget", r"spend(ing)",
|
39 |
+
r"jobs?", r"wage", r"growth", r"inflation", r"invest(ment)?"
|
40 |
+
],
|
41 |
+
"moral": [
|
42 |
+
r"values?", r"ethic(s|al)", r"moral(s|ity)", r"right(s|eous)",
|
43 |
+
r"wrong", r"good", r"bad", r"faith", r"belief", r"tradition(al)?"
|
44 |
+
],
|
45 |
+
"security": [
|
46 |
+
r"secur(e|ity)", r"defense", r"protect(ion)?", r"threat",
|
47 |
+
r"danger(ous)?", r"safe(ty)?", r"nation(al)?", r"terror(ism|ist)"
|
48 |
+
],
|
49 |
+
"social_welfare": [
|
50 |
+
r"health(care)?", r"education", r"welfare", r"benefit", r"program",
|
51 |
+
r"help", r"assist(ance)?", r"support", r"service", r"care"
|
52 |
+
]
|
53 |
+
}
|
54 |
+
|
55 |
+
def detect_sentiment_bias(text):
|
56 |
+
"""
|
57 |
+
Analyze the sentiment of a text to identify potential bias
|
58 |
+
|
59 |
+
Args:
|
60 |
+
text (str): The text to analyze
|
61 |
+
|
62 |
+
Returns:
|
63 |
+
dict: Sentiment analysis results
|
64 |
+
"""
|
65 |
+
sia = SentimentIntensityAnalyzer()
|
66 |
+
sentiment = sia.polarity_scores(text)
|
67 |
+
|
68 |
+
# Determine if sentiment indicates bias
|
69 |
+
if sentiment['compound'] >= 0.25:
|
70 |
+
bias_direction = "positive"
|
71 |
+
bias_strength = min(1.0, sentiment['compound'] * 2) # Scale to 0-1
|
72 |
+
elif sentiment['compound'] <= -0.25:
|
73 |
+
bias_direction = "negative"
|
74 |
+
bias_strength = min(1.0, abs(sentiment['compound'] * 2)) # Scale to 0-1
|
75 |
+
else:
|
76 |
+
bias_direction = "neutral"
|
77 |
+
bias_strength = 0.0
|
78 |
+
|
79 |
+
return {
|
80 |
+
"sentiment_scores": sentiment,
|
81 |
+
"bias_direction": bias_direction,
|
82 |
+
"bias_strength": bias_strength
|
83 |
+
}
|
84 |
+
|
85 |
+
def detect_partisan_leaning(text):
|
86 |
+
"""
|
87 |
+
Analyze text for partisan-leaning language
|
88 |
+
|
89 |
+
Args:
|
90 |
+
text (str): The text to analyze
|
91 |
+
|
92 |
+
Returns:
|
93 |
+
dict: Partisan leaning analysis results
|
94 |
+
"""
|
95 |
+
text_lower = text.lower()
|
96 |
+
|
97 |
+
# Count partisan words
|
98 |
+
liberal_count = 0
|
99 |
+
conservative_count = 0
|
100 |
+
|
101 |
+
liberal_matches = []
|
102 |
+
conservative_matches = []
|
103 |
+
|
104 |
+
# Search for partisan words in text
|
105 |
+
for word in PARTISAN_WORDS["liberal"]:
|
106 |
+
matches = re.findall(r'\b' + word + r'\b', text_lower)
|
107 |
+
if matches:
|
108 |
+
liberal_count += len(matches)
|
109 |
+
liberal_matches.extend(matches)
|
110 |
+
|
111 |
+
for word in PARTISAN_WORDS["conservative"]:
|
112 |
+
matches = re.findall(r'\b' + word + r'\b', text_lower)
|
113 |
+
if matches:
|
114 |
+
conservative_count += len(matches)
|
115 |
+
conservative_matches.extend(matches)
|
116 |
+
|
117 |
+
# Calculate partisan lean score (-1 to 1, negative = liberal, positive = conservative)
|
118 |
+
total_count = liberal_count + conservative_count
|
119 |
+
if total_count > 0:
|
120 |
+
lean_score = (conservative_count - liberal_count) / total_count
|
121 |
+
else:
|
122 |
+
lean_score = 0
|
123 |
+
|
124 |
+
# Determine leaning based on score
|
125 |
+
if lean_score <= -0.2:
|
126 |
+
leaning = "liberal"
|
127 |
+
strength = min(1.0, abs(lean_score * 2))
|
128 |
+
elif lean_score >= 0.2:
|
129 |
+
leaning = "conservative"
|
130 |
+
strength = min(1.0, lean_score * 2)
|
131 |
+
else:
|
132 |
+
leaning = "balanced"
|
133 |
+
strength = 0.0
|
134 |
+
|
135 |
+
return {
|
136 |
+
"liberal_count": liberal_count,
|
137 |
+
"conservative_count": conservative_count,
|
138 |
+
"liberal_terms": liberal_matches,
|
139 |
+
"conservative_terms": conservative_matches,
|
140 |
+
"lean_score": lean_score,
|
141 |
+
"leaning": leaning,
|
142 |
+
"strength": strength
|
143 |
+
}
|
144 |
+
|
145 |
+
def detect_framing_bias(text):
|
146 |
+
"""
|
147 |
+
Analyze how the text frames issues
|
148 |
+
|
149 |
+
Args:
|
150 |
+
text (str): The text to analyze
|
151 |
+
|
152 |
+
Returns:
|
153 |
+
dict: Framing analysis results
|
154 |
+
"""
|
155 |
+
text_lower = text.lower()
|
156 |
+
framing_counts = {}
|
157 |
+
framing_examples = {}
|
158 |
+
|
159 |
+
# Count framing patterns
|
160 |
+
for frame, patterns in FRAMING_PATTERNS.items():
|
161 |
+
framing_counts[frame] = 0
|
162 |
+
framing_examples[frame] = []
|
163 |
+
|
164 |
+
for pattern in patterns:
|
165 |
+
matches = re.findall(pattern, text_lower)
|
166 |
+
if matches:
|
167 |
+
framing_counts[frame] += len(matches)
|
168 |
+
# Store up to 5 examples of each frame
|
169 |
+
unique_matches = set(matches)
|
170 |
+
framing_examples[frame].extend(list(unique_matches)[:5])
|
171 |
+
|
172 |
+
# Calculate dominant frame
|
173 |
+
total_framing = sum(framing_counts.values())
|
174 |
+
framing_distribution = {}
|
175 |
+
|
176 |
+
if total_framing > 0:
|
177 |
+
for frame, count in framing_counts.items():
|
178 |
+
framing_distribution[frame] = count / total_framing
|
179 |
+
|
180 |
+
dominant_frame = max(framing_counts.items(), key=lambda x: x[1])[0]
|
181 |
+
frame_bias_strength = max(0.0, framing_distribution[dominant_frame] - 0.25)
|
182 |
+
else:
|
183 |
+
dominant_frame = "none"
|
184 |
+
frame_bias_strength = 0.0
|
185 |
+
framing_distribution = {frame: 0.0 for frame in FRAMING_PATTERNS.keys()}
|
186 |
+
|
187 |
+
return {
|
188 |
+
"framing_counts": framing_counts,
|
189 |
+
"framing_examples": framing_examples,
|
190 |
+
"framing_distribution": framing_distribution,
|
191 |
+
"dominant_frame": dominant_frame,
|
192 |
+
"frame_bias_strength": frame_bias_strength
|
193 |
+
}
|
194 |
+
|
195 |
+
def compare_bias(text1, text2, model_names=None):
|
196 |
+
"""
|
197 |
+
Compare potential bias in two texts
|
198 |
+
|
199 |
+
Args:
|
200 |
+
text1 (str): First text to analyze
|
201 |
+
text2 (str): Second text to analyze
|
202 |
+
model_names (list): Optional names of models being compared
|
203 |
+
|
204 |
+
Returns:
|
205 |
+
dict: Comparative bias analysis
|
206 |
+
"""
|
207 |
+
# Set default model names if not provided
|
208 |
+
if model_names is None or len(model_names) < 2:
|
209 |
+
model_names = ["Model 1", "Model 2"]
|
210 |
+
|
211 |
+
model1_name, model2_name = model_names[0], model_names[1]
|
212 |
+
|
213 |
+
# Analyze each text
|
214 |
+
sentiment_results1 = detect_sentiment_bias(text1)
|
215 |
+
sentiment_results2 = detect_sentiment_bias(text2)
|
216 |
+
|
217 |
+
partisan_results1 = detect_partisan_leaning(text1)
|
218 |
+
partisan_results2 = detect_partisan_leaning(text2)
|
219 |
+
|
220 |
+
framing_results1 = detect_framing_bias(text1)
|
221 |
+
framing_results2 = detect_framing_bias(text2)
|
222 |
+
|
223 |
+
# Determine if there's a significant difference in bias
|
224 |
+
sentiment_difference = abs(sentiment_results1["bias_strength"] - sentiment_results2["bias_strength"])
|
225 |
+
|
226 |
+
# For partisan leaning, compare the scores (negative is liberal, positive is conservative)
|
227 |
+
partisan_difference = abs(partisan_results1["lean_score"] - partisan_results2["lean_score"])
|
228 |
+
|
229 |
+
# Calculate overall bias difference
|
230 |
+
overall_difference = (sentiment_difference + partisan_difference) / 2
|
231 |
+
|
232 |
+
# Compare dominant frames
|
233 |
+
frame_difference = framing_results1["dominant_frame"] != framing_results2["dominant_frame"] and \
|
234 |
+
(framing_results1["frame_bias_strength"] > 0.1 or framing_results2["frame_bias_strength"] > 0.1)
|
235 |
+
|
236 |
+
# Create comparative analysis
|
237 |
+
comparative = {
|
238 |
+
"sentiment": {
|
239 |
+
model1_name: sentiment_results1["bias_direction"],
|
240 |
+
model2_name: sentiment_results2["bias_direction"],
|
241 |
+
"difference": sentiment_difference,
|
242 |
+
"significant": sentiment_difference > 0.3
|
243 |
+
},
|
244 |
+
"partisan": {
|
245 |
+
model1_name: partisan_results1["leaning"],
|
246 |
+
model2_name: partisan_results2["leaning"],
|
247 |
+
"difference": partisan_difference,
|
248 |
+
"significant": partisan_difference > 0.4
|
249 |
+
},
|
250 |
+
"framing": {
|
251 |
+
model1_name: framing_results1["dominant_frame"],
|
252 |
+
model2_name: framing_results2["dominant_frame"],
|
253 |
+
"different_frames": frame_difference
|
254 |
+
},
|
255 |
+
"overall": {
|
256 |
+
"difference": overall_difference,
|
257 |
+
"significant_bias_difference": overall_difference > 0.35
|
258 |
+
}
|
259 |
+
}
|
260 |
+
|
261 |
+
return {
|
262 |
+
"models": model_names,
|
263 |
+
model1_name: {
|
264 |
+
"sentiment": sentiment_results1,
|
265 |
+
"partisan": partisan_results1,
|
266 |
+
"framing": framing_results1
|
267 |
+
},
|
268 |
+
model2_name: {
|
269 |
+
"sentiment": sentiment_results2,
|
270 |
+
"partisan": partisan_results2,
|
271 |
+
"framing": framing_results2
|
272 |
+
},
|
273 |
+
"comparative": comparative
|
274 |
+
}
|
ui/analysis_screen.py
CHANGED
@@ -7,6 +7,7 @@ from processors.topic_modeling import compare_topics
|
|
7 |
from processors.ngram_analysis import compare_ngrams
|
8 |
from processors.bow_analysis import compare_bow
|
9 |
from processors.text_classifiers import classify_formality, classify_sentiment, classify_complexity, compare_classifications
|
|
|
10 |
|
11 |
def create_analysis_screen():
|
12 |
"""
|
@@ -435,6 +436,50 @@ def process_analysis_request(dataset, selected_analysis, parameters):
|
|
435 |
},
|
436 |
"differences": compare_classifications(model1_response, model2_response)
|
437 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
438 |
|
439 |
else:
|
440 |
# Unknown analysis type
|
|
|
7 |
from processors.ngram_analysis import compare_ngrams
|
8 |
from processors.bow_analysis import compare_bow
|
9 |
from processors.text_classifiers import classify_formality, classify_sentiment, classify_complexity, compare_classifications
|
10 |
+
from processors.bias_detection import compare_bias
|
11 |
|
12 |
def create_analysis_screen():
|
13 |
"""
|
|
|
436 |
},
|
437 |
"differences": compare_classifications(model1_response, model2_response)
|
438 |
}
|
439 |
+
|
440 |
+
elif selected_analysis == "Bias Detection":
|
441 |
+
# Get the bias detection methods from parameters
|
442 |
+
bias_methods = parameters.get("bias_methods",
|
443 |
+
["Sentiment Analysis", "Partisan Leaning", "Framing Analysis"])
|
444 |
+
|
445 |
+
try:
|
446 |
+
# Perform bias detection analysis
|
447 |
+
bias_results = compare_bias(
|
448 |
+
model1_response,
|
449 |
+
model2_response,
|
450 |
+
model_names=[model1_name, model2_name]
|
451 |
+
)
|
452 |
+
|
453 |
+
# Filter results based on selected methods
|
454 |
+
filtered_results = {"models": [model1_name, model2_name]}
|
455 |
+
|
456 |
+
# Always include comparative data
|
457 |
+
if "comparative" in bias_results:
|
458 |
+
filtered_results["comparative"] = bias_results["comparative"]
|
459 |
+
|
460 |
+
# Include individual model results based on selected methods
|
461 |
+
for model in [model1_name, model2_name]:
|
462 |
+
filtered_results[model] = {}
|
463 |
+
|
464 |
+
if "Sentiment Analysis" in bias_methods and model in bias_results:
|
465 |
+
filtered_results[model]["sentiment"] = bias_results[model]["sentiment"]
|
466 |
+
|
467 |
+
if "Partisan Leaning" in bias_methods and model in bias_results:
|
468 |
+
filtered_results[model]["partisan"] = bias_results[model]["partisan"]
|
469 |
+
|
470 |
+
if "Framing Analysis" in bias_methods and model in bias_results:
|
471 |
+
filtered_results[model]["framing"] = bias_results[model]["framing"]
|
472 |
+
|
473 |
+
results["analyses"][prompt_text]["bias_detection"] = filtered_results
|
474 |
+
|
475 |
+
except Exception as e:
|
476 |
+
import traceback
|
477 |
+
print(f"Bias detection error: {str(e)}\n{traceback.format_exc()}")
|
478 |
+
results["analyses"][prompt_text]["bias_detection"] = {
|
479 |
+
"models": [model1_name, model2_name],
|
480 |
+
"error": str(e),
|
481 |
+
"message": "Bias detection failed. Try with different parameters."
|
482 |
+
}
|
483 |
|
484 |
else:
|
485 |
# Unknown analysis type
|
visualization/__init__.py
CHANGED
@@ -5,9 +5,11 @@ Visualization components for LLM Response Comparator
|
|
5 |
from .bow_visualizer import process_and_visualize_analysis
|
6 |
from .topic_visualizer import process_and_visualize_topic_analysis
|
7 |
from .ngram_visualizer import process_and_visualize_ngram_analysis
|
|
|
8 |
|
9 |
__all__ = [
|
10 |
'process_and_visualize_analysis',
|
11 |
'process_and_visualize_topic_analysis',
|
12 |
-
'process_and_visualize_ngram_analysis'
|
13 |
-
|
|
|
|
5 |
from .bow_visualizer import process_and_visualize_analysis
|
6 |
from .topic_visualizer import process_and_visualize_topic_analysis
|
7 |
from .ngram_visualizer import process_and_visualize_ngram_analysis
|
8 |
+
from .bias_visualizer import process_and_visualize_bias_analysis
|
9 |
|
10 |
__all__ = [
|
11 |
'process_and_visualize_analysis',
|
12 |
'process_and_visualize_topic_analysis',
|
13 |
+
'process_and_visualize_ngram_analysis',
|
14 |
+
'process_and_visualize_bias_analysis'
|
15 |
+
]
|
visualization/bias_visualizer.py
ADDED
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import plotly.graph_objects as go
|
3 |
+
import plotly.express as px
|
4 |
+
import pandas as pd
|
5 |
+
|
6 |
+
def create_bias_visualization(analysis_results):
|
7 |
+
"""
|
8 |
+
Create visualizations for bias detection analysis results
|
9 |
+
|
10 |
+
Args:
|
11 |
+
analysis_results (dict): Analysis results from the bias detection
|
12 |
+
|
13 |
+
Returns:
|
14 |
+
list: List of gradio components with visualizations
|
15 |
+
"""
|
16 |
+
output_components = []
|
17 |
+
|
18 |
+
# Check if we have valid results
|
19 |
+
if not analysis_results or "analyses" not in analysis_results:
|
20 |
+
return [gr.Markdown("No analysis results found.")]
|
21 |
+
|
22 |
+
# Process each prompt
|
23 |
+
for prompt, analyses in analysis_results["analyses"].items():
|
24 |
+
# Process Bias Detection analysis if available
|
25 |
+
if "bias_detection" in analyses:
|
26 |
+
bias_results = analyses["bias_detection"]
|
27 |
+
|
28 |
+
# Show models being compared
|
29 |
+
models = bias_results.get("models", [])
|
30 |
+
if len(models) >= 2:
|
31 |
+
output_components.append(gr.Markdown(f"### Bias Analysis: Comparing responses from {models[0]} and {models[1]}"))
|
32 |
+
|
33 |
+
# Check if there's an error
|
34 |
+
if "error" in bias_results:
|
35 |
+
output_components.append(gr.Markdown(f"**Error in bias detection:** {bias_results['error']}"))
|
36 |
+
continue
|
37 |
+
|
38 |
+
model1_name, model2_name = models[0], models[1]
|
39 |
+
|
40 |
+
# Comparative results
|
41 |
+
if "comparative" in bias_results:
|
42 |
+
comparative = bias_results["comparative"]
|
43 |
+
|
44 |
+
output_components.append(gr.Markdown("#### Comparative Bias Analysis"))
|
45 |
+
|
46 |
+
# Create summary table
|
47 |
+
summary_html = f"""
|
48 |
+
<table style="width:100%; border-collapse: collapse; margin-bottom: 20px;">
|
49 |
+
<tr>
|
50 |
+
<th style="border: 1px solid #ddd; padding: 8px; text-align: left; background-color: #f2f2f2;">Bias Category</th>
|
51 |
+
<th style="border: 1px solid #ddd; padding: 8px; text-align: left; background-color: #f2f2f2;">{model1_name}</th>
|
52 |
+
<th style="border: 1px solid #ddd; padding: 8px; text-align: left; background-color: #f2f2f2;">{model2_name}</th>
|
53 |
+
<th style="border: 1px solid #ddd; padding: 8px; text-align: left; background-color: #f2f2f2;">Significant Difference?</th>
|
54 |
+
</tr>
|
55 |
+
"""
|
56 |
+
|
57 |
+
# Sentiment row
|
58 |
+
if "sentiment" in comparative:
|
59 |
+
sent_sig = comparative["sentiment"].get("significant", False)
|
60 |
+
summary_html += f"""
|
61 |
+
<tr>
|
62 |
+
<td style="border: 1px solid #ddd; padding: 8px;">Sentiment Bias</td>
|
63 |
+
<td style="border: 1px solid #ddd; padding: 8px;">{comparative["sentiment"].get(model1_name, "N/A").title()}</td>
|
64 |
+
<td style="border: 1px solid #ddd; padding: 8px;">{comparative["sentiment"].get(model2_name, "N/A").title()}</td>
|
65 |
+
<td style="border: 1px solid #ddd; padding: 8px; font-weight: bold; color: {'red' if sent_sig else 'green'}">{"Yes" if sent_sig else "No"}</td>
|
66 |
+
</tr>
|
67 |
+
"""
|
68 |
+
|
69 |
+
# Partisan row
|
70 |
+
if "partisan" in comparative:
|
71 |
+
part_sig = comparative["partisan"].get("significant", False)
|
72 |
+
summary_html += f"""
|
73 |
+
<tr>
|
74 |
+
<td style="border: 1px solid #ddd; padding: 8px;">Partisan Leaning</td>
|
75 |
+
<td style="border: 1px solid #ddd; padding: 8px;">{comparative["partisan"].get(model1_name, "N/A").title()}</td>
|
76 |
+
<td style="border: 1px solid #ddd; padding: 8px;">{comparative["partisan"].get(model2_name, "N/A").title()}</td>
|
77 |
+
<td style="border: 1px solid #ddd; padding: 8px; font-weight: bold; color: {'red' if part_sig else 'green'}">{"Yes" if part_sig else "No"}</td>
|
78 |
+
</tr>
|
79 |
+
"""
|
80 |
+
|
81 |
+
# Framing row
|
82 |
+
if "framing" in comparative:
|
83 |
+
frame_diff = comparative["framing"].get("different_frames", False)
|
84 |
+
summary_html += f"""
|
85 |
+
<tr>
|
86 |
+
<td style="border: 1px solid #ddd; padding: 8px;">Dominant Frame</td>
|
87 |
+
<td style="border: 1px solid #ddd; padding: 8px;">{comparative["framing"].get(model1_name, "N/A").title().replace('_', ' ')}</td>
|
88 |
+
<td style="border: 1px solid #ddd; padding: 8px;">{comparative["framing"].get(model2_name, "N/A").title().replace('_', ' ')}</td>
|
89 |
+
<td style="border: 1px solid #ddd; padding: 8px; font-weight: bold; color: {'red' if frame_diff else 'green'}">{"Yes" if frame_diff else "No"}</td>
|
90 |
+
</tr>
|
91 |
+
"""
|
92 |
+
|
93 |
+
# Overall row
|
94 |
+
if "overall" in comparative:
|
95 |
+
overall_sig = comparative["overall"].get("significant_bias_difference", False)
|
96 |
+
summary_html += f"""
|
97 |
+
<tr>
|
98 |
+
<td style="border: 1px solid #ddd; padding: 8px; font-weight: bold;">Overall Bias Difference</td>
|
99 |
+
<td colspan="2" style="border: 1px solid #ddd; padding: 8px; text-align: center;">{comparative["overall"].get("difference", 0):.2f} / 1.0</td>
|
100 |
+
<td style="border: 1px solid #ddd; padding: 8px; font-weight: bold; color: {'red' if overall_sig else 'green'}">{"Yes" if overall_sig else "No"}</td>
|
101 |
+
</tr>
|
102 |
+
"""
|
103 |
+
|
104 |
+
summary_html += "</table>"
|
105 |
+
|
106 |
+
# Add the HTML table to the components
|
107 |
+
output_components.append(gr.HTML(summary_html))
|
108 |
+
|
109 |
+
# Create detailed visualizations for each model if available
|
110 |
+
for model_name in [model1_name, model2_name]:
|
111 |
+
if model_name in bias_results:
|
112 |
+
model_data = bias_results[model_name]
|
113 |
+
|
114 |
+
# Sentiment visualization
|
115 |
+
if "sentiment" in model_data:
|
116 |
+
sentiment = model_data["sentiment"]
|
117 |
+
if "sentiment_scores" in sentiment:
|
118 |
+
# Create sentiment score chart
|
119 |
+
sentiment_df = pd.DataFrame({
|
120 |
+
'Score': [
|
121 |
+
sentiment["sentiment_scores"]["pos"],
|
122 |
+
sentiment["sentiment_scores"]["neg"],
|
123 |
+
sentiment["sentiment_scores"]["neu"]
|
124 |
+
],
|
125 |
+
'Category': ['Positive', 'Negative', 'Neutral']
|
126 |
+
})
|
127 |
+
|
128 |
+
fig = px.bar(
|
129 |
+
sentiment_df,
|
130 |
+
x='Category',
|
131 |
+
y='Score',
|
132 |
+
title=f"Sentiment Analysis for {model_name}",
|
133 |
+
height=300
|
134 |
+
)
|
135 |
+
|
136 |
+
output_components.append(gr.Plot(value=fig))
|
137 |
+
|
138 |
+
# Partisan leaning visualization
|
139 |
+
if "partisan" in model_data:
|
140 |
+
partisan = model_data["partisan"]
|
141 |
+
if "liberal_count" in partisan and "conservative_count" in partisan:
|
142 |
+
# Create partisan terms chart
|
143 |
+
partisan_df = pd.DataFrame({
|
144 |
+
'Count': [partisan["liberal_count"], partisan["conservative_count"]],
|
145 |
+
'Category': ['Liberal Terms', 'Conservative Terms']
|
146 |
+
})
|
147 |
+
|
148 |
+
fig = px.bar(
|
149 |
+
partisan_df,
|
150 |
+
x='Category',
|
151 |
+
y='Count',
|
152 |
+
title=f"Partisan Term Usage for {model_name}",
|
153 |
+
color='Category',
|
154 |
+
color_discrete_map={
|
155 |
+
'Liberal Terms': 'blue',
|
156 |
+
'Conservative Terms': 'red'
|
157 |
+
},
|
158 |
+
height=300
|
159 |
+
)
|
160 |
+
|
161 |
+
output_components.append(gr.Plot(value=fig))
|
162 |
+
|
163 |
+
# Show example partisan terms
|
164 |
+
if "liberal_terms" in partisan or "conservative_terms" in partisan:
|
165 |
+
lib_terms = ", ".join(partisan.get("liberal_terms", []))
|
166 |
+
con_terms = ", ".join(partisan.get("conservative_terms", []))
|
167 |
+
|
168 |
+
if lib_terms or con_terms:
|
169 |
+
terms_md = f"**Partisan Terms Used by {model_name}**\n\n"
|
170 |
+
if lib_terms:
|
171 |
+
terms_md += f"- Liberal terms: {lib_terms}\n"
|
172 |
+
if con_terms:
|
173 |
+
terms_md += f"- Conservative terms: {con_terms}\n"
|
174 |
+
|
175 |
+
output_components.append(gr.Markdown(terms_md))
|
176 |
+
|
177 |
+
# Framing visualization
|
178 |
+
if "framing" in model_data:
|
179 |
+
framing = model_data["framing"]
|
180 |
+
if "framing_distribution" in framing:
|
181 |
+
# Create framing distribution chart
|
182 |
+
frame_items = []
|
183 |
+
for frame, value in framing["framing_distribution"].items():
|
184 |
+
frame_items.append({
|
185 |
+
'Frame': frame.replace('_', ' ').title(),
|
186 |
+
'Proportion': value
|
187 |
+
})
|
188 |
+
|
189 |
+
frame_df = pd.DataFrame(frame_items)
|
190 |
+
|
191 |
+
fig = px.pie(
|
192 |
+
frame_df,
|
193 |
+
values='Proportion',
|
194 |
+
names='Frame',
|
195 |
+
title=f"Issue Framing Distribution for {model_name}",
|
196 |
+
height=400
|
197 |
+
)
|
198 |
+
|
199 |
+
output_components.append(gr.Plot(value=fig))
|
200 |
+
|
201 |
+
# Show example framing terms
|
202 |
+
if "framing_examples" in framing:
|
203 |
+
examples_md = f"**Example Framing Terms Used by {model_name}**\n\n"
|
204 |
+
for frame, examples in framing["framing_examples"].items():
|
205 |
+
if examples:
|
206 |
+
examples_md += f"- {frame.replace('_', ' ').title()}: {', '.join(examples)}\n"
|
207 |
+
|
208 |
+
output_components.append(gr.Markdown(examples_md))
|
209 |
+
|
210 |
+
# If no components were added, show a message
|
211 |
+
if len(output_components) <= 1:
|
212 |
+
output_components.append(gr.Markdown("No detailed bias detection analysis found in results."))
|
213 |
+
|
214 |
+
return output_components
|
215 |
+
|
216 |
+
def process_and_visualize_bias_analysis(analysis_results):
|
217 |
+
"""
|
218 |
+
Process the bias detection analysis results and create visualization components
|
219 |
+
|
220 |
+
Args:
|
221 |
+
analysis_results (dict): The analysis results
|
222 |
+
|
223 |
+
Returns:
|
224 |
+
list: List of gradio components for visualization
|
225 |
+
"""
|
226 |
+
try:
|
227 |
+
print(f"Starting visualization of bias detection analysis results")
|
228 |
+
return create_bias_visualization(analysis_results)
|
229 |
+
except Exception as e:
|
230 |
+
import traceback
|
231 |
+
error_msg = f"Bias detection visualization error: {str(e)}\n{traceback.format_exc()}"
|
232 |
+
print(error_msg)
|
233 |
+
return [gr.Markdown(f"**Error during bias detection visualization:**\n\n```\n{error_msg}\n```")]
|