Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -1,105 +1,86 @@
|
|
1 |
import gradio as gr
|
2 |
import torch
|
3 |
-
from transformers import RobertaForSequenceClassification, RobertaTokenizer
|
4 |
import numpy as np
|
5 |
-
from transformers import
|
|
|
6 |
|
7 |
-
# Load sentiment
|
8 |
-
|
|
|
|
|
9 |
|
10 |
-
# Load
|
11 |
model_name = "SamanthaStorm/abuse-pattern-detector-v2"
|
12 |
model = RobertaForSequenceClassification.from_pretrained(model_name, trust_remote_code=True)
|
13 |
tokenizer = RobertaTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
14 |
|
15 |
-
# Define labels (18 total)
|
16 |
LABELS = [
|
17 |
-
"gaslighting", "mockery", "dismissiveness", "control",
|
18 |
-
"
|
19 |
-
"
|
20 |
-
"obscure_formal", "recovery_phase", "non_abusive", "suicidal_threat", "physical_threat",
|
21 |
-
"extreme_control"
|
22 |
]
|
23 |
|
24 |
-
# Custom thresholds for each label
|
25 |
THRESHOLDS = {
|
26 |
-
"gaslighting": 0.25,
|
27 |
-
"
|
28 |
-
"
|
29 |
-
"
|
30 |
-
"guilt_tripping": 0.19,
|
31 |
-
"apology_baiting": 0.45,
|
32 |
-
"blame_shifting": 0.23,
|
33 |
-
"projection": 0.50,
|
34 |
-
"contradictory_statements": 0.25,
|
35 |
-
"manipulation": 0.25,
|
36 |
-
"deflection": 0.30,
|
37 |
-
"insults": 0.34,
|
38 |
-
"obscure_formal": 0.25,
|
39 |
-
"recovery_phase": 0.25,
|
40 |
-
"non_abusive": 2.0,
|
41 |
-
"suicidal_threat": 0.45,
|
42 |
-
"physical_threat": 0.10,
|
43 |
-
"extreme_control": 0.36
|
44 |
}
|
45 |
|
46 |
PATTERN_LABELS = LABELS[:15]
|
47 |
DANGER_LABELS = LABELS[15:18]
|
48 |
|
49 |
EXPLANATIONS = {
|
50 |
-
"gaslighting": "Gaslighting involves making someone question their own reality or perceptions
|
51 |
-
"blame_shifting": "Blame-shifting is when one person redirects the responsibility
|
52 |
-
"projection": "Projection involves accusing the victim of behaviors
|
53 |
-
"dismissiveness": "Dismissiveness is
|
54 |
-
"mockery": "Mockery
|
55 |
-
"recovery_phase": "Recovery phase
|
56 |
-
"insults": "Insults are derogatory remarks aimed at degrading
|
57 |
-
"apology_baiting": "Apology-baiting
|
58 |
-
"deflection": "Deflection
|
59 |
-
"control": "Control
|
60 |
-
"extreme_control": "Extreme control
|
61 |
-
"physical_threat": "Physical threats
|
62 |
-
"suicidal_threat": "Suicidal threats
|
63 |
-
"guilt_tripping": "Guilt-tripping
|
64 |
-
"
|
65 |
-
"
|
66 |
-
"
|
67 |
-
"obscure_formal": "Obscure or overly formal language used manipulatively to create confusion, avoid responsibility, or assert superiority."
|
68 |
}
|
69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
70 |
|
71 |
def calculate_abuse_level(scores, thresholds):
|
72 |
triggered_scores = [score for label, score in zip(LABELS, scores) if score > thresholds[label]]
|
73 |
-
if
|
74 |
-
return 0.0
|
75 |
-
return round(np.mean(triggered_scores) * 100, 2)
|
76 |
-
|
77 |
|
78 |
def interpret_abuse_level(score):
|
79 |
-
if score > 80:
|
80 |
-
|
81 |
-
elif score >
|
82 |
-
|
83 |
-
|
84 |
-
return "Likely Abuse"
|
85 |
-
elif score > 20:
|
86 |
-
return "Mild Concern"
|
87 |
-
else:
|
88 |
-
return "Very Low / Likely Safe"
|
89 |
-
|
90 |
|
91 |
def analyze_messages(input_text, risk_flags):
|
92 |
input_text = input_text.strip()
|
93 |
if not input_text:
|
94 |
return "Please enter a message for analysis."
|
95 |
|
96 |
-
sentiment =
|
97 |
sentiment_label = sentiment['label']
|
98 |
sentiment_score = sentiment['score']
|
99 |
|
100 |
-
adjusted_thresholds = THRESHOLDS.copy()
|
101 |
-
if sentiment_label == "NEGATIVE":
|
102 |
-
adjusted_thresholds = {key: val * 0.8 for key, val in THRESHOLDS.items()}
|
103 |
|
104 |
inputs = tokenizer(input_text, return_tensors="pt", truncation=True, padding=True)
|
105 |
with torch.no_grad():
|
@@ -110,8 +91,7 @@ def analyze_messages(input_text, risk_flags):
|
|
110 |
danger_flag_count = sum(score > adjusted_thresholds[label] for label, score in zip(DANGER_LABELS, scores[15:18]))
|
111 |
|
112 |
contextual_flags = risk_flags if risk_flags else []
|
113 |
-
|
114 |
-
if contextual_risk_score >= 2:
|
115 |
danger_flag_count += 1
|
116 |
|
117 |
critical_flags = ["They've threatened harm", "They monitor/follow me", "I feel unsafe when alone with them"]
|
@@ -141,10 +121,8 @@ def analyze_messages(input_text, risk_flags):
|
|
141 |
f"Abuse Risk Score: {abuse_level}% – {abuse_description}\n\n"
|
142 |
f"Most Likely Patterns:\n{top_pattern_explanations}\n\n"
|
143 |
f"⚠️ Critical Danger Flags Detected: {danger_flag_count} of 3\n"
|
144 |
-
"The Danger Assessment is a validated tool that helps identify serious risk in intimate partner violence
|
145 |
-
"
|
146 |
-
"For more info, consider reaching out to support groups or professionals.\n\n"
|
147 |
-
f"Resources: {resources} \n\n"
|
148 |
f"Sentiment: {sentiment_label} (Confidence: {sentiment_score*100:.2f}%)"
|
149 |
)
|
150 |
|
@@ -155,22 +133,16 @@ def analyze_messages(input_text, risk_flags):
|
|
155 |
|
156 |
return result
|
157 |
|
158 |
-
|
159 |
iface = gr.Interface(
|
160 |
fn=analyze_messages,
|
161 |
inputs=[
|
162 |
gr.Textbox(lines=10, placeholder="Enter message here..."),
|
163 |
gr.CheckboxGroup(label="Do any of these apply to your situation?", choices=[
|
164 |
-
"They've threatened harm",
|
165 |
-
"They
|
166 |
-
"I’ve changed my behavior out of fear",
|
167 |
-
"They monitor/follow me",
|
168 |
-
"I feel unsafe when alone with them"
|
169 |
])
|
170 |
],
|
171 |
-
outputs=[
|
172 |
-
gr.Textbox(label="Analysis Result"),
|
173 |
-
],
|
174 |
title="Abuse Pattern Detector"
|
175 |
)
|
176 |
|
|
|
1 |
import gradio as gr
|
2 |
import torch
|
|
|
3 |
import numpy as np
|
4 |
+
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
5 |
+
from transformers import RobertaForSequenceClassification, RobertaTokenizer
|
6 |
|
7 |
+
# Load fine-tuned sentiment model
|
8 |
+
sentiment_model_path = "./sentiment_model"
|
9 |
+
sentiment_model = AutoModelForSequenceClassification.from_pretrained(sentiment_model_path)
|
10 |
+
sentiment_tokenizer = AutoTokenizer.from_pretrained(sentiment_model_path)
|
11 |
|
12 |
+
# Load abuse pattern model
|
13 |
model_name = "SamanthaStorm/abuse-pattern-detector-v2"
|
14 |
model = RobertaForSequenceClassification.from_pretrained(model_name, trust_remote_code=True)
|
15 |
tokenizer = RobertaTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
16 |
|
|
|
17 |
LABELS = [
|
18 |
+
"gaslighting", "mockery", "dismissiveness", "control", "guilt_tripping", "apology_baiting", "blame_shifting", "projection",
|
19 |
+
"contradictory_statements", "manipulation", "deflection", "insults", "obscure_formal", "recovery_phase", "non_abusive",
|
20 |
+
"suicidal_threat", "physical_threat", "extreme_control"
|
|
|
|
|
21 |
]
|
22 |
|
|
|
23 |
THRESHOLDS = {
|
24 |
+
"gaslighting": 0.25, "mockery": 0.15, "dismissiveness": 0.30, "control": 0.43, "guilt_tripping": 0.19,
|
25 |
+
"apology_baiting": 0.45, "blame_shifting": 0.23, "projection": 0.50, "contradictory_statements": 0.25,
|
26 |
+
"manipulation": 0.25, "deflection": 0.30, "insults": 0.34, "obscure_formal": 0.25, "recovery_phase": 0.25,
|
27 |
+
"non_abusive": 2.0, "suicidal_threat": 0.45, "physical_threat": 0.10, "extreme_control": 0.36
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
28 |
}
|
29 |
|
30 |
PATTERN_LABELS = LABELS[:15]
|
31 |
DANGER_LABELS = LABELS[15:18]
|
32 |
|
33 |
EXPLANATIONS = {
|
34 |
+
"gaslighting": "Gaslighting involves making someone question their own reality or perceptions...",
|
35 |
+
"blame_shifting": "Blame-shifting is when one person redirects the responsibility...",
|
36 |
+
"projection": "Projection involves accusing the victim of behaviors the abuser exhibits.",
|
37 |
+
"dismissiveness": "Dismissiveness is belittling or disregarding another person’s feelings.",
|
38 |
+
"mockery": "Mockery ridicules someone in a hurtful, humiliating way.",
|
39 |
+
"recovery_phase": "Recovery phase dismisses someone's emotional healing process.",
|
40 |
+
"insults": "Insults are derogatory remarks aimed at degrading someone.",
|
41 |
+
"apology_baiting": "Apology-baiting manipulates victims into apologizing for abuser's behavior.",
|
42 |
+
"deflection": "Deflection avoids accountability by redirecting blame.",
|
43 |
+
"control": "Control restricts autonomy through manipulation or coercion.",
|
44 |
+
"extreme_control": "Extreme control dominates decisions and behaviors entirely.",
|
45 |
+
"physical_threat": "Physical threats signal risk of bodily harm.",
|
46 |
+
"suicidal_threat": "Suicidal threats manipulate others using self-harm threats.",
|
47 |
+
"guilt_tripping": "Guilt-tripping uses guilt to manipulate someone’s actions.",
|
48 |
+
"manipulation": "Manipulation deceives to influence or control outcomes.",
|
49 |
+
"non_abusive": "Non-abusive language is respectful and free of coercion.",
|
50 |
+
"obscure_formal": "Obscure/formal language manipulates through confusion or superiority."
|
|
|
51 |
}
|
52 |
|
53 |
+
def custom_sentiment(text):
|
54 |
+
inputs = sentiment_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
|
55 |
+
with torch.no_grad():
|
56 |
+
outputs = sentiment_model(**inputs)
|
57 |
+
probs = torch.nn.functional.softmax(outputs.logits, dim=1)
|
58 |
+
label_idx = torch.argmax(probs).item()
|
59 |
+
label = sentiment_model.config.id2label[label_idx]
|
60 |
+
score = probs[0][label_idx].item()
|
61 |
+
return {"label": label, "score": score}
|
62 |
|
63 |
def calculate_abuse_level(scores, thresholds):
|
64 |
triggered_scores = [score for label, score in zip(LABELS, scores) if score > thresholds[label]]
|
65 |
+
return round(np.mean(triggered_scores) * 100, 2) if triggered_scores else 0.0
|
|
|
|
|
|
|
66 |
|
67 |
def interpret_abuse_level(score):
|
68 |
+
if score > 80: return "Extreme / High Risk"
|
69 |
+
elif score > 60: return "Severe / Harmful Pattern Present"
|
70 |
+
elif score > 40: return "Likely Abuse"
|
71 |
+
elif score > 20: return "Mild Concern"
|
72 |
+
return "Very Low / Likely Safe"
|
|
|
|
|
|
|
|
|
|
|
|
|
73 |
|
74 |
def analyze_messages(input_text, risk_flags):
|
75 |
input_text = input_text.strip()
|
76 |
if not input_text:
|
77 |
return "Please enter a message for analysis."
|
78 |
|
79 |
+
sentiment = custom_sentiment(input_text)
|
80 |
sentiment_label = sentiment['label']
|
81 |
sentiment_score = sentiment['score']
|
82 |
|
83 |
+
adjusted_thresholds = {k: v * 0.8 for k, v in THRESHOLDS.items()} if sentiment_label == "NEGATIVE" else THRESHOLDS.copy()
|
|
|
|
|
84 |
|
85 |
inputs = tokenizer(input_text, return_tensors="pt", truncation=True, padding=True)
|
86 |
with torch.no_grad():
|
|
|
91 |
danger_flag_count = sum(score > adjusted_thresholds[label] for label, score in zip(DANGER_LABELS, scores[15:18]))
|
92 |
|
93 |
contextual_flags = risk_flags if risk_flags else []
|
94 |
+
if len(contextual_flags) >= 2:
|
|
|
95 |
danger_flag_count += 1
|
96 |
|
97 |
critical_flags = ["They've threatened harm", "They monitor/follow me", "I feel unsafe when alone with them"]
|
|
|
121 |
f"Abuse Risk Score: {abuse_level}% – {abuse_description}\n\n"
|
122 |
f"Most Likely Patterns:\n{top_pattern_explanations}\n\n"
|
123 |
f"⚠️ Critical Danger Flags Detected: {danger_flag_count} of 3\n"
|
124 |
+
"The Danger Assessment is a validated tool that helps identify serious risk in intimate partner violence.\n\n"
|
125 |
+
f"Resources: {resources}\n\n"
|
|
|
|
|
126 |
f"Sentiment: {sentiment_label} (Confidence: {sentiment_score*100:.2f}%)"
|
127 |
)
|
128 |
|
|
|
133 |
|
134 |
return result
|
135 |
|
|
|
136 |
iface = gr.Interface(
|
137 |
fn=analyze_messages,
|
138 |
inputs=[
|
139 |
gr.Textbox(lines=10, placeholder="Enter message here..."),
|
140 |
gr.CheckboxGroup(label="Do any of these apply to your situation?", choices=[
|
141 |
+
"They've threatened harm", "They isolate me", "I’ve changed my behavior out of fear",
|
142 |
+
"They monitor/follow me", "I feel unsafe when alone with them"
|
|
|
|
|
|
|
143 |
])
|
144 |
],
|
145 |
+
outputs=[gr.Textbox(label="Analysis Result")],
|
|
|
|
|
146 |
title="Abuse Pattern Detector"
|
147 |
)
|
148 |
|