Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -62,9 +62,18 @@ def custom_sentiment(text):
|
|
62 |
score = probs[0][label_idx].item()
|
63 |
return {"label": label, "score": score}
|
64 |
|
65 |
-
def calculate_abuse_level(scores, thresholds):
|
66 |
-
triggered_scores = [
|
67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
68 |
|
69 |
def interpret_abuse_level(score):
|
70 |
if score > 80:
|
@@ -110,7 +119,7 @@ def analyze_messages(input_text, risk_flags):
|
|
110 |
if non_abusive_score > adjusted_thresholds['non_abusive']:
|
111 |
return "This message is classified as non-abusive."
|
112 |
|
113 |
-
abuse_level = calculate_abuse_level(scores, adjusted_thresholds)
|
114 |
abuse_description = interpret_abuse_level(abuse_level)
|
115 |
|
116 |
if danger_flag_count >= 2:
|
|
|
62 |
score = probs[0][label_idx].item()
|
63 |
return {"label": label, "score": score}
|
64 |
|
65 |
+
def calculate_abuse_level(scores, thresholds, motif_hits=None):
|
66 |
+
triggered_scores = [
|
67 |
+
score for label, score in zip(LABELS, scores) if score > thresholds[label]
|
68 |
+
]
|
69 |
+
base_score = round(np.mean(triggered_scores) * 100, 2) if triggered_scores else 0.0
|
70 |
+
|
71 |
+
# Boost score if high-risk motifs were detected
|
72 |
+
motif_hits = motif_hits or []
|
73 |
+
if any(label in motif_hits for label in {"physical_threat", "suicidal_threat", "extreme_control"}):
|
74 |
+
base_score = max(base_score, 60.0) # Push to "Severe / Harmful Pattern Present"
|
75 |
+
|
76 |
+
return base_score
|
77 |
|
78 |
def interpret_abuse_level(score):
|
79 |
if score > 80:
|
|
|
119 |
if non_abusive_score > adjusted_thresholds['non_abusive']:
|
120 |
return "This message is classified as non-abusive."
|
121 |
|
122 |
+
abuse_level = calculate_abuse_level(scores, adjusted_thresholds, motif_hits=[label for label, _ in matched_phrases])
|
123 |
abuse_description = interpret_abuse_level(abuse_level)
|
124 |
|
125 |
if danger_flag_count >= 2:
|