Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
|
@@ -62,9 +62,18 @@ def custom_sentiment(text):
|
|
| 62 |
score = probs[0][label_idx].item()
|
| 63 |
return {"label": label, "score": score}
|
| 64 |
|
| 65 |
-
def calculate_abuse_level(scores, thresholds):
|
| 66 |
-
triggered_scores = [
|
| 67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
def interpret_abuse_level(score):
|
| 70 |
if score > 80:
|
|
@@ -110,7 +119,7 @@ def analyze_messages(input_text, risk_flags):
|
|
| 110 |
if non_abusive_score > adjusted_thresholds['non_abusive']:
|
| 111 |
return "This message is classified as non-abusive."
|
| 112 |
|
| 113 |
-
abuse_level = calculate_abuse_level(scores, adjusted_thresholds)
|
| 114 |
abuse_description = interpret_abuse_level(abuse_level)
|
| 115 |
|
| 116 |
if danger_flag_count >= 2:
|
|
|
|
| 62 |
score = probs[0][label_idx].item()
|
| 63 |
return {"label": label, "score": score}
|
| 64 |
|
| 65 |
+
def calculate_abuse_level(scores, thresholds, motif_hits=None):
|
| 66 |
+
triggered_scores = [
|
| 67 |
+
score for label, score in zip(LABELS, scores) if score > thresholds[label]
|
| 68 |
+
]
|
| 69 |
+
base_score = round(np.mean(triggered_scores) * 100, 2) if triggered_scores else 0.0
|
| 70 |
+
|
| 71 |
+
# Boost score if high-risk motifs were detected
|
| 72 |
+
motif_hits = motif_hits or []
|
| 73 |
+
if any(label in motif_hits for label in {"physical_threat", "suicidal_threat", "extreme_control"}):
|
| 74 |
+
base_score = max(base_score, 60.0) # Push to "Severe / Harmful Pattern Present"
|
| 75 |
+
|
| 76 |
+
return base_score
|
| 77 |
|
| 78 |
def interpret_abuse_level(score):
|
| 79 |
if score > 80:
|
|
|
|
| 119 |
if non_abusive_score > adjusted_thresholds['non_abusive']:
|
| 120 |
return "This message is classified as non-abusive."
|
| 121 |
|
| 122 |
+
abuse_level = calculate_abuse_level(scores, adjusted_thresholds, motif_hits=[label for label, _ in matched_phrases])
|
| 123 |
abuse_description = interpret_abuse_level(abuse_level)
|
| 124 |
|
| 125 |
if danger_flag_count >= 2:
|