SamanthaStorm commited on
Commit
313c859
·
verified ·
1 Parent(s): c7df978

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +5 -2
app.py CHANGED
@@ -648,8 +648,11 @@ def analyze_composite(msg1, date1, msg2, date2, msg3, date3, *answers_and_none):
648
  import re
649
 
650
  def normalize(text):
651
- # Lowercase, strip whitespace, and remove most punctuation
652
- return re.sub(r"[^a-z0-9 ]", "", text.lower().strip())
 
 
 
653
 
654
  def detect_threat_motifs(message, motif_list):
655
  norm_msg = normalize(message)
 
648
  import re
649
 
650
  def normalize(text):
651
+ import unicodedata
652
+ text = text.lower().strip()
653
+ text = unicodedata.normalize("NFKD", text) # handles curly quotes
654
+ text = text.replace("’", "'") # smart to straight
655
+ return re.sub(r"[^a-z0-9 ]", "", text)
656
 
657
  def detect_threat_motifs(message, motif_list):
658
  norm_msg = normalize(message)