Update app.py

app.py

@@ -1,3 +1,4 @@
+# app_updated_with_filter_sets.py
 import re
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline
@@ -29,6 +30,15 @@ tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModelForTokenClassification.from_pretrained(model_name)
 ner_pipeline = pipeline("ner", model=model, tokenizer=tokenizer, aggregation_strategy="simple")
 
+# ✅ 예외 필터
+NAME_ENTITY_EXCEPTIONS = set([
+    '법적', '군의', '사회적', '심리적', '행정적', '의료적', '법률적',
+    '개인정보', '본인', '해당', '현재', '아래', '위치', '소속'
+])
+REGEX_KEYWORDS_TO_MASK = set([
+    '이메일', '전화번호', '연락처', '주소', '센터', '카드번호', '주민등록번호', 'IP', 'IP주소'
+])
+
 def extract_names(text):
     try:
         results = ner_pipeline(text)
@@ -37,17 +47,10 @@ def extract_names(text):
         return []
 
     names = []
-
-    NAME_FILTER_EXCEPTIONS = set([
-        '법적', '군의', '사회적', '심리적', '행정적', '의료적', '법률적',
-        '개인정보', '본인', '직통번호', '이메일', '전화번호', '연락처', '주소',
-        '해당', '현재', '상담', '예약', '센터', '아래', '위치', '소속', 
-    ])
-
     for entity in results:
         if entity.get("entity_group") == "PS":
             name = entity["word"].replace("##", "").strip()
-            if len(name) >= 2 and name not in names and name not in NAME_FILTER_EXCEPTIONS:
+            if len(name) >= 2 and name not in names and name not in NAME_ENTITY_EXCEPTIONS:
                 names.append(name)
 
     COMMON_SUFFIXES = [
@@ -66,21 +69,15 @@ def extract_names(text):
     ]
 
     KOREAN_JOSA = r'(이[가]|은|는|을|를|과|와|의|도|만|께서|에서|으로|에게|한테|보다|까지|부터)?'
-
     attached_pattern = r'([가-힣]{2,4})(' + '|'.join(COMMON_SUFFIXES) + r')' + KOREAN_JOSA
-    attached_matches = re.findall(attached_pattern, text)
-    for match in attached_matches:
-        name = match[0]
-        if name not in names and name not in NAME_FILTER_EXCEPTIONS:
-            names.append(name)
-
     spaced_pattern = r'([가-힣]{2,4})\s+(' + '|'.join(COMMON_SUFFIXES) + r')' + KOREAN_JOSA
-    spaced_matches = re.findall(spaced_pattern, text)
-    for match in spaced_matches:
-        name = match[0]
-        if name not in names and name not in NAME_FILTER_EXCEPTIONS:
-            names.append(name)
 
+    for pattern in [attached_pattern, spaced_pattern]:
+        matches = re.findall(pattern, text)
+        for match in matches:
+            name = match[0]
+            if name not in names and name not in NAME_ENTITY_EXCEPTIONS:
+                names.append(name)
     return names
 
 def to_chosung(text):
@@ -95,13 +92,10 @@ def to_chosung(text):
             result += ch
     return result
 
-
-
 def mask_department(text):
     text = re.sub(r"([가-힣]{2,20}학과)", lambda m: to_chosung(m.group(1)[:-2]) + "학과", text)
     return text
 
-
 def sanitize_sensitive_info(text, keyword_string, replace_word):
     text = mask_school_names(text)
     text = mask_department(text)
@@ -110,9 +104,11 @@ def sanitize_sensitive_info(text, keyword_string, replace_word):
     text = re.sub(r"(\d)학년\s?(\d)반", r"*학년 *반", text)
 
     keywords = [k.strip() for k in keyword_string.split(",") if k.strip()]
+    keywords += list(REGEX_KEYWORDS_TO_MASK)
     for kw in keywords:
         pattern = rf"\b{re.escape(kw)}\b"
         text = re.sub(pattern, replace_word, text, flags=re.IGNORECASE)
+
     text = re.sub(r"(\d{3})-(\d{4})-(\d{4})", r"\1-****-\3", text)
     text = re.sub(r"(\d{4})년 (\d{1,2})월 (\d{1,2})일", r"19**년 \2월 *일", text)
     text = re.sub(r"(\d{1,3})번지", r"***번지", text)
@@ -121,19 +117,45 @@ def sanitize_sensitive_info(text, keyword_string, replace_word):
     text = re.sub(r"[\w\.-]+@[\w\.-]+", r"******@****", text)
     text = re.sub(r"(\d{6})[-](\d)\d{6}", r"*******-\2*****", text)
     text = re.sub(r"([가-힣]+(대로|로|길))\s?(\d+)(호|번길|가)?", r"\1 ***", text)
-    text = re.sub(r"(\d{2,6})[-]?(\d{2,6})[-]?(\d{2,6})",
-                  lambda m: f"{m.group(1)[:2]}{'*'*(len(m.group(1))-2)}{'*'*len(m.group(2))}{m.group(3)[-4:]}", text)
     text = re.sub(r"(\d{4})[- ]?(\d{4})[- ]?(\d{4})[- ]?(\d{4})",
                   lambda m: f"{m.group(1)}-****-****-{m.group(4)}", text)
     return text
 
-
 def final_name_remask_exact_only(text, mapping_dict):
     for tag, name in mapping_dict.items():
         pattern = rf'(?<![\w가-힣]){re.escape(name)}(?![\w가-힣])'
         text = re.sub(pattern, tag, text)
     return text
 
+def refactored_mask_names(original_text, names, start_counter=100):
+    korean_josa = ['이가','를','은','는','을','도','만','과','와','에게','에서','으로',
+                   '까지','조차','마저','이며','이다','이나','이나마','밖에','이든','이라도',
+                   '이','가','의']
+    masked = original_text
+    mapping = {}
+    counter = start_counter
+    used_names = set()
+    for name in names:
+        for josa in korean_josa:
+            full = name + josa
+            pattern = rf'(?<![\w가-힣]){re.escape(full)}(?![\w가-힣])'
+            if re.search(pattern, masked):
+                tag = f"N{counter:03d}"
+                mapping[tag] = name
+                masked = re.sub(pattern, tag + josa, masked)
+                counter += 1
+                used_names.add(name)
+                break
+    for name in names:
+        if name in used_names:
+            continue
+        pattern = rf'(?<![\w가-힣]){re.escape(name)}(?![\w가-힣])'
+        if re.search(pattern, masked):
+            tag = f"N{counter:03d}"
+            mapping[tag] = name
+            masked = re.sub(pattern, tag, masked)
+            counter += 1
+    return masked, mapping
 
 def apply_masking(text, keywords, replace_word):
     names = extract_names(text)
@@ -143,7 +165,6 @@ def apply_masking(text, keywords, replace_word):
     mapping_table = "\n".join([f"{k} → {v}" for k, v in mapping.items()])
     return sanitized, mapping_table
 
-
 with gr.Blocks() as demo:
     gr.Markdown("""
 🛡️ **민감정보 마스킹 [땡땡이 마스킹]**