Update app.py

app.py

@@ -13,13 +13,13 @@ os.environ["MACRO_CORRECT_FLAG_CSC_TOKEN"] = "1"
 os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
 os.environ["USE_TORCH"] = "1"
 
-from macro_correct.pytorch_textcorrection.tcTools import cut_sent_by_stay
+from macro_correct.pytorch_textcorrection.tcTools import preprocess_same_with_training
 from macro_correct import correct_basic
 from macro_correct import correct_long
 from macro_correct import correct
 import gradio as gr
 
-
+# pyinstaller -F xxxx.py
 
 # pretrained_model_name_or_path = "shibing624/macbert4csc-base-chinese"
 pretrained_model_name_or_path = "Macadam/macbert4mdcspell_v2"
@@ -31,9 +31,54 @@ pretrained_model_name_or_path = "Macadam/macbert4mdcspell_v2"
 # device = torch.device("cuda")
 
 
+def cut_sent_by_stay_and_maxlen(text, max_len=126, return_length=True):
+    """
+    分句但是保存原标点符号, 如果长度还是太长的话就切为固定长度的句子
+    Args:
+        text: str, sentence of input text;
+        max_len: int, max_len of traing texts;
+        return_length: bool, wether return length or not
+    Returns:
+        res: List<tuple>
+    """
+    ### text_sp = re.split(r"！”|？”|。”|……”|”！|”？|”。|”……|》。|）。|！|？|。|…|\!|\?", text)
+    text_sp = re.split(r"[》）！？。…”；;!?\n]+", text)
+    conn_symbol = "！？。…”；;!?》）\n"
+    text_length_s = []
+    text_cut = []
+    len_text = len(text) - 1
+    # signal_symbol = "—”>；？…）‘《’（·》“~，、！。：<"
+    len_global = 0
+    for idx, text_sp_i in enumerate(text_sp):
+        text_cut_idx = text_sp[idx]
+        len_global_before = copy.deepcopy(len_global)
+        len_global += len(text_sp_i)
+        while True:
+            if len_global <= len_text and text[len_global] in conn_symbol:
+                text_cut_idx += text[len_global]
+            else:
+                # len_global += 1
+                if text_cut_idx:
+                    ### 如果标点符号依旧切分不了, 就强行切
+                    if len(text_cut_idx) > max_len:
+                        text_cut_i, text_length_s_i = cut_sent_by_maxlen(
+                            text=text, max_len=max_len, return_length=True)
+                        text_length_s.extend(text_length_s_i)
+                        text_cut.extend(text_cut_i)
+                    else:
+                        text_length_s.append([len_global_before, len_global])
+                        text_cut.append(text_cut_idx)
+                break
+            len_global += 1
+    if return_length:
+        return text_cut, text_length_s
+    return text_cut
+
+
 def macro_correct(text):
     print(text)
-    texts, texts_length = cut_sent_by_stay(text, return_length=True, add_semicolon=True)
+    text = preprocess_same_with_training(text)
+    texts, texts_length = cut_sent_by_stay_and_maxlen(text, return_length=True)
     text_str = ""
     text_list = []
     for t in texts:
@@ -73,10 +118,16 @@ if __name__ == '__main__':
         macro_correct,
         inputs='text',
         outputs='text',
-        title="Chinese Spelling Correction Model Macropodus/macbert4csc_v2",
+        title="Chinese Spelling Correction Model Macropodus/macbert4mdcspell_v2",
         description="Copy or input error Chinese text. Submit and the machine will correct text.",
         article="Link to <a href='https://github.com/yongzhuo/macro-correct' style='color:blue;' target='_blank\'>Github REPO: macro-correct</a>",
         examples=examples
     ).launch()
     # ).launch(server_name="0.0.0.0", server_port=8066, share=False, debug=True)
 
+"""
+赤热的阳光烘烤大地，婵鸣撕破树荫的宁净。少年咬下鲜红西瓜，糖汁溶化在沙摊上。孩童举着冰其淋奔跑，浪花打湿嘻闹的脚丫。威风卷起碎花裙摆，牵牛花在篱笆绽方。这个季结，连空气都浸着清谅的甜。
+炽热的阳光烘烤大地,蝉鸣撕破树荫的宁静。少年咬下鲜红西瓜,糖汁溶化在沙滩上。孩童举着冰淇淋奔跑,浪花打湿嬉闹的脚丫。微风卷起碎花裙摆,牵牛花在篱笆绽放。这个季节,连空气都浸着清凉的甜。
+
+pyinstaller --onefile app_macro_correct_std.py
+"""