|
|
"""Set of default text cleaners"""
|
|
|
|
|
|
|
|
|
import re
|
|
|
from .french_abbreviations import abbreviations_fr
|
|
|
|
|
|
|
|
|
_whitespace_re = re.compile(r"\s+")
|
|
|
|
|
|
|
|
|
rep_map = {
|
|
|
"：": ",",
|
|
|
"；": ",",
|
|
|
"，": ",",
|
|
|
"。": ".",
|
|
|
"！": "!",
|
|
|
"？": "?",
|
|
|
"\n": ".",
|
|
|
"·": ",",
|
|
|
"、": ",",
|
|
|
"...": ".",
|
|
|
"…": ".",
|
|
|
"$": ".",
|
|
|
"“": "",
|
|
|
"”": "",
|
|
|
"‘": "",
|
|
|
"’": "",
|
|
|
"（": "",
|
|
|
"）": "",
|
|
|
"(": "",
|
|
|
")": "",
|
|
|
"《": "",
|
|
|
"》": "",
|
|
|
"【": "",
|
|
|
"】": "",
|
|
|
"[": "",
|
|
|
"]": "",
|
|
|
"—": "",
|
|
|
"～": "-",
|
|
|
"~": "-",
|
|
|
"「": "",
|
|
|
"」": "",
|
|
|
"¿" : "",
|
|
|
"¡" : ""
|
|
|
}
|
|
|
|
|
|
|
|
|
def replace_punctuation(text):
|
|
|
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
|
|
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
|
|
return replaced_text
|
|
|
|
|
|
def expand_abbreviations(text, lang="fr"):
|
|
|
if lang == "fr":
|
|
|
_abbreviations = abbreviations_fr
|
|
|
for regex, replacement in _abbreviations:
|
|
|
text = re.sub(regex, replacement, text)
|
|
|
return text
|
|
|
|
|
|
|
|
|
def lowercase(text):
|
|
|
return text.lower()
|
|
|
|
|
|
|
|
|
def collapse_whitespace(text):
|
|
|
return re.sub(_whitespace_re, " ", text).strip()
|
|
|
|
|
|
def remove_punctuation_at_begin(text):
|
|
|
return re.sub(r'^[,.!?]+', '', text)
|
|
|
|
|
|
def remove_aux_symbols(text):
|
|
|
text = re.sub(r"[\<\>\(\)\[\]\"\«\»]+", "", text)
|
|
|
return text
|
|
|
|
|
|
|
|
|
def replace_symbols(text, lang="en"):
|
|
|
"""Replace symbols based on the lenguage tag.
|
|
|
|
|
|
Args:
|
|
|
text:
|
|
|
Input text.
|
|
|
lang:
|
|
|
Lenguage identifier. ex: "en", "fr", "pt", "ca".
|
|
|
|
|
|
Returns:
|
|
|
The modified text
|
|
|
example:
|
|
|
input args:
|
|
|
text: "si l'avi cau, diguem-ho"
|
|
|
lang: "ca"
|
|
|
Output:
|
|
|
text: "si lavi cau, diguemho"
|
|
|
"""
|
|
|
text = text.replace(";", ",")
|
|
|
text = text.replace("-", " ") if lang != "ca" else text.replace("-", "")
|
|
|
text = text.replace(":", ",")
|
|
|
if lang == "en":
|
|
|
text = text.replace("&", " and ")
|
|
|
elif lang == "fr":
|
|
|
text = text.replace("&", " et ")
|
|
|
elif lang == "pt":
|
|
|
text = text.replace("&", " e ")
|
|
|
elif lang == "ca":
|
|
|
text = text.replace("&", " i ")
|
|
|
text = text.replace("'", "")
|
|
|
elif lang== "es":
|
|
|
text=text.replace("&","y")
|
|
|
text = text.replace("'", "")
|
|
|
return text
|
|
|
|
|
|
def french_cleaners(text):
|
|
|
"""Pipeline for French text. There is no need to expand numbers, phonemizer already does that"""
|
|
|
text = expand_abbreviations(text, lang="fr")
|
|
|
|
|
|
text = replace_punctuation(text)
|
|
|
text = replace_symbols(text, lang="fr")
|
|
|
text = remove_aux_symbols(text)
|
|
|
text = remove_punctuation_at_begin(text)
|
|
|
text = collapse_whitespace(text)
|
|
|
text = re.sub(r'([^\.,!\?\-…])$', r'\1.', text)
|
|
|
return text
|
|
|
|
|
|
|
