Update app.py

app.py

@@ -1,4 +1,8 @@
-
+import requests
+from bs4 import BeautifulSoup
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, Optional, Union
 from langdetect import detect as get_language
 from collections import namedtuple
 from inspect import signature
@@ -38,7 +42,6 @@ from refiners.foundationals.latent_diffusion.stable_diffusion_1.multi_upscaler i
     UpscalerCheckpoints,
 )
 from datetime import datetime
-from translate import Translator
 
 model = T5ForConditionalGeneration.from_pretrained("t5-large")
 tokenizer = T5Tokenizer.from_pretrained("t5-large")
@@ -657,63 +660,310 @@ def all_pipes(pos,neg,artist,song):
     return imgs
 
 language_codes = {
-    "af": "Afrikaans",
-    "ar": "Arabic",
-    "bg": "Bulgarian",
-    "bn": "Bengali",
-    "ca": "Catalan",
-    "cs": "Czech",
-    "cy": "Welsh",
-    "da": "Danish",
-    "de": "German",
-    "el": "Greek",
-    "en": "English",
-    "es": "Spanish",
-    "et": "Estonian",
-    "fa": "Persian (Farsi)",
-    "fi": "Finnish",
-    "fr": "French",
-    "gu": "Gujarati",
-    "he": "Hebrew",
-    "hi": "Hindi",
-    "hr": "Croatian",
-    "hu": "Hungarian",
-    "id": "Indonesian",
-    "it": "Italian",
-    "ja": "Japanese",
-    "kn": "Kannada",
-    "ko": "Korean",
-    "lt": "Lithuanian",
-    "lv": "Latvian",
-    "mk": "Macedonian",
-    "ml": "Malayalam",
-    "mr": "Marathi",
-    "ne": "Nepali",
-    "nl": "Dutch",
-    "no": "Norwegian",
-    "pa": "Punjabi",
-    "pl": "Polish",
-    "pt": "Portuguese",
-    "ro": "Romanian",
-    "ru": "Russian",
-    "sk": "Slovak",
-    "sl": "Slovenian",
-    "so": "Somali",
-    "sq": "Albanian",
-    "sv": "Swedish",
-    "sw": "Swahili",
-    "ta": "Tamil",
-    "te": "Telugu",
-    "th": "Thai",
-    "tl": "Tagalog (Filipino)",
-    "tr": "Turkish",
-    "uk": "Ukrainian",
-    "ur": "Urdu",
-    "vi": "Vietnamese",
-    "zh-cn": "Chinese (Simplified)",
-    "zh-tw": "Chinese (Traditional)",
+    "afrikaans": "af",
+    "albanian": "sq",
+    "amharic": "am",
+    "arabic": "ar",
+    "armenian": "hy",
+    "assamese": "as",
+    "aymara": "ay",
+    "azerbaijani": "az",
+    "bambara": "bm",
+    "basque": "eu",
+    "belarusian": "be",
+    "bengali": "bn",
+    "bhojpuri": "bho",
+    "bosnian": "bs",
+    "bulgarian": "bg",
+    "catalan": "ca",
+    "cebuano": "ceb",
+    "chichewa": "ny",
+    "chinese (simplified)": "zh-CN",
+    "chinese (traditional)": "zh-TW",
+    "corsican": "co",
+    "croatian": "hr",
+    "czech": "cs",
+    "danish": "da",
+    "dhivehi": "dv",
+    "dogri": "doi",
+    "dutch": "nl",
+    "english": "en",
+    "esperanto": "eo",
+    "estonian": "et",
+    "ewe": "ee",
+    "filipino": "tl",
+    "finnish": "fi",
+    "french": "fr",
+    "frisian": "fy",
+    "galician": "gl",
+    "georgian": "ka",
+    "german": "de",
+    "greek": "el",
+    "guarani": "gn",
+    "gujarati": "gu",
+    "haitian creole": "ht",
+    "hausa": "ha",
+    "hawaiian": "haw",
+    "hebrew": "iw",
+    "hindi": "hi",
+    "hmong": "hmn",
+    "hungarian": "hu",
+    "icelandic": "is",
+    "igbo": "ig",
+    "ilocano": "ilo",
+    "indonesian": "id",
+    "irish": "ga",
+    "italian": "it",
+    "japanese": "ja",
+    "javanese": "jw",
+    "kannada": "kn",
+    "kazakh": "kk",
+    "khmer": "km",
+    "kinyarwanda": "rw",
+    "konkani": "gom",
+    "korean": "ko",
+    "krio": "kri",
+    "kurdish (kurmanji)": "ku",
+    "kurdish (sorani)": "ckb",
+    "kyrgyz": "ky",
+    "lao": "lo",
+    "latin": "la",
+    "latvian": "lv",
+    "lingala": "ln",
+    "lithuanian": "lt",
+    "luganda": "lg",
+    "luxembourgish": "lb",
+    "macedonian": "mk",
+    "maithili": "mai",
+    "malagasy": "mg",
+    "malay": "ms",
+    "malayalam": "ml",
+    "maltese": "mt",
+    "maori": "mi",
+    "marathi": "mr",
+    "meiteilon (manipuri)": "mni-Mtei",
+    "mizo": "lus",
+    "mongolian": "mn",
+    "myanmar": "my",
+    "nepali": "ne",
+    "norwegian": "no",
+    "odia (oriya)": "or",
+    "oromo": "om",
+    "pashto": "ps",
+    "persian": "fa",
+    "polish": "pl",
+    "portuguese": "pt",
+    "punjabi": "pa",
+    "quechua": "qu",
+    "romanian": "ro",
+    "russian": "ru",
+    "samoan": "sm",
+    "sanskrit": "sa",
+    "scots gaelic": "gd",
+    "sepedi": "nso",
+    "serbian": "sr",
+    "sesotho": "st",
+    "shona": "sn",
+    "sindhi": "sd",
+    "sinhala": "si",
+    "slovak": "sk",
+    "slovenian": "sl",
+    "somali": "so",
+    "spanish": "es",
+    "sundanese": "su",
+    "swahili": "sw",
+    "swedish": "sv",
+    "tajik": "tg",
+    "tamil": "ta",
+    "tatar": "tt",
+    "telugu": "te",
+    "thai": "th",
+    "tigrinya": "ti",
+    "tsonga": "ts",
+    "turkish": "tr",
+    "turkmen": "tk",
+    "twi": "ak",
+    "ukrainian": "uk",
+    "urdu": "ur",
+    "uyghur": "ug",
+    "uzbek": "uz",
+    "vietnamese": "vi",
+    "welsh": "cy",
+    "xhosa": "xh",
+    "yiddish": "yi",
+    "yoruba": "yo",
+    "zulu": "zu",
 }
 
+class BaseTranslator(ABC):
+    """
+    Abstract class that serve as a base translator for other different translators
+    """
+
+    def __init__(
+        self,
+        base_url: str = None,
+        languages: dict = language_codes,
+        source: str = "auto",
+        target: str = "en",
+        payload_key: Optional[str] = None,
+        element_tag: Optional[str] = None,
+        element_query: Optional[dict] = None,
+        **url_params,
+    ):
+        """
+        @param source: source language to translate from
+        @param target: target language to translate to
+        """
+        self._base_url = base_url
+        self._languages = languages
+        self._supported_languages = list(self._languages.keys())
+        if not source:
+            raise InvalidSourceOrTargetLanguage(source)
+        if not target:
+            raise InvalidSourceOrTargetLanguage(target)
+
+        self._source, self._target = self._map_language_to_code(source, target)
+        self._url_params = url_params
+        self._element_tag = element_tag
+        self._element_query = element_query
+        self.payload_key = payload_key
+        super().__init__()
+
+    @property
+    def source(self):
+        return self._source
+
+    @source.setter
+    def source(self, lang):
+        self._source = lang
+
+    @property
+    def target(self):
+        return self._target
+
+    @target.setter
+    def target(self, lang):
+        self._target = lang
+
+    def _type(self):
+        return self.__class__.__name__
+
+    def _map_language_to_code(self, *languages):
+        """
+        map language to its corresponding code (abbreviation) if the language was passed
+        by its full name by the user
+        @param languages: list of languages
+        @return: mapped value of the language or raise an exception if the language is
+        not supported
+        """
+        for language in languages:
+            if language in self._languages.values() or language == "auto":
+                yield language
+            elif language in self._languages.keys():
+                yield self._languages[language]
+            else:
+                raise LanguageNotSupportedException(
+                    language,
+                    message=f"No support for the provided language.\n"
+                    f"Please select on of the supported languages:\n"
+                    f"{self._languages}",
+                )
+
+    def _same_source_target(self) -> bool:
+        return self._source == self._target
+
+    def get_supported_languages(
+        self, as_dict: bool = False, **kwargs
+    ) -> Union[list, dict]:
+        """
+        return the supported languages by the Google translator
+        @param as_dict: if True, the languages will be returned as a dictionary
+        mapping languages to their abbreviations
+        @return: list or dict
+        """
+        return self._supported_languages if not as_dict else self._languages
+
+    def is_language_supported(self, language: str, **kwargs) -> bool:
+        """
+        check if the language is supported by the translator
+        @param language: a string for 1 language
+        @return: bool or raise an Exception
+        """
+        if (
+            language == "auto"
+            or language in self._languages.keys()
+            or language in self._languages.values()
+        ):
+            return True
+        else:
+            return False
+
+    @abstractmethod
+    def translate(self, text: str, **kwargs) -> str:
+        """
+        translate a text using a translator under the hood and return
+        the translated text
+        @param text: text to translate
+        @param kwargs: additional arguments
+        @return: str
+        """
+        return NotImplemented("You need to implement the translate method!")
+
+    def _read_docx(self, f: str):
+        import docx2txt
+
+        return docx2txt.process(f)
+
+    def _read_pdf(self, f: str):
+        import pypdf
+
+        reader = pypdf.PdfReader(f)
+        page = reader.pages[0]
+        return page.extract_text()
+
+    def _translate_file(self, path: str, **kwargs) -> str:
+        """
+        translate directly from file
+        @param path: path to the target file
+        @type path: str
+        @param kwargs: additional args
+        @return: str
+        """
+        if not isinstance(path, Path):
+            path = Path(path)
+
+        if not path.exists():
+            print("Path to the file is wrong!")
+            exit(1)
+
+        ext = path.suffix
+
+        if ext == ".docx":
+            text = self._read_docx(f=str(path))
+
+        elif ext == ".pdf":
+            text = self._read_pdf(f=str(path))
+        else:
+            with open(path, "r", encoding="utf-8") as f:
+                text = f.read().strip()
+
+        return self.translate(text)
+
+    def _translate_batch(self, batch: List[str], **kwargs) -> List[str]:
+        """
+        translate a list of texts
+        @param batch: list of texts you want to translate
+        @return: list of translations
+        """
+        if not batch:
+            raise Exception("Enter your text list that you want to translate")
+        arr = []
+        for i, text in enumerate(batch):
+            translated = self.translate(text, **kwargs)
+            arr.append(translated)
+        return arr
+    
 def translate(txt,to_lang="en",from_lang=False):
     log(f'CALL translate')
     if not from_lang:
@@ -721,7 +971,10 @@ def translate(txt,to_lang="en",from_lang=False):
     if(from_lang == to_lang):
         log(f'RET translate with txt as {txt}')
         return txt
-        
+
+
+    translator_endpoint = "https://api.cognitive.microsofttranslator.com/translate?api-version=3.0"
+    
     translator = Translator(from_lang=from_lang,to_lang=to_lang)
     translation = ""
     if len(txt) > 490: