Update prepare_everything.py

prepare_everything.py

@@ -1,14 +1,5 @@
-import os
-import subprocess
-import sys
-from pathlib import Path
-
 import requests
 
-from parltopic.utils.helper import get_main_config
-
-config = get_main_config()
-
 
 def download_model(url, save_path):
     # Send a GET request to the URL
@@ -26,46 +17,3 @@ def download_model(url, save_path):
         # Print an error message if the request was not successful
         print(f"Failed to download model. Status code: {response.status_code}")
 
-
-def set_tokenizers_parallelism(value):
-    """Set the TOKENIZERS_PARALLELISM environment variable."""
-    os.environ["TOKENIZERS_PARALLELISM"] = "true" if value else "false"
-    print(f"TOKENIZERS_PARALLELISM set to {os.environ['TOKENIZERS_PARALLELISM']}")
-
-
-def install_requirements():
-    """Install packages listed in requirements.txt"""
-    try:
-        subprocess.check_call(
-            [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]
-        )
-        print("All packages from requirements.txt installed successfully.")
-    except subprocess.CalledProcessError as e:
-        print(f"Failed to install packages from requirements.txt: {e}")
-        sys.exit(1)
-
-
-def install_spacy_model(model_name):
-    """Install a specific spaCy model"""
-    try:
-        subprocess.check_call([sys.executable, "-m", "spacy", "download", model_name])
-        print(f"spaCy model '{model_name}' installed successfully.")
-    except subprocess.CalledProcessError as e:
-        print(f"Failed to install spaCy model '{model_name}': {e}")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-
-    for name, path in config["paths"].items():
-        Path(path).mkdir(exist_ok=True)
-
-    download_model(
-        "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin",
-        Path(config["paths"]["resources"]) / "lid.176.bin",
-    )
-
-    install_spacy_model("de_core_news_lg")
-    install_spacy_model("fr_core_news_lg")
-    install_spacy_model("it_core_news_lg")
-    set_tokenizers_parallelism(True)