First commit for migrating the swiss topic modelling space.

README.md

@@ -1,10 +1,10 @@
 ---
 title: SwissParlTopicModelling
-emoji: 🦀
-colorFrom: green
+emoji: 📉
+colorFrom: indigo
 colorTo: red
 sdk: gradio
-sdk_version: 4.43.0
+sdk_version: 4.32.2
 app_file: app.py
 pinned: false
 ---

app.py

@@ -0,0 +1,100 @@
+import json as js
+import os
+import re
+from typing import List
+
+import fasttext
+import gradio as gr
+import joblib
+import omikuji
+from huggingface_hub import snapshot_download
+from install_packages import download_model
+
+download_model('https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin', 'lid.176.bin')
+
+# Download the model files from Hugging Face
+for repo_id in ['kapllan/omikuji-bonsai-parliament-de-spacy', 'kapllan/omikuji-bonsai-parliament-fr-spacy',
+                'kapllan/omikuji-bonsai-parliament-it-spacy']:
+    if not os.path.exists(repo_id):
+        os.makedirs(repo_id)
+    model_dir = snapshot_download(repo_id=repo_id, local_dir=repo_id)
+
+lang_model = fasttext.load_model('lid.176.bin')
+
+with open('./id2label.json', 'r') as f:
+    id2label = js.load(f)
+
+
+def map_language(language: str) -> str:
+    language_mapping = {'de': 'German',
+                        'it': 'Italian',
+                        'fr': 'French'}
+    if language in language_mapping.keys():
+        return language_mapping[language]
+    else:
+        return language
+
+
+def find_model(language: str):
+    vectorizer, model = None, None
+    if language in ['de', 'fr', 'it']:
+        path_to_vectorizer = f'./kapllan/omikuji-bonsai-parliament-{language}-spacy/vectorizer'
+        path_to_model = f'./kapllan/omikuji-bonsai-parliament-{language}-spacy/omikuji-model'
+        vectorizer = joblib.load(path_to_vectorizer)
+        model = omikuji.Model.load(path_to_model)
+    return vectorizer, model
+
+
+def predict_lang(text: str) -> str:
+    text = re.sub(r'\n', '', text)  # Remove linebreaks because fasttext cannot process that otherwise
+    predictions = lang_model.predict(text, k=1)  # returns top 2 matching languages
+    language = predictions[0][0]  # returns top 2 matching languages
+    language = re.sub(r'__label__', '', language)  # returns top 2 matching languages
+    return language
+
+
+def predict_topic(text: str) -> [List[str], str]:
+    results = []
+    language = predict_lang(text)
+    vectorizer, model = find_model(language)
+    language = map_language(language)
+    if vectorizer is not None:
+        texts = [text]
+        vector = vectorizer.transform(texts)
+        for row in vector:
+            if row.nnz == 0:  # All zero vector, empty result
+                continue
+            feature_values = [(col, row[0, col]) for col in row.nonzero()[1]]
+            for subj_id, score in model.predict(feature_values, top_k=1000):
+                results.append((id2label[str(subj_id)], score))
+    return results, language
+
+
+def topic_modeling(text: str, threshold: float) -> [List[str], str]:
+    # Prepare labels and scores for the plot
+    sorted_topics, language = predict_topic(text)
+    if len(sorted_topics) > 0 and language in ['German', 'French', 'Italian']:
+        sorted_topics = [t for t in sorted_topics if t[1] >= threshold]
+    else:
+        sorted_topics = []
+    return sorted_topics, language
+
+
+with gr.Blocks() as iface:
+    gr.Markdown("# Topic Modeling")
+    gr.Markdown("Enter a document and get each topic along with its score.")
+
+    with gr.Row():
+        with gr.Column():
+            input_text = gr.Textbox(lines=10, placeholder="Enter a document")
+            submit_button = gr.Button("Submit")
+            threshold_slider = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label="Score Threshold", value=0.0)
+            language_text = gr.Textbox(lines=1, placeholder="Detected language will be shown here...",
+                                       interactive=False, label="Detected Language")
+        with gr.Column():
+            output_data = gr.Dataframe(headers=["Label", "Score"])
+
+    submit_button.click(topic_modeling, inputs=[input_text, threshold_slider], outputs=[output_data, language_text])
+
+    # Launch the app
+iface.launch(share=True)

id2label.json

@@ -0,0 +1,227 @@
+{
+  "0": "AHV",
+  "1": "Abfall",
+  "2": "Abgabe",
+  "3": "Agrarpolitik",
+  "4": "Agrarproduktion",
+  "5": "Altersfragen",
+  "6": "Arbeit",
+  "7": "Arbeitslosenversicherung",
+  "8": "Arbeitslosigkeit",
+  "9": "Arbeitsmarkt",
+  "10": "Arbeitsrecht",
+  "11": "Armee",
+  "12": "Armut/Ungleichheit",
+  "13": "Artenvielfalt",
+  "14": "Asylpolitik",
+  "15": "Auslandschweizer",
+  "16": "Ausländerpolitik",
+  "17": "Aussenpolitik : Ausland",
+  "18": "Aussenpolitik : Schweiz",
+  "19": "Aussenwirtschaftspolitik",
+  "20": "Bankenkrise",
+  "21": "Bauwesen/Immobilien",
+  "22": "Behinderung",
+  "23": "Berg",
+  "24": "Berufliche Vorsorge",
+  "25": "Berufsbildung",
+  "26": "Beschwerderecht",
+  "27": "Beschäftigung und Arbeit",
+  "28": "Bewaffnung",
+  "29": "Beziehung Schweiz - EU",
+  "30": "Bildung",
+  "31": "Boden",
+  "32": "Bürgerrecht",
+  "33": "Datenschutz",
+  "34": "Demokratie",
+  "35": "Digitalisierung",
+  "36": "Diplomatie",
+  "37": "Diskriminierung",
+  "38": "Elektrizität",
+  "39": "Energie",
+  "40": "Energiepolitik",
+  "41": "Erberecht",
+  "42": "Ernährung",
+  "43": "Erwerbsersatzordnung",
+  "44": "Europapolitik",
+  "45": "Europarat",
+  "46": "Europarecht",
+  "47": "Europäische Union",
+  "48": "Europäisches Parlament",
+  "49": "Exekutive",
+  "50": "Familienfragen",
+  "51": "Familienrecht",
+  "52": "Familienzulage",
+  "53": "Finanzausgleich",
+  "54": "Finanzmarkt",
+  "55": "Finanzplatz",
+  "56": "Finanzrecht",
+  "57": "Finanzwesen",
+  "58": "Flüchtling",
+  "59": "Forschung",
+  "60": "Fortpflanzung",
+  "61": "Fossile Energie",
+  "62": "Föderalismus",
+  "63": "Geld- und Währungspolitik",
+  "64": "Geldwäscherei",
+  "65": "Gentechnologie",
+  "66": "Gerichtswesen",
+  "67": "Geschichte Ausland",
+  "68": "Geschichte Schweiz",
+  "69": "Geschlechterfragen",
+  "70": "Gesellschaftsfragen",
+  "71": "Gesundheit",
+  "72": "Gesundheitspolitik",
+  "73": "Gewalt",
+  "74": "Gewerkschaft",
+  "75": "Globalisierung",
+  "76": "Grenze",
+  "77": "Grundrechte",
+  "78": "Güterverkehr",
+  "79": "Handel",
+  "80": "Heil- und Hilfsmittel",
+  "81": "Informatik",
+  "82": "Information",
+  "83": "Informationswissenschaft",
+  "84": "Internationale Politik",
+  "85": "Internationales Recht",
+  "86": "Internet und soziale Medien",
+  "87": "Interventionspolitik",
+  "88": "Invalidenversicherung",
+  "89": "Jagd und Fischerei",
+  "90": "Kapital",
+  "91": "Katastrophe",
+  "92": "Kernenergie",
+  "93": "Kinder- und Jugendfragen",
+  "94": "Kinderrechte",
+  "95": "Kindes- und Erwachsenenschutzrecht",
+  "96": "Klimafragen",
+  "97": "Konkursrecht",
+  "98": "Konsum",
+  "99": "Korruption",
+  "100": "Krankenversicherung",
+  "101": "Krieg",
+  "102": "Krise",
+  "103": "Kultur",
+  "104": "Landwirtschaft",
+  "105": "Luft",
+  "106": "Luftfahrt",
+  "107": "Lärm",
+  "108": "Medien",
+  "109": "Medien / Kommunikation",
+  "110": "Medienrecht",
+  "111": "Medizinalberuf",
+  "112": "Menschenrechte",
+  "113": "Miet- und Wohnungswesen",
+  "114": "Migration",
+  "115": "Migrationsbewegung",
+  "116": "Mutterschaftsversicherung",
+  "117": "Nationalbank",
+  "118": "Obligationenrecht",
+  "119": "Parlament",
+  "120": "Parlament Ausland",
+  "121": "Parlament Schweiz",
+  "122": "Patient",
+  "123": "Personenrecht",
+  "124": "Pflege",
+  "125": "Post",
+  "126": "Presse",
+  "127": "Privatversicherung",
+  "128": "Produktion",
+  "129": "Radio und Fernsehen",
+  "130": "Rassismus",
+  "131": "Ratsmitglied",
+  "132": "Raumplanung",
+  "133": "Raumplanung und Wohnungswesen",
+  "134": "Recht Allgemein",
+  "135": "Rechte und Freiheiten",
+  "136": "Rechtswissenschaft",
+  "137": "Religionsfragen",
+  "138": "Sachenrecht",
+  "139": "Sans-Papiers",
+  "140": "Schiedsgerichtsbarkeit",
+  "141": "Schienenverkehr",
+  "142": "Schifffahrt",
+  "143": "Schule",
+  "144": "Service public",
+  "145": "Sicherheitspolitik",
+  "146": "Sicherheitspolitik/Friedenspolitik",
+  "147": "Soziale Fragen",
+  "148": "Sozialer Schutz",
+  "149": "Sozialhilfe",
+  "150": "Sozialpolitik",
+  "151": "Sozialversicherung",
+  "152": "Spiel",
+  "153": "Spital",
+  "154": "Sport",
+  "155": "Sprache",
+  "156": "Staat",
+  "157": "Staatspolitik",
+  "158": "Staatssouveränität",
+  "159": "Sterben und Tod",
+  "160": "Steuer",
+  "161": "Steuerhinterziehung",
+  "162": "Steuerrecht",
+  "163": "Steuerwettbewerb",
+  "164": "Stiftung",
+  "165": "Strafprozessordnung",
+  "166": "Strafrecht",
+  "167": "Straftat",
+  "168": "Strassenverkehr",
+  "169": "Sucht",
+  "170": "Telefonie",
+  "171": "Terrorismus",
+  "172": "Tierschutz",
+  "173": "Tierversuch",
+  "174": "Tourismus",
+  "175": "Umwelt",
+  "176": "Umweltpolitik",
+  "177": "Umweltschutz",
+  "178": "Unfallversicherung",
+  "179": "Universität/Hochschule/Fachhochschule",
+  "180": "Unternehmen",
+  "181": "Urheberrecht",
+  "182": "Verfahrensrecht",
+  "183": "Verfassung",
+  "184": "Vergaberecht",
+  "185": "Verkehr",
+  "186": "Verkehrspolitik",
+  "187": "Vertrag",
+  "188": "Verwaltungsrecht",
+  "189": "Volksabstimmung",
+  "190": "Vorrechte und Immunität",
+  "191": "Wahlen",
+  "192": "Wald",
+  "193": "Wasser",
+  "194": "Weiterbildung",
+  "195": "Wettbewerb",
+  "196": "Wirtschaft",
+  "197": "Wirtschaftsleben",
+  "198": "Wirtschaftspolitik",
+  "199": "Wissenschaft / Forschung",
+  "200": "Zivilprozessordnung",
+  "201": "Zivilrecht",
+  "202": "Zivilschutz und Bevölkerungsschutz/Zivildienst",
+  "203": "Zoll",
+  "204": "erneuerbare Energie",
+  "205": "innere Sicherheit",
+  "206": "internationale Beziehungen",
+  "207": "internationale Organisation",
+  "208": "internationale Politik",
+  "209": "internationale Rechtshilfe",
+  "210": "internationale Strafjustiz",
+  "211": "internationale Zusammenarbeit",
+  "212": "internationaler Konflikt",
+  "213": "internationales Abkommen",
+  "214": "internationales Privatrecht",
+  "215": "internationales Recht",
+  "216": "internationales humanitäres Recht",
+  "217": "kantonales Parlament",
+  "218": "politische Partei",
+  "219": "politische Rechte",
+  "220": "politisches Leben",
+  "221": "politisches System",
+  "222": "öffentliche Finanzen",
+  "223": "öffentliche Verwaltung",
+  "224": "öffentlicher Verkehr"
+}

install_packages.py

@@ -0,0 +1,57 @@
+import os
+import subprocess
+import sys
+
+import requests
+
+
+def download_model(url, save_path):
+    # Send a GET request to the URL
+    response = requests.get(url, stream=True)
+
+    # Check if the request was successful (status code 200)
+    if response.status_code == 200:
+        # Open a file in binary write mode to save the downloaded content
+        with open(save_path, 'wb') as f:
+            # Iterate over the response content in chunks and write to the file
+            for chunk in response.iter_content(chunk_size=1024):
+                f.write(chunk)
+        print("Model downloaded successfully!")
+    else:
+        # Print an error message if the request was not successful
+        print(f"Failed to download model. Status code: {response.status_code}")
+
+
+def set_tokenizers_parallelism(value):
+    """Set the TOKENIZERS_PARALLELISM environment variable."""
+    os.environ['TOKENIZERS_PARALLELISM'] = 'true' if value else 'false'
+    print(f"TOKENIZERS_PARALLELISM set to {os.environ['TOKENIZERS_PARALLELISM']}")
+
+
+def install_requirements():
+    """Install packages listed in requirements.txt"""
+    try:
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"])
+        print("All packages from requirements.txt installed successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to install packages from requirements.txt: {e}")
+        sys.exit(1)
+
+
+def install_spacy_model(model_name):
+    """Install a specific spaCy model"""
+    try:
+        subprocess.check_call([sys.executable, "-m", "spacy", "download", model_name])
+        print(f"spaCy model '{model_name}' installed successfully.")
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to install spaCy model '{model_name}': {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    install_requirements()
+    install_spacy_model("de_core_news_lg")
+    install_spacy_model("fr_core_news_lg")
+    install_spacy_model("it_core_news_lg")
+    set_tokenizers_parallelism(True)
+    download_model('https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin', 'lid.176.bin')

requirements.txt

@@ -0,0 +1,21 @@
+Requests==2.32.2
+annif
+beautifulsoup4==4.12.3
+datasets==2.14.5
+fasttext==0.9.2
+gradio
+iterative_stratification==0.1.7
+nltk==3.8.1
+numpy==1.24.4
+omikuji==0.5.1
+openpyxl
+pandas==2.2.2
+pytz==2023.3.post1
+scikit_learn==1.3.2
+sentence_transformers==2.2.2
+swissparlpy==0.3.0
+tqdm==4.66.1
+transformers==4.39.3
+spacy==3.7.4
+huggingface_hub
+requests