check existing sheet if dump and DL for all BD

.gitignore

@@ -1,5 +1,4 @@
 /.history
 /.venv
 /__pycache__
-/.github
 __pycache__

README.md

@@ -7,4 +7,36 @@ sdk: streamlit
 sdk_version: 1.37.1
 app_file: app.py
 pinned: false
----
+---
+
+## Db Query
+
+This is a Streamlit app for generating 2G, 3G, and LTE databases from an Excel dump file. The app reads the dump file and processes the data. The data is then saved to an Excel file. The Excel file is then downloaded.
+
+## How to use
+
+1. Run the app by executing `streamlit run app.py` in the terminal.
+2. Upload the dump file to the app.
+3. Click the buttons to generate the databases.
+4. Download the databases.
+
+## How it works
+
+The app reads the dump file and processes the data. The data is then saved to an excel file. The excel file is then downloaded.
+
+## Why
+
+The app is a simple way to generate the databases from the dump file. It saves time and effort.
+
+## Hosted Version
+
+You can access the hosted version of the app at [https://davmelchi-db-query.hf.space/](https://davmelchi-db-query.hf.space/).
+
+## TODO
+
+- [*] check if required sheets exist in the dump file
+- [*] Add a download button for all databases
+- [ ] Add option to download Neighbors database
+- [ ] Add dashboards for each database (Count of NE)
+- [ ] Add the ability to select columns
+- [ ] Error handling

app.py

@@ -2,9 +2,11 @@ import time
 
 import streamlit as st
 
-from queries.process_gsm import process_gsm_data
-from queries.process_lte import process_lte_data
-from queries.process_wcdma import process_wcdma_data
+from queries.process_all_db import process_all_tech_db
+from queries.process_gsm import process_gsm_data_to_excel
+from queries.process_lte import process_lte_data_to_excel
+from queries.process_wcdma import process_wcdma_data_to_excel
+from utils.check_sheet_exist import Technology, execute_checks_sheets_exist
 from utils.utils_vars import UtilsVars
 
 st.title("Database processing")
@@ -31,9 +33,14 @@ def download_button(database_type):
         data = UtilsVars.final_wcdma_database
         file_name = f"3G database_{time.time()}.xlsx"
     elif database_type == "LTE":
+        # data = UtilsVars.final_lte_database
         data = UtilsVars.final_lte_database
         file_name = f"LTE database_{time.time()}.xlsx"
+    elif database_type == "All":
+        data = UtilsVars.final_all_database
+        file_name = f"All database_{time.time()}.xlsx"
     st.download_button(
+        type="primary",
         label=f"Download {database_type} Database File",
         data=data,
         file_name=file_name,
@@ -41,20 +48,62 @@ def download_button(database_type):
     )
 
 
-col1, col2, col3 = st.columns(3)
-if uploaded_file is not None:
-    with col1:
-        st.button(
-            "Generate 2G Database",
-            on_click=lambda: process_database(process_gsm_data, "2G"),
-        )
-    with col2:
-        st.button(
-            "Generate 3G Database",
-            on_click=lambda: process_database(process_wcdma_data, "3G"),
-        )
-    with col3:
-        st.button(
-            "Generate LTE Database",
-            on_click=lambda: process_database(process_lte_data, "LTE"),
+def execute_process_all_tech_db(uploaded_file):
+    if uploaded_file is not None:
+        start_time = time.time()
+        process_all_tech_db(uploaded_file)
+        execution_time = time.time() - start_time
+        st.write(
+            f"All databases are generated. Execution time: {execution_time:.2f} seconds"
         )
+        download_button("All")
+
+
+col1, col2, col3, col4 = st.columns(4)
+if uploaded_file is not None:
+    # UtilsVars.file_path = uploaded_file
+    try:
+        execute_checks_sheets_exist(uploaded_file)
+        if (
+            Technology.gsm == False
+            and Technology.wcdma == False
+            and Technology.lte == False
+        ):
+            st.error(
+                """
+                     Uploaded file does not contain required sheets for any technology.
+                    "gsm": ["BTS", "BCF", "TRX"],
+                    "wcdma": ["WCEL", "WBTS", "WNCEL"],
+                    "lte": ["LNBTS", "LNCEL", "LNCEL_FDD", "LNCEL_TDD"],
+                """
+            )
+
+        if Technology.gsm == True:
+            with col1:
+                st.button(
+                    "Generate 2G DB",
+                    on_click=lambda: process_database(process_gsm_data_to_excel, "2G"),
+                )
+        if Technology.wcdma == True:
+            with col2:
+                st.button(
+                    "Generate 3G DB",
+                    on_click=lambda: process_database(
+                        process_wcdma_data_to_excel, "3G"
+                    ),
+                )
+        if Technology.lte == True:
+            with col3:
+                st.button(
+                    "Generate LTE DB",
+                    on_click=lambda: process_database(process_lte_data_to_excel, "LTE"),
+                )
+        if Technology.gsm == True or Technology.wcdma == True or Technology.lte == True:
+            with col4:
+                st.button(
+                    "Generate All DBs",
+                    on_click=lambda: execute_process_all_tech_db(uploaded_file),
+                )
+
+    except Exception as e:
+        st.error(f"Error: {e}")

queries/process_all_db.py

@@ -0,0 +1,16 @@
+from queries.process_gsm import process_gsm_data
+from queries.process_lte import process_lte_data
+from queries.process_wcdma import process_wcdma_data
+from utils.convert_to_excel import convert_dfs
+from utils.utils_vars import UtilsVars
+
+
+def process_all_tech_db(filepath: str):
+    UtilsVars.all_db_dfs.clear()
+    process_gsm_data(filepath)
+    process_wcdma_data(filepath)
+    process_lte_data(filepath)
+
+    UtilsVars.final_all_database = convert_dfs(
+        UtilsVars.all_db_dfs, ["GSM", "WCDMA", "LTE_FDD", "LTE_TDD"]
+    )

queries/process_gsm.py

@@ -107,4 +107,12 @@ def process_gsm_data(file_path: str):
     # save_dataframe(df_trx, "trx")
     # df_2g2 = save_dataframe(df_2g, "2g")
 
-    UtilsVars.final_gsm_database = convert_dfs([df_2g], ["GSM"])
+    UtilsVars.all_db_dfs.append(df_2g)
+    # UtilsVars.final_gsm_database = convert_dfs([df_2g], ["GSM"])
+    # UtilsVars.final_gsm_database = [df_2g]
+    return df_2g
+
+
+def process_gsm_data_to_excel(file_path: str):
+    gsm_dfs = process_gsm_data(file_path)
+    UtilsVars.final_gsm_database = convert_dfs([gsm_dfs], ["GSM"])

queries/process_lte.py

@@ -139,10 +139,17 @@ def process_lte_data(file_path: str):
     # Save dataframes
     # save_dataframe(df_fdd_final, "fdd")
     # save_dataframe(df_tdd_final, "tdd")
+    UtilsVars.all_db_dfs.extend([df_fdd_final, df_tdd_final])
 
-    UtilsVars.final_lte_database = convert_dfs(
-        [df_fdd_final, df_tdd_final], ["lte_fdd", "lte_tdd"]
-    )
+    return [df_fdd_final, df_tdd_final]
+    # add the fdd and tdd to the list
+
+    # UtilsVars.final_lte_database = [df_fdd_final, df_tdd_final]
+
+
+def process_lte_data_to_excel(file_path: str):
+    lte_dfs = process_lte_data(file_path)
+    UtilsVars.final_lte_database = convert_dfs(lte_dfs, ["LTE_FDD", "LTE_TDD"])
 
 
 # process_lte_data(r"data2\20240805_5810_05082024_Dump.xml.gz.xlsb")

queries/process_wcdma.py

@@ -122,7 +122,15 @@ def process_wcdma_data(file_path: str):
     # save_dataframe(df_wcel_bcf, "wbts")
     # save_dataframe(df_wncel, "wncel")
     # df_3g = save_dataframe(df_3g, "3G")
+    UtilsVars.all_db_dfs.append(df_3g)
 
-    UtilsVars.final_wcdma_database = convert_dfs([df_3g], ["WCDMA"])
+    # UtilsVars.final_wcdma_database = convert_dfs([df_3g], ["WCDMA"])
+    return df_3g
+    # UtilsVars.final_wcdma_database = [df_3g]
 
     # BTS.process_ok = "Done"
+
+
+def process_wcdma_data_to_excel(file_path: str):
+    wcdma_dfs = process_wcdma_data(file_path)
+    UtilsVars.final_wcdma_database = convert_dfs([wcdma_dfs], ["WCDMA"])

utils/check_sheet_exist.py

@@ -0,0 +1,61 @@
+import pandas as pd
+
+
+class Technology:
+    gsm = False
+    wcdma = False
+    lte = False
+
+
+# Dictionary of sheet groups to check
+sheets_to_check = {
+    "gsm": ["BTS", "BCF", "TRX"],
+    "wcdma": ["WCEL", "WBTS", "WNCEL"],
+    "lte": ["LNBTS", "LNCEL", "LNCEL_FDD", "LNCEL_TDD"],
+}
+
+
+def load(file_path):
+    # Load the Excel file
+    xlsb_file = pd.ExcelFile(file_path, engine="calamine")
+
+    # Get all sheet names in the file
+    available_sheets = xlsb_file.sheet_names
+    return available_sheets
+
+
+def check_sheets(technology_attr, sheet_list, file_path):
+    """
+    Check if all sheets in the given sheet_list exist in the Excel file.
+
+    Parameters
+    ----------
+    technology_attr : str
+        The attribute of the Technology class to set.
+    sheet_list : list[str]
+        The list of sheet names to check.
+
+    Returns
+    -------
+    None
+    """
+    available_sheets = load(file_path)
+    missing_sheets = [sheet for sheet in sheet_list if sheet not in available_sheets]
+    if not missing_sheets:
+        setattr(Technology, technology_attr, True)
+        print(getattr(Technology, technology_attr))
+        print("All sheets exist")
+    else:
+        print(f"Missing sheets: {missing_sheets}")
+        print(getattr(Technology, technology_attr))
+
+
+# Check each technology's sheets
+def execute_checks_sheets_exist(file_path):
+    for tech_attr, sheets in sheets_to_check.items():
+        check_sheets(tech_attr, sheets, file_path)
+
+
+# execute_checks(
+#     r"C:\Users\HP\Desktop\LTE\PROJET 2023\DUMP\2024\AOUT\20240822_7837_22082024_Dump.xml.gz.xlsb"
+# )

utils/convert_to_excel.py

@@ -26,23 +26,6 @@ def convert_dfs(dfs: list[pd.DataFrame], sheet_names: list[str]) -> bytes:
     return bytes_data
 
 
-# def save_dataframes(dfs: list[pd.DataFrame], sheet_names: list[str], folder_path: str):
-#     """
-#     Save the dataframes to an excel file. The excel file will be saved in the
-#     folder_path directory.
-
-#     Args:
-#         dfs (list[pd.DataFrame]): The list of dataframes to save.
-#         sheet_names (list[str]): The list of names for each sheet.
-#         folder_path (str): The path to the folder where the excel file will be saved.
-#     """
-#     bytes_data = convert_dfs(dfs, sheet_names)
-#     timestamp = int(time.time())
-#     file_name = f"{folder_path}/data_{timestamp}.xlsx"
-#     with open(file_name, "wb") as f:
-#         f.write(bytes_data)
-
-
 def save_dataframe(df: pd.DataFrame, sheet_name: str):
     """
     Save the dataframe to a csv file.

utils/utils_vars.py

@@ -22,6 +22,9 @@ class UtilsVars:
     final_lte_database = ""
     final_gsm_database = ""
     final_wcdma_database = ""
+    all_db_dfs = []
+    final_all_database = ""
+    file_path = ""
     physisal_db = get_physical_db()