Adding MRBTS and MAL. Compare MAL vs TCH ins BTS

README.md

@@ -41,6 +41,8 @@ You can access the hosted version of the app at [https://davmelchi-db-query.hf.s
 - [x] Add Core dump checking App
 - [x] Add site config band in database
 - [x] Add TRX database
+- [x] Add  MRBTS with code
+- [x] Check TCH from MAL sheet
 - [ ] Add dashboards for each database (Count of NE)
 - [ ] Add the ability to select columns
 - [ ] Error handling

apps/database_page.py

@@ -6,10 +6,9 @@ import streamlit as st
 from queries.process_all_db import process_all_tech_db
 from queries.process_gsm import process_gsm_data_to_excel
 from queries.process_lte import process_lte_data_to_excel
-from queries.process_neighbors import (
-    process_neighbors_data,
-    process_neighbors_data_to_excel,
-)
+from queries.process_mal import process_mal_data_to_excel
+from queries.process_mrbts import process_mrbts_data_to_excel
+from queries.process_neighbors import process_neighbors_data_to_excel
 from queries.process_trx import process_trx_with_bts_name_data_to_excel
 from queries.process_wcdma import process_wcdma_data_to_excel
 from utils.check_sheet_exist import Technology, execute_checks_sheets_exist
@@ -50,6 +49,12 @@ def download_button(database_type):
     elif database_type == "TRX":
         data = UtilsVars.final_trx_database
         file_name = f"TRX database_{datetime.now()}.xlsx"
+    elif database_type == "MRBTS":
+        data = UtilsVars.final_mrbts_database
+        file_name = f"MRBTS database_{datetime.now()}.xlsx"
+    elif database_type == "MAL":
+        data = UtilsVars.final_mal_database
+        file_name = f"MAL database_{datetime.now()}.xlsx"
     st.download_button(
         type="primary",
         label=f"Download {database_type} Database File",
@@ -82,15 +87,17 @@ if uploaded_file is not None:
             and Technology.lte == False
             and Technology.neighbors == False
             and Technology.trx == False
+            and Technology.mrbts == False
         ):
             st.error(
                 """
                      Uploaded file does not contain required sheets for any technology.
-                    "gsm": ["BTS", "BCF", "TRX"],
+                    "gsm": ["BTS", "BCF", "TRX","MAL"],
                     "wcdma": ["WCEL", "WBTS", "WNCEL"],
                     "lte": ["LNBTS", "LNCEL", "LNCEL_FDD", "LNCEL_TDD"],
                     "neighbors": ["ADCE", "ADJS", "ADJI", "ADJG", "ADJW", "BTS", "WCEL"],
                     "trx": ["TRX", "BTS"],
+                    "mrbts": ["MRBTS"],
                 """
             )
 
@@ -137,6 +144,20 @@ if uploaded_file is not None:
                         process_trx_with_bts_name_data_to_excel, "TRX"
                     ),
                 )
+        if Technology.mrbts == True:
+            with col7:
+                st.button(
+                    "Generate MRBTS",
+                    on_click=lambda: process_database(
+                        process_mrbts_data_to_excel, "MRBTS"
+                    ),
+                )
 
+        if Technology.mal == True:
+            with col8:
+                st.button(
+                    "Generate MAL",
+                    on_click=lambda: process_database(process_mal_data_to_excel, "MAL"),
+                )
     except Exception as e:
         st.error(f"Error: {e}")

queries/process_all_db.py

@@ -1,6 +1,8 @@
 from queries.process_gsm import process_gsm_data
 from queries.process_lte import process_lte_data
-from queries.process_trx import trx_with_bts_name
+from queries.process_mal import process_mal_with_bts_name
+from queries.process_mrbts import process_mrbts_data
+from queries.process_trx import process_trx_with_bts_name
 from queries.process_wcdma import process_wcdma_data
 from utils.convert_to_excel import convert_dfs
 from utils.utils_vars import UtilsVars
@@ -11,8 +13,11 @@ def process_all_tech_db(filepath: str):
     process_gsm_data(filepath)
     process_wcdma_data(filepath)
     process_lte_data(filepath)
-    trx_with_bts_name(filepath)
+    process_trx_with_bts_name(filepath)
+    process_mrbts_data(filepath)
+    process_mal_with_bts_name(filepath)
 
     UtilsVars.final_all_database = convert_dfs(
-        UtilsVars.all_db_dfs, ["GSM", "WCDMA", "LTE_FDD", "LTE_TDD", "TRX"]
+        UtilsVars.all_db_dfs,
+        ["GSM", "WCDMA", "LTE_FDD", "LTE_TDD", "TRX", "MRBTS", "MAL"],
     )

queries/process_gsm.py

@@ -1,5 +1,6 @@
 import pandas as pd
 
+from queries.process_mal import process_mal_data
 from queries.process_trx import process_trx_data
 from utils.config_band import config_band
 from utils.convert_to_excel import convert_dfs, save_dataframe
@@ -8,6 +9,7 @@ from utils.utils_vars import UtilsVars
 BTS_COLUMNS = [
     "ID_BCF",
     "ID_BTS",
+    "ID_MAL",
     "BSC",
     "BCF",
     "BTS",
@@ -54,6 +56,13 @@ BCF_COLUMNS = [
 ]
 
 
+def compare_trx_tch_versus_mal(tch1, tch2):
+    # Split the strings by commas, convert to sets, and compare
+    set1 = set(str(tch1).split(",")) if isinstance(tch1, str) else set()
+    set2 = set(str(tch2).split(",")) if isinstance(tch2, str) else set()
+    return set1 == set2
+
+
 def process_gsm_data(file_path: str):
     """
     Process data from the specified file path.
@@ -75,6 +84,7 @@ def process_gsm_data(file_path: str):
     df_bts["code"] = df_bts["name"].str.split("_").str[0].astype(int)
     df_bts["Region"] = df_bts["name"].str.split("_").str[1]
     df_bts["ID_BTS"] = df_bts[["BSC", "BCF", "BTS"]].astype(str).apply("_".join, axis=1)
+    df_bts["ID_MAL"] = df_bts[["BSC", "BTS"]].astype(str).apply("_".join, axis=1)
     df_bts["BSIC"] = (
         df_bts[["bsIdentityCodeNCC", "bsIdentityCodeBCC"]]
         .astype(str)
@@ -117,13 +127,20 @@ def process_gsm_data(file_path: str):
     # Process TRX data
     df_trx = process_trx_data(file_path)
 
+    # Process MAL data
+    df_mal = process_mal_data(file_path)
+
     # create band dataframe
     df_band = config_band(df_bts)
 
     # Merge dataframes
-    df_bts_bcf = pd.merge(df_bts, df_bcf, on="ID_BCF", how="left")
+    df_bts_bcf = pd.merge(df_bcf, df_bts, on="ID_BCF", how="left")
     df_2g = pd.merge(df_bts_bcf, df_trx, on="ID_BTS", how="left")
     df_2g = pd.merge(df_2g, df_band, on="code", how="left")
+    df_2g = pd.merge(df_2g, df_mal, on="ID_MAL", how="left")
+    df_2g["TRX_TCH_VS_MAL"] = df_2g.apply(
+        lambda row: compare_trx_tch_versus_mal(row["TRX_TCH"], row["MAL_TCH"]), axis=1
+    )
 
     df_physical_db = UtilsVars.physisal_db
     df_2g = pd.merge(df_2g, df_physical_db, on="Code_Sector", how="left")

queries/process_mal.py

@@ -0,0 +1,86 @@
+import pandas as pd
+
+from queries.process_small_bts import process_small_bts_data
+from utils.convert_to_excel import convert_dfs
+from utils.utils_vars import UtilsVars
+
+MAL_COLUMNS = [
+    "ID_MAL",
+    "MAL_TCH",
+    "number_mal_tch",
+]
+
+
+MAL_BTS_COLUMNS = [
+    "ID_MAL",
+    "code",
+    "name",
+    "MAL_TCH",
+    "number_mal_tch",
+]
+
+
+def process_mal_data(file_path: str):
+    """
+    Process data from the specified file path.
+
+    Args:
+        file_path (str): The path to the file.
+    """
+    # Read the specific sheet into a DataFrame
+    df_mal = pd.read_excel(
+        file_path,
+        sheet_name="MAL",
+        engine="calamine",
+        skiprows=[0],
+    )
+
+    df_mal.columns = df_mal.columns.str.replace(r"[ ]", "", regex=True)
+
+    df_mal["ID_MAL"] = df_mal[["BSC", "MAL"]].astype(str).apply("_".join, axis=1)
+
+    df_mal["frequency"] = df_mal["frequency"].str.replace("List;", "")
+    df_mal["MAL_TCH"] = df_mal["frequency"].str.replace(";", ",")
+    df_mal["number_mal_tch"] = df_mal["MAL_TCH"].apply(
+        lambda x: len(str(x).split(",")) if isinstance(x, str) else 0
+    )
+
+    df_mal = df_mal[MAL_COLUMNS]
+
+    # UtilsVars.all_db_dfs.append(df_mal)
+    # save_dataframe(df_mal, "MAL")
+    return df_mal
+
+
+def process_mal_with_bts_name(file_path: str):
+    """
+    Process data from the specified file path and merge it with the BTS data to get
+    the BTS name associated with each MAL.
+
+    Args:
+        file_path (str): The path to the file.
+
+    Returns:
+        pd.DataFrame: A DataFrame with the MAL data and the BTS name associated with
+            each MAL.
+    """
+    mal_df = process_mal_data(file_path=file_path)
+    df_bts = process_small_bts_data(file_path=file_path)
+
+    df_mal_bts_name = pd.merge(mal_df, df_bts, on="ID_MAL", how="left")
+    df_mal_bts_name = df_mal_bts_name[MAL_BTS_COLUMNS]
+
+    UtilsVars.all_db_dfs.append(df_mal_bts_name)
+    return df_mal_bts_name
+
+
+def process_mal_data_to_excel(file_path: str):
+    """
+    Process data from the specified file path and save it to a excel file.
+
+    Args:
+        file_path (str): The path to the file.
+    """
+    mal_df = process_mal_with_bts_name(file_path)
+
+    UtilsVars.final_mal_database = convert_dfs([mal_df], ["MAL"])

queries/process_mrbts.py

@@ -0,0 +1,41 @@
+import pandas as pd
+
+from utils.convert_to_excel import convert_dfs
+from utils.extract_code import extract_code_from_mrbts
+from utils.utils_vars import UtilsVars
+
+
+def process_mrbts_data(file_path: str) -> pd.DataFrame:
+    """
+    Process data from the specified file path.
+
+    Args:
+        file_path (str): The path to the file.
+    """
+    dfs = pd.read_excel(
+        file_path,
+        sheet_name=["MRBTS"],
+        engine="calamine",
+        skiprows=[0],
+    )
+
+    df_mrbts = dfs["MRBTS"]
+    df_mrbts.columns = df_mrbts.columns.str.replace(r"[ ]", "", regex=True)
+
+    df_mrbts = df_mrbts[df_mrbts["MRBTS"].apply(lambda x: str(x).isnumeric())]
+    df_mrbts["CODE"] = df_mrbts["MRBTS"].apply(extract_code_from_mrbts)
+    df_mrbts = df_mrbts[["MRBTS", "CODE", "name", "btsName"]]
+
+    UtilsVars.all_db_dfs.append(df_mrbts)
+    return df_mrbts
+
+
+def process_mrbts_data_to_excel(file_path: str) -> None:
+    """
+    Process data from the specified file path and save it to a excel file.
+
+    Args:
+        file_path (str): The path to the file.
+    """
+    mrbts_df = process_mrbts_data(file_path)
+    UtilsVars.final_mrbts_database = convert_dfs([mrbts_df], ["MRBTS"])

queries/process_small_bts.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+
+def process_small_bts_data(file_path: str):
+    dfs = pd.read_excel(
+        file_path,
+        sheet_name=["BTS"],
+        engine="calamine",
+        skiprows=[0],
+    )
+    df_bts = dfs["BTS"]
+    df_bts.columns = df_bts.columns.str.replace(r"[ ]", "", regex=True)
+    df_bts["code"] = df_bts["name"].str.split("_").str[0].astype(int)
+    df_bts["ID_BTS"] = df_bts[["BSC", "BCF", "BTS"]].astype(str).apply("_".join, axis=1)
+    df_bts["ID_MAL"] = df_bts[["BSC", "BTS"]].astype(str).apply("_".join, axis=1)
+    df_bts = df_bts[["ID_BTS", "ID_MAL", "code", "name"]]
+
+    return df_bts

queries/process_trx.py

@@ -1,5 +1,6 @@
 import pandas as pd
 
+from queries.process_small_bts import process_small_bts_data
 from utils.convert_to_excel import convert_dfs, save_dataframe
 from utils.utils_vars import UtilsVars
 
@@ -7,7 +8,7 @@ TRX_COLUMNS = [
     "ID_BTS",
     "trxRfPower",
     "BCCH",
-    "TCH",
+    "TRX_TCH",
     "number_trx_per_cell",
     "number_trx_per_site",
 ]
@@ -98,13 +99,13 @@ def process_trx_data(file_path: str):
     tch = tch.pivot_table(
         index="ID_BTS",
         values="initialFrequency",
-        aggfunc=lambda x: " ".join(map(str, x)),
+        aggfunc=lambda x: ",".join(map(str, x)),
     )
 
     tch = tch.reset_index()
 
     # rename the columns
-    tch.columns = ["ID_BTS", "TCH"]
+    tch.columns = ["ID_BTS", "TRX_TCH"]
 
     df_gsm_trx = pd.merge(bcch, tch, on="ID_BTS", how="left")
     # rename "initialFrequency" to "BCCH"
@@ -114,23 +115,24 @@ def process_trx_data(file_path: str):
     return df_gsm_trx
 
 
-def trx_with_bts_name(file_path: str):
+def process_trx_with_bts_name(file_path: str):
 
     df_gsm_trx = process_brute_trx_data(file_path=file_path).copy()
     df_gsm_trx.drop(["name"], axis=1, inplace=True)
 
     # Process TRX data
-    dfs = pd.read_excel(
-        file_path,
-        sheet_name=["BTS"],
-        engine="calamine",
-        skiprows=[0],
-    )
-    df_bts = dfs["BTS"]
-    df_bts.columns = df_bts.columns.str.replace(r"[ ]", "", regex=True)
-    df_bts["code"] = df_bts["name"].str.split("_").str[0].astype(int)
-    df_bts["ID_BTS"] = df_bts[["BSC", "BCF", "BTS"]].astype(str).apply("_".join, axis=1)
-    df_bts = df_bts[["ID_BTS", "code", "name"]]
+    # dfs = pd.read_excel(
+    #     file_path,
+    #     sheet_name=["BTS"],
+    #     engine="calamine",
+    #     skiprows=[0],
+    # )
+    # df_bts = dfs["BTS"]
+    df_bts = process_small_bts_data(file_path=file_path)
+    # df_bts.columns = df_bts.columns.str.replace(r"[ ]", "", regex=True)
+    # df_bts["code"] = df_bts["name"].str.split("_").str[0].astype(int)
+    # df_bts["ID_BTS"] = df_bts[["BSC", "BCF", "BTS"]].astype(str).apply("_".join, axis=1)
+    # df_bts = df_bts[["ID_BTS", "code", "name"]]
 
     df_trx_bts_name = pd.merge(df_gsm_trx, df_bts, on="ID_BTS", how="left")
     df_trx_bts_name = df_trx_bts_name[TRX_BTS_COLUMNS]
@@ -147,5 +149,5 @@ def process_trx_with_bts_name_data_to_excel(file_path: str):
     Args:
         file_path (str): The path to the file.
     """
-    trx_bts_name = trx_with_bts_name(file_path)
+    trx_bts_name = process_trx_with_bts_name(file_path)
     UtilsVars.final_trx_database = convert_dfs([trx_bts_name], ["TRX"])

utils/check_sheet_exist.py

@@ -7,15 +7,19 @@ class Technology:
     lte = False
     neighbors = False
     trx = False
+    mrbts = False
+    mal = False
 
 
 # Dictionary of sheet groups to check
 sheets_to_check = {
-    "gsm": ["BTS", "BCF", "TRX"],
+    "gsm": ["BTS", "BCF", "TRX", "MAL"],
     "neighbors": ["ADCE", "ADJS", "ADJI", "ADJG", "ADJW", "BTS", "WCEL"],
     "wcdma": ["WCEL", "WBTS", "WNCEL"],
     "lte": ["LNBTS", "LNCEL", "LNCEL_FDD", "LNCEL_TDD"],
     "trx": ["TRX", "BTS"],
+    "mrbts": ["MRBTS"],
+    "mal": ["MAL", "BTS"],
 }
 
 

utils/utils_vars.py

@@ -22,7 +22,7 @@ class UtilsVars:
     oml_band_frequence = {1: "OML BAND GSM 1800", 0: "OML BAND GSM 900"}
     gsm_band = {1: "G1800", 0: "G900"}
     configuration_schema = {1: "EGPRS 1800", 0: "EGPRS 900"}
-    channeltype_mapping = {4: "BCCH", 3: "TCH"}
+    channeltype_mapping = {4: "BCCH", 3: "TRX_TCH"}
     porteuse_mapping = {
         3004: "OML UTRA Band VIII",
         3006: "OML UTRA Band VIII",
@@ -41,6 +41,8 @@ class UtilsVars:
     final_gsm_database = ""
     final_wcdma_database = ""
     final_trx_database = ""
+    final_mrbts_database = ""
+    final_mal_database = ""
     all_db_dfs = []
     final_all_database = ""
     neighbors_database = ""