Commit
·
939b332
0
Parent(s):
1st commit
Browse files- .gitignore +5 -0
- README.md +10 -0
- main.py +60 -0
- physical_db/physical_database.csv +0 -0
- queries/process_gsm.py +110 -0
- queries/process_lte.py +148 -0
- queries/process_trx.py +72 -0
- queries/process_wcdma.py +128 -0
- requirements.txt +0 -0
- utils/convert_to_excel.py +54 -0
- utils/extract_code.py +34 -0
- utils/utils_vars.py +53 -0
.gitignore
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
/.history
|
2 |
+
/.venv
|
3 |
+
/__pycache__
|
4 |
+
|
5 |
+
__pycache__
|
README.md
ADDED
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
title: Db Query
|
3 |
+
emoji: 🏢
|
4 |
+
colorFrom: red
|
5 |
+
colorTo: green
|
6 |
+
sdk: streamlit
|
7 |
+
sdk_version: 1.37.1
|
8 |
+
app_file: app.py
|
9 |
+
pinned: false
|
10 |
+
---
|
main.py
ADDED
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import time
|
2 |
+
|
3 |
+
import streamlit as st
|
4 |
+
|
5 |
+
from queries.process_gsm import process_gsm_data
|
6 |
+
from queries.process_lte import process_lte_data
|
7 |
+
from queries.process_wcdma import process_wcdma_data
|
8 |
+
from utils.utils_vars import UtilsVars
|
9 |
+
|
10 |
+
st.title("Database processing")
|
11 |
+
|
12 |
+
uploaded_file = st.file_uploader("Upload updated dump file", type="xlsb")
|
13 |
+
|
14 |
+
|
15 |
+
def process_database(process_func, database_type):
|
16 |
+
if uploaded_file is not None:
|
17 |
+
start_time = time.time()
|
18 |
+
process_func(uploaded_file)
|
19 |
+
execution_time = time.time() - start_time
|
20 |
+
st.write(
|
21 |
+
f"{database_type} database is generated. Execution time: {execution_time:.2f} seconds"
|
22 |
+
)
|
23 |
+
download_button(database_type)
|
24 |
+
|
25 |
+
|
26 |
+
def download_button(database_type):
|
27 |
+
if database_type == "2G":
|
28 |
+
data = UtilsVars.final_gsm_database
|
29 |
+
file_name = f"2G database_{time.time()}.xlsx"
|
30 |
+
elif database_type == "3G":
|
31 |
+
data = UtilsVars.final_wcdma_database
|
32 |
+
file_name = f"3G database_{time.time()}.xlsx"
|
33 |
+
elif database_type == "LTE":
|
34 |
+
data = UtilsVars.final_lte_database
|
35 |
+
file_name = f"LTE database_{time.time()}.xlsx"
|
36 |
+
st.download_button(
|
37 |
+
label=f"Download {database_type} Database File",
|
38 |
+
data=data,
|
39 |
+
file_name=file_name,
|
40 |
+
mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
41 |
+
)
|
42 |
+
|
43 |
+
|
44 |
+
col1, col2, col3 = st.columns(3)
|
45 |
+
if uploaded_file is not None:
|
46 |
+
with col1:
|
47 |
+
st.button(
|
48 |
+
"Generate 2G Database",
|
49 |
+
on_click=lambda: process_database(process_gsm_data, "2G"),
|
50 |
+
)
|
51 |
+
with col2:
|
52 |
+
st.button(
|
53 |
+
"Generate 3G Database",
|
54 |
+
on_click=lambda: process_database(process_wcdma_data, "3G"),
|
55 |
+
)
|
56 |
+
with col3:
|
57 |
+
st.button(
|
58 |
+
"Generate LTE Database",
|
59 |
+
on_click=lambda: process_database(process_lte_data, "LTE"),
|
60 |
+
)
|
physical_db/physical_database.csv
ADDED
The diff for this file is too large to render.
See raw diff
|
|
queries/process_gsm.py
ADDED
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pandas as pd
|
2 |
+
|
3 |
+
from queries.process_trx import process_trx_data
|
4 |
+
from utils.convert_to_excel import convert_dfs, save_dataframe
|
5 |
+
from utils.utils_vars import UtilsVars
|
6 |
+
|
7 |
+
BTS_COLUMNS = [
|
8 |
+
"ID_BCF",
|
9 |
+
"ID_BTS",
|
10 |
+
"BSC",
|
11 |
+
"BCF",
|
12 |
+
"BTS",
|
13 |
+
"code",
|
14 |
+
"plmnPermitted",
|
15 |
+
"frequencyBandInUse",
|
16 |
+
"name",
|
17 |
+
"adminState",
|
18 |
+
"allowIMSIAttachDetach",
|
19 |
+
"amrSegLoadDepTchRateLower",
|
20 |
+
"amrSegLoadDepTchRateUpper",
|
21 |
+
"antennaHopping",
|
22 |
+
"bcchTrxPower",
|
23 |
+
"bsIdentityCodeBCC",
|
24 |
+
"bsIdentityCodeNCC",
|
25 |
+
"BSIC",
|
26 |
+
"cellId",
|
27 |
+
"dedicatedGPRScapacity",
|
28 |
+
"defaultGPRScapacity",
|
29 |
+
"fddQMin",
|
30 |
+
"fddQOffset",
|
31 |
+
"fddRscpMin",
|
32 |
+
"gprsEnabled",
|
33 |
+
"locationAreaIdLAC",
|
34 |
+
"locationAreaIdMCC",
|
35 |
+
"locationAreaIdMNC",
|
36 |
+
"rac",
|
37 |
+
"rachDropRxLevelThreshold",
|
38 |
+
"sectorId",
|
39 |
+
"SectorId2",
|
40 |
+
"segmentId",
|
41 |
+
"fastReturnToLTE",
|
42 |
+
"gsmPriority",
|
43 |
+
"segmentName",
|
44 |
+
"Code_Sector",
|
45 |
+
]
|
46 |
+
|
47 |
+
BCF_COLUMNS = [
|
48 |
+
"ID_BCF",
|
49 |
+
"site_name",
|
50 |
+
]
|
51 |
+
|
52 |
+
|
53 |
+
def process_gsm_data(file_path: str):
|
54 |
+
"""
|
55 |
+
Process data from the specified file path.
|
56 |
+
|
57 |
+
Args:
|
58 |
+
file_path (str): The path to the file.
|
59 |
+
"""
|
60 |
+
# Read the specific sheet into a DataFrame
|
61 |
+
dfs = pd.read_excel(
|
62 |
+
file_path,
|
63 |
+
sheet_name=["BTS", "BCF", "TRX"],
|
64 |
+
engine="calamine",
|
65 |
+
skiprows=[0],
|
66 |
+
)
|
67 |
+
|
68 |
+
# Process BTS data
|
69 |
+
df_bts = dfs["BTS"]
|
70 |
+
df_bts.columns = df_bts.columns.str.replace(r"[ ]", "", regex=True)
|
71 |
+
df_bts["code"] = df_bts["name"].str.split("_").str[0].astype(int)
|
72 |
+
df_bts["ID_BTS"] = df_bts[["BSC", "BCF", "BTS"]].astype(str).apply("_".join, axis=1)
|
73 |
+
df_bts["BSIC"] = (
|
74 |
+
df_bts[["bsIdentityCodeNCC", "bsIdentityCodeBCC"]]
|
75 |
+
.astype(str)
|
76 |
+
.apply("".join, axis=1)
|
77 |
+
)
|
78 |
+
df_bts["SectorId2"] = (
|
79 |
+
df_bts["sectorId"].map(UtilsVars.sector_mapping).fillna(df_bts["sectorId"])
|
80 |
+
)
|
81 |
+
df_bts["ID_BCF"] = df_bts[["BSC", "BCF"]].astype(str).apply("_".join, axis=1)
|
82 |
+
df_bts["Code_Sector"] = (
|
83 |
+
df_bts[["code", "SectorId2"]].astype(str).apply("_".join, axis=1)
|
84 |
+
)
|
85 |
+
df_bts["Code_Sector"] = df_bts["Code_Sector"].str.replace(".0", "")
|
86 |
+
df_bts = df_bts[BTS_COLUMNS]
|
87 |
+
|
88 |
+
# Process BCF data
|
89 |
+
df_bcf = dfs["BCF"]
|
90 |
+
df_bcf.columns = df_bcf.columns.str.replace(r"[ ]", "", regex=True)
|
91 |
+
df_bcf["ID_BCF"] = df_bcf[["BSC", "BCF"]].astype(str).apply("_".join, axis=1)
|
92 |
+
df_bcf.rename(columns={"name": "site_name"}, inplace=True)
|
93 |
+
df_bcf = df_bcf[BCF_COLUMNS]
|
94 |
+
|
95 |
+
df_trx = process_trx_data(file_path)
|
96 |
+
|
97 |
+
# Merge dataframes
|
98 |
+
df_bts_bcf = pd.merge(df_bts, df_bcf, on="ID_BCF", how="left")
|
99 |
+
df_2g = pd.merge(df_bts_bcf, df_trx, on="ID_BTS", how="left")
|
100 |
+
|
101 |
+
df_physical_db = UtilsVars.physisal_db
|
102 |
+
df_2g = pd.merge(df_2g, df_physical_db, on="Code_Sector", how="left")
|
103 |
+
|
104 |
+
# Save dataframes
|
105 |
+
# save_dataframe(df_bts, "bts")
|
106 |
+
# save_dataframe(df_bcf, "bcf")
|
107 |
+
save_dataframe(df_trx, "trx")
|
108 |
+
# df_2g2 = save_dataframe(df_2g, "2g")
|
109 |
+
|
110 |
+
UtilsVars.final_gsm_database = convert_dfs([df_2g], ["GSM"])
|
queries/process_lte.py
ADDED
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import pandas as pd
|
3 |
+
|
4 |
+
from utils.convert_to_excel import convert_dfs, save_dataframe
|
5 |
+
from utils.utils_vars import UtilsVars, get_band
|
6 |
+
|
7 |
+
LNCEL_COLUMNS = [
|
8 |
+
"ID_LNBTS",
|
9 |
+
"ID_LNCEL",
|
10 |
+
"MRBTS",
|
11 |
+
"LNBTS",
|
12 |
+
"LNCEL",
|
13 |
+
"final_name",
|
14 |
+
"name",
|
15 |
+
"cellName",
|
16 |
+
"code",
|
17 |
+
"SectorId",
|
18 |
+
"Code_Sector",
|
19 |
+
"actModulationSchemeDl",
|
20 |
+
"actModulationSchemeUL",
|
21 |
+
"administrativeState",
|
22 |
+
"eutraCelId",
|
23 |
+
"lcrId",
|
24 |
+
"pMax",
|
25 |
+
"phyCellId",
|
26 |
+
"tac",
|
27 |
+
"Region",
|
28 |
+
"band",
|
29 |
+
"band_type",
|
30 |
+
]
|
31 |
+
|
32 |
+
|
33 |
+
LNCEL_FDD_COLUMNS = [
|
34 |
+
"ID_LNCEL",
|
35 |
+
"dlChBw",
|
36 |
+
"dlMimoMode",
|
37 |
+
"dlRsBoost",
|
38 |
+
"earfcnDL",
|
39 |
+
"earfcnUL",
|
40 |
+
"prachCS",
|
41 |
+
"rootSeqIndex",
|
42 |
+
"ulChBw",
|
43 |
+
]
|
44 |
+
|
45 |
+
LNCEL_TDD_COLUMNS = [
|
46 |
+
"ID_LNCEL",
|
47 |
+
"chBw",
|
48 |
+
"dlMimoMode",
|
49 |
+
"dlRsBoost",
|
50 |
+
"earfcn",
|
51 |
+
"prachCS",
|
52 |
+
"rootSeqIndex",
|
53 |
+
]
|
54 |
+
|
55 |
+
|
56 |
+
def process_lte_data(file_path: str):
|
57 |
+
"""
|
58 |
+
Process data from the specified file path.
|
59 |
+
|
60 |
+
Args:
|
61 |
+
file_path (str): The path to the file.
|
62 |
+
"""
|
63 |
+
# Read excel sheets into dataframes
|
64 |
+
dfs = pd.read_excel(
|
65 |
+
file_path,
|
66 |
+
sheet_name=["LNCEL", "LNBTS", "LNCEL_FDD", "LNCEL_TDD"],
|
67 |
+
engine="calamine",
|
68 |
+
skiprows=[0],
|
69 |
+
)
|
70 |
+
|
71 |
+
# Process LNCEL data
|
72 |
+
df_lncel = dfs["LNCEL"]
|
73 |
+
df_lncel.columns = df_lncel.columns.str.replace(r"[ ]", "", regex=True)
|
74 |
+
df_lncel["final_name"] = df_lncel["name"].fillna(df_lncel["cellName"])
|
75 |
+
df_lncel["code"] = df_lncel["final_name"].str.split("_").str[0]
|
76 |
+
df_lncel["SectorId"] = (
|
77 |
+
df_lncel["lcrId"].map(UtilsVars.sector_mapping).fillna(df_lncel["lcrId"])
|
78 |
+
)
|
79 |
+
df_lncel["Code_Sector"] = (
|
80 |
+
df_lncel[["code", "SectorId"]]
|
81 |
+
.astype(str)
|
82 |
+
.apply("_".join, axis=1)
|
83 |
+
.str.replace(".0", "")
|
84 |
+
.str.lstrip("0")
|
85 |
+
)
|
86 |
+
df_lncel["ID_LNCEL"] = (
|
87 |
+
df_lncel[["MRBTS", "LNBTS", "LNCEL"]].astype(str).apply("_".join, axis=1)
|
88 |
+
)
|
89 |
+
df_lncel["ID_LNBTS"] = (
|
90 |
+
df_lncel[["MRBTS", "LNBTS"]].astype(str).apply("_".join, axis=1)
|
91 |
+
)
|
92 |
+
df_lncel["Region"] = df_lncel["final_name"].str.split("_").str[1]
|
93 |
+
df_lncel["band"] = df_lncel["final_name"].apply(get_band)
|
94 |
+
df_lncel["band_type"] = np.where(df_lncel["band"] == "L2300", "TDD", "FDD")
|
95 |
+
df_lncel = df_lncel[LNCEL_COLUMNS]
|
96 |
+
|
97 |
+
# Process LNBTS data
|
98 |
+
df_lnbts = dfs["LNBTS"]
|
99 |
+
df_lnbts.columns = df_lnbts.columns.str.replace(r"[ ]", "", regex=True)
|
100 |
+
df_lnbts["ID_LNBTS"] = (
|
101 |
+
df_lnbts[["MRBTS", "LNBTS"]].astype(str).apply("_".join, axis=1)
|
102 |
+
)
|
103 |
+
df_lnbts.rename(columns={"name": "lnbts_name"}, inplace=True)
|
104 |
+
df_lnbts = df_lnbts[["ID_LNBTS", "lnbts_name"]]
|
105 |
+
|
106 |
+
# Merge dataframes
|
107 |
+
df_lncel_lnbts = pd.merge(df_lncel, df_lnbts, on="ID_LNBTS", how="left")
|
108 |
+
|
109 |
+
df_physical_db = UtilsVars.physisal_db
|
110 |
+
df_physical_db = df_physical_db[
|
111 |
+
["Code_Sector", "Azimut", "Longitude", "Latitude", "Hauteur"]
|
112 |
+
]
|
113 |
+
df_lncel_lnbts = pd.merge(
|
114 |
+
df_lncel_lnbts, df_physical_db, on="Code_Sector", how="left"
|
115 |
+
)
|
116 |
+
|
117 |
+
# Process LNCEL_FDD and LNCEL_TDD data
|
118 |
+
df_lncel_fdd = dfs["LNCEL_FDD"]
|
119 |
+
df_lncel_fdd.columns = df_lncel_fdd.columns.str.replace(r"[ ]", "", regex=True)
|
120 |
+
df_lncel_fdd["ID_LNCEL"] = (
|
121 |
+
df_lncel_fdd[["MRBTS", "LNBTS", "LNCEL"]].astype(str).apply("_".join, axis=1)
|
122 |
+
)
|
123 |
+
df_lncel_fdd = df_lncel_fdd[LNCEL_FDD_COLUMNS]
|
124 |
+
|
125 |
+
df_lncel_tdd = dfs["LNCEL_TDD"]
|
126 |
+
df_lncel_tdd.columns = df_lncel_tdd.columns.str.replace(r"[ ]", "", regex=True)
|
127 |
+
df_lncel_tdd["ID_LNCEL"] = (
|
128 |
+
df_lncel_tdd[["MRBTS", "LNBTS", "LNCEL"]].astype(str).apply("_".join, axis=1)
|
129 |
+
)
|
130 |
+
df_lncel_tdd = df_lncel_tdd[LNCEL_TDD_COLUMNS]
|
131 |
+
|
132 |
+
# Create df_fdd and df_tdd base on "band"
|
133 |
+
df_fdd = df_lncel_lnbts[df_lncel_lnbts["band"] != "L2300"]
|
134 |
+
df_tdd = df_lncel_lnbts[df_lncel_lnbts["band"] == "L2300"]
|
135 |
+
|
136 |
+
df_fdd_final = pd.merge(df_fdd, df_lncel_fdd, on="ID_LNCEL", how="left")
|
137 |
+
df_tdd_final = pd.merge(df_tdd, df_lncel_tdd, on="ID_LNCEL", how="left")
|
138 |
+
|
139 |
+
# Save dataframes
|
140 |
+
# save_dataframe(df_fdd_final, "fdd")
|
141 |
+
# save_dataframe(df_tdd_final, "tdd")
|
142 |
+
|
143 |
+
UtilsVars.final_lte_database = convert_dfs(
|
144 |
+
[df_fdd_final, df_tdd_final], ["lte_fdd", "lte_tdd"]
|
145 |
+
)
|
146 |
+
|
147 |
+
|
148 |
+
# process_lte_data(r"data2\20240805_5810_05082024_Dump.xml.gz.xlsb")
|
queries/process_trx.py
ADDED
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pandas as pd
|
2 |
+
|
3 |
+
from utils.convert_to_excel import convert_dfs, save_dataframe
|
4 |
+
from utils.utils_vars import UtilsVars
|
5 |
+
|
6 |
+
TRX_COLUMNS = [
|
7 |
+
"ID_BTS",
|
8 |
+
"trxRfPower",
|
9 |
+
"BCCH",
|
10 |
+
"TCH",
|
11 |
+
"number_trx_per_cell",
|
12 |
+
"number_trx_per_site",
|
13 |
+
]
|
14 |
+
|
15 |
+
|
16 |
+
def process_trx_data(file_path: str):
|
17 |
+
"""
|
18 |
+
Process data from the specified file path.
|
19 |
+
|
20 |
+
Args:
|
21 |
+
file_path (str): The path to the file.
|
22 |
+
"""
|
23 |
+
# Read the specific sheet into a DataFrame
|
24 |
+
dfs = pd.read_excel(
|
25 |
+
file_path,
|
26 |
+
sheet_name=["BTS", "BCF", "TRX"],
|
27 |
+
engine="calamine",
|
28 |
+
skiprows=[0],
|
29 |
+
)
|
30 |
+
|
31 |
+
# Process TRX data
|
32 |
+
df_trx = dfs["TRX"]
|
33 |
+
df_trx.columns = df_trx.columns.str.replace(r"[ ]", "", regex=True)
|
34 |
+
df_trx["ID_BTS"] = df_trx[["BSC", "BCF", "BTS"]].astype(str).apply("_".join, axis=1)
|
35 |
+
df_trx["ID_BCF"] = df_trx[["BSC", "BCF"]].astype(str).apply("_".join, axis=1)
|
36 |
+
df_trx["number_trx_per_cell"] = df_trx.groupby("ID_BTS")["ID_BTS"].transform(
|
37 |
+
"count"
|
38 |
+
)
|
39 |
+
df_trx["number_trx_per_site"] = df_trx.groupby("ID_BCF")["ID_BCF"].transform(
|
40 |
+
"count"
|
41 |
+
)
|
42 |
+
|
43 |
+
bcch = df_trx[df_trx["channel0Type"] == 4]
|
44 |
+
tch = df_trx[df_trx["channel0Type"] == 3][["ID_BTS", "initialFrequency"]]
|
45 |
+
|
46 |
+
tch = tch.pivot_table(
|
47 |
+
index="ID_BTS",
|
48 |
+
values="initialFrequency",
|
49 |
+
aggfunc=lambda x: " ".join(map(str, x)),
|
50 |
+
)
|
51 |
+
|
52 |
+
tch = tch.reset_index()
|
53 |
+
|
54 |
+
# rename the columns
|
55 |
+
tch.columns = ["ID_BTS", "TCH"]
|
56 |
+
|
57 |
+
# Merge dataframes
|
58 |
+
|
59 |
+
df_trx = pd.merge(bcch, tch, on="ID_BTS", how="left")
|
60 |
+
# rename "initialFrequency" to "BCCH"
|
61 |
+
df_trx = df_trx.rename(columns={"initialFrequency": "BCCH"})
|
62 |
+
df_trx = df_trx[TRX_COLUMNS]
|
63 |
+
|
64 |
+
# Save dataframes
|
65 |
+
# save_dataframe(df_trx, "trx")
|
66 |
+
# df_2g2 = save_dataframe(df_2g, "2g")
|
67 |
+
|
68 |
+
# UtilsVars.final_gsm_database = convert_dfs([df_2g], ["GSM"])
|
69 |
+
return df_trx
|
70 |
+
|
71 |
+
|
72 |
+
# process_trx_data(r"data2\20240805_5810_05082024_Dump.xml.gz.xlsb")
|
queries/process_wcdma.py
ADDED
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pandas as pd
|
2 |
+
|
3 |
+
from utils.convert_to_excel import convert_dfs, save_dataframe
|
4 |
+
from utils.extract_code import extract_code_from_mrbts
|
5 |
+
from utils.utils_vars import UtilsVars
|
6 |
+
|
7 |
+
WCEL_COLUMNS = [
|
8 |
+
"ID_WBTS",
|
9 |
+
"ID_WCEL",
|
10 |
+
"RNC",
|
11 |
+
"WBTS",
|
12 |
+
"WCEL",
|
13 |
+
"site_name",
|
14 |
+
"name",
|
15 |
+
"code",
|
16 |
+
"AdminCellState",
|
17 |
+
"CId",
|
18 |
+
"LAC",
|
19 |
+
"UARFCN",
|
20 |
+
"PriScrCode",
|
21 |
+
"SAC",
|
22 |
+
"maxCarrierPower",
|
23 |
+
"PtxPrimaryCPICH",
|
24 |
+
"CellRange",
|
25 |
+
"CodeTreeOptTimer",
|
26 |
+
"CodeTreeOptimisation",
|
27 |
+
"CodeTreeUsage",
|
28 |
+
"PRACHDelayRange",
|
29 |
+
"PrxOffset",
|
30 |
+
"PrxTarget",
|
31 |
+
"PrxTargetMax",
|
32 |
+
"PrxTargetPSMax",
|
33 |
+
"PrxTargetPSMaxtHSRACH",
|
34 |
+
"PtxCellMax",
|
35 |
+
"PtxOffset",
|
36 |
+
"PtxTarget",
|
37 |
+
"SmartLTELayeringEnabled",
|
38 |
+
"SectorID",
|
39 |
+
"Code_Sector",
|
40 |
+
"code_wcel",
|
41 |
+
]
|
42 |
+
|
43 |
+
WBTS_COLUMNS = [
|
44 |
+
"ID_WBTS",
|
45 |
+
"site_name",
|
46 |
+
]
|
47 |
+
|
48 |
+
WNCEL_COLUMNS = [
|
49 |
+
"code_wcel",
|
50 |
+
"maxCarrierPower",
|
51 |
+
]
|
52 |
+
|
53 |
+
|
54 |
+
def process_wcdma_data(file_path: str):
|
55 |
+
"""
|
56 |
+
Process data from the specified file path.
|
57 |
+
|
58 |
+
Args:
|
59 |
+
file_path (str): The path to the file.
|
60 |
+
"""
|
61 |
+
# Read the specific sheet into a DataFrame
|
62 |
+
# df_wcel = pd.read_excel(
|
63 |
+
# file_path, sheet_name="WCEL", engine="calamine", skiprows=[0]
|
64 |
+
# )
|
65 |
+
# df_wbts = pd.read_excel(
|
66 |
+
# file_path, sheet_name="WBTS", engine="calamine", skiprows=[0]
|
67 |
+
# )
|
68 |
+
# df_wncel = pd.read_excel(
|
69 |
+
# file_path, sheet_name="WNCEL", engine="calamine", skiprows=[0]
|
70 |
+
# )
|
71 |
+
|
72 |
+
dfs = pd.read_excel(
|
73 |
+
file_path,
|
74 |
+
sheet_name=["WCEL", "WBTS", "WNCEL"],
|
75 |
+
engine="calamine",
|
76 |
+
skiprows=[0],
|
77 |
+
)
|
78 |
+
|
79 |
+
# Process BTS data
|
80 |
+
df_wcel = dfs["WCEL"]
|
81 |
+
df_wcel.columns = df_wcel.columns.str.replace(r"[ ]", "", regex=True)
|
82 |
+
df_wcel["code"] = df_wcel["name"].str.split("_").str[0].astype(int)
|
83 |
+
df_wcel["ID_WCEL"] = (
|
84 |
+
df_wcel[["RNC", "WBTS", "WCEL"]].astype(str).apply("_".join, axis=1)
|
85 |
+
)
|
86 |
+
|
87 |
+
df_wcel["ID_WBTS"] = df_wcel[["RNC", "WBTS"]].astype(str).apply("_".join, axis=1)
|
88 |
+
df_wcel["Code_Sector"] = (
|
89 |
+
df_wcel[["code", "SectorID"]].astype(str).apply("_".join, axis=1)
|
90 |
+
)
|
91 |
+
df_wcel["code_wcel"] = df_wcel[["code", "WCEL"]].astype(str).apply("_".join, axis=1)
|
92 |
+
|
93 |
+
df_wcel["Code_Sector"] = df_wcel["Code_Sector"].str.replace(".0", "")
|
94 |
+
|
95 |
+
# Process WBTS data
|
96 |
+
df_wbts = dfs["WBTS"]
|
97 |
+
df_wbts.columns = df_wbts.columns.str.replace(r"[ ]", "", regex=True)
|
98 |
+
df_wbts["ID_WBTS"] = df_wbts[["RNC", "WBTS"]].astype(str).apply("_".join, axis=1)
|
99 |
+
df_wbts.rename(columns={"name": "site_name"}, inplace=True)
|
100 |
+
df_wbts = df_wbts[WBTS_COLUMNS]
|
101 |
+
|
102 |
+
# Process WNCEL data
|
103 |
+
df_wncel = dfs["WNCEL"]
|
104 |
+
df_wncel.columns = df_wncel.columns.str.replace(r"[ ]", "", regex=True)
|
105 |
+
df_wncel["CODE"] = df_wncel["MRBTS"].apply(extract_code_from_mrbts)
|
106 |
+
df_wncel["code_wcel"] = (
|
107 |
+
df_wncel[["CODE", "WNCEL"]].astype(str).apply("_".join, axis=1)
|
108 |
+
)
|
109 |
+
df_wncel = df_wncel[WNCEL_COLUMNS]
|
110 |
+
|
111 |
+
# Merge dataframes
|
112 |
+
df_wcel_bcf = pd.merge(df_wcel, df_wbts, on="ID_WBTS", how="left")
|
113 |
+
|
114 |
+
df_3g = pd.merge(df_wcel_bcf, df_wncel, on="code_wcel", how="left")
|
115 |
+
|
116 |
+
df_3g = df_3g[WCEL_COLUMNS]
|
117 |
+
|
118 |
+
df_physical_db = UtilsVars.physisal_db
|
119 |
+
df_3g = pd.merge(df_3g, df_physical_db, on="Code_Sector", how="left")
|
120 |
+
# Save dataframes
|
121 |
+
# save_dataframe(df_wcel, "wcel")
|
122 |
+
# save_dataframe(df_wcel_bcf, "wbts")
|
123 |
+
# save_dataframe(df_wncel, "wncel")
|
124 |
+
# df_3g = save_dataframe(df_3g, "3G")
|
125 |
+
|
126 |
+
UtilsVars.final_wcdma_database = convert_dfs([df_3g], ["WCDMA"])
|
127 |
+
|
128 |
+
# BTS.process_ok = "Done"
|
requirements.txt
ADDED
Binary file (126 Bytes). View file
|
|
utils/convert_to_excel.py
ADDED
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import io
|
2 |
+
import time
|
3 |
+
|
4 |
+
import pandas as pd
|
5 |
+
import streamlit as st
|
6 |
+
|
7 |
+
|
8 |
+
@st.cache_data
|
9 |
+
def convert_dfs(dfs: list[pd.DataFrame], sheet_names: list[str]) -> bytes:
|
10 |
+
# IMPORTANT: Cache the conversion to prevent computation on every rerun
|
11 |
+
|
12 |
+
# Create a BytesIO object
|
13 |
+
bytes_io = io.BytesIO()
|
14 |
+
|
15 |
+
# Write the dataframes to the BytesIO object
|
16 |
+
with pd.ExcelWriter(bytes_io, engine="xlsxwriter") as writer:
|
17 |
+
for df, sheet_name in zip(dfs, sheet_names):
|
18 |
+
df.to_excel(writer, sheet_name=sheet_name, index=False)
|
19 |
+
|
20 |
+
# Get the bytes data
|
21 |
+
bytes_data = bytes_io.getvalue()
|
22 |
+
|
23 |
+
# Close the BytesIO object
|
24 |
+
bytes_io.close()
|
25 |
+
|
26 |
+
return bytes_data
|
27 |
+
|
28 |
+
|
29 |
+
# def save_dataframes(dfs: list[pd.DataFrame], sheet_names: list[str], folder_path: str):
|
30 |
+
# """
|
31 |
+
# Save the dataframes to an excel file. The excel file will be saved in the
|
32 |
+
# folder_path directory.
|
33 |
+
|
34 |
+
# Args:
|
35 |
+
# dfs (list[pd.DataFrame]): The list of dataframes to save.
|
36 |
+
# sheet_names (list[str]): The list of names for each sheet.
|
37 |
+
# folder_path (str): The path to the folder where the excel file will be saved.
|
38 |
+
# """
|
39 |
+
# bytes_data = convert_dfs(dfs, sheet_names)
|
40 |
+
# timestamp = int(time.time())
|
41 |
+
# file_name = f"{folder_path}/data_{timestamp}.xlsx"
|
42 |
+
# with open(file_name, "wb") as f:
|
43 |
+
# f.write(bytes_data)
|
44 |
+
|
45 |
+
|
46 |
+
def save_dataframe(df: pd.DataFrame, sheet_name: str):
|
47 |
+
"""
|
48 |
+
Save the dataframe to a csv file.
|
49 |
+
|
50 |
+
Args:
|
51 |
+
df (pd.DataFrame): The dataframe to save.
|
52 |
+
sheet_name (str): The name of the sheet.
|
53 |
+
"""
|
54 |
+
df.to_csv(f"data2/{sheet_name}_{time.time()}.csv", index=False)
|
utils/extract_code.py
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
def extract_code_from_mrbts(mrbts):
|
2 |
+
"""
|
3 |
+
Extracts the code from a MRBTS (Mobile Radio Base Transceiver Station) string.
|
4 |
+
|
5 |
+
Args:
|
6 |
+
mrbts (int or str): The MRBTS string to extract the code from.
|
7 |
+
|
8 |
+
Returns:
|
9 |
+
int: The extracted code from the MRBTS string.
|
10 |
+
|
11 |
+
Raises:
|
12 |
+
None.
|
13 |
+
|
14 |
+
Notes:
|
15 |
+
This function handles MRBTS strings that start with '10' and have a length greater than 5,
|
16 |
+
as well as MRBTS strings that start with '1', '2', or '3'. For MRBTS strings that do not
|
17 |
+
meet these criteria, the entire MRBTS string is returned as an integer.
|
18 |
+
"""
|
19 |
+
str_mrbts = str(mrbts)
|
20 |
+
|
21 |
+
if len(str_mrbts) > 5 and str_mrbts.startswith("10"):
|
22 |
+
# For MRBTS starting with '10' and having length greater than 5
|
23 |
+
return int(str_mrbts[2:])
|
24 |
+
elif len(str_mrbts) > 4 and str_mrbts.startswith("1"):
|
25 |
+
return int(str_mrbts[1:])
|
26 |
+
elif len(str_mrbts) > 4 and str_mrbts.startswith("2"):
|
27 |
+
# For MRBTS starting with '2' (like 20000 + code)
|
28 |
+
return int(str_mrbts[1:])
|
29 |
+
elif len(str_mrbts) > 4 and str_mrbts.startswith("3"):
|
30 |
+
# For MRBTS starting with '3' (like 30000 + code)
|
31 |
+
return int(str_mrbts[1:])
|
32 |
+
else:
|
33 |
+
# Default case
|
34 |
+
return int(str_mrbts)
|
utils/utils_vars.py
ADDED
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import pandas as pd
|
3 |
+
|
4 |
+
|
5 |
+
def get_physical_db():
|
6 |
+
"""
|
7 |
+
Reads the physical_database.csv file from the physical_db directory and
|
8 |
+
returns a pandas DataFrame containing only the columns 'Code_Sector',
|
9 |
+
'Azimut', 'Longitude', 'Latitude', and 'Hauteur'.
|
10 |
+
|
11 |
+
Returns:
|
12 |
+
pd.DataFrame: A DataFrame containing the filtered columns.
|
13 |
+
"""
|
14 |
+
physical = pd.read_csv(r"physical_db\physical_database.csv")
|
15 |
+
physical = physical[["Code_Sector", "Azimut", "Longitude", "Latitude", "Hauteur"]]
|
16 |
+
return physical
|
17 |
+
|
18 |
+
|
19 |
+
class UtilsVars:
|
20 |
+
sector_mapping = {4: 1, 5: 2, 6: 3, 11: 1, 12: 2, 13: 3}
|
21 |
+
channeltype_mapping = {4: "BCCH", 3: "TCH"}
|
22 |
+
final_lte_database = ""
|
23 |
+
final_gsm_database = ""
|
24 |
+
final_wcdma_database = ""
|
25 |
+
physisal_db = get_physical_db()
|
26 |
+
|
27 |
+
|
28 |
+
# print(UtilsVars.physisal_db)
|
29 |
+
|
30 |
+
|
31 |
+
def get_band(text):
|
32 |
+
"""
|
33 |
+
Extract the band from the given string.
|
34 |
+
|
35 |
+
Parameters
|
36 |
+
----------
|
37 |
+
text : str
|
38 |
+
The string to extract the band from.
|
39 |
+
|
40 |
+
Returns
|
41 |
+
-------
|
42 |
+
str or np.nan
|
43 |
+
The extracted band, or NaN if the text was not a string or did not contain
|
44 |
+
any of the recognized bands (L1800, L2300, L800).
|
45 |
+
"""
|
46 |
+
if isinstance(text, str): # Check if text is a string
|
47 |
+
if "L1800" in text:
|
48 |
+
return "L1800"
|
49 |
+
elif "L2300" in text:
|
50 |
+
return "L2300"
|
51 |
+
elif "L800" in text:
|
52 |
+
return "L800"
|
53 |
+
return np.nan # or return None
|