pages/1_Data_Validation.py

import streamlit as st
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from Eda_functions import *
import numpy as np
import pickle
from streamlit_pandas_profiling import st_profile_report
import streamlit as st
import streamlit.components.v1 as components
import sweetviz as sv
from utilities import set_header,load_local_css
from st_aggrid import GridOptionsBuilder,GridUpdateMode
from st_aggrid import GridOptionsBuilder
from st_aggrid import AgGrid
import base64
import os
import tempfile
from ydata_profiling import ProfileReport
import re

st.set_page_config(
  page_title="Data Validation",
  page_icon=":shark:",
  layout="wide",
  initial_sidebar_state='collapsed'
)
load_local_css('styles.css')
set_header()





with open('data_import.pkl', 'rb') as f:
    data = pickle.load(f)
 
st.session_state['cleaned_data']= data['final_df']
st.session_state['category_dict'] = data['bin_dict']

st.title('Data Validation and Insights')


target_variables=[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Response Metrics']

target_column = st.selectbox('Select the Target Feature/Dependent Variable (will be used in all charts as reference)',list(*target_variables))
st.session_state['target_column']=target_column
panels=st.session_state['category_dict']['Panel Level 1'][0]
selected_panels=st.multiselect('Please choose the panels you wish to analyze.If no panels are selected, insights will be derived from the overall data.',st.session_state['cleaned_data'][panels].unique())
aggregation_dict = {item: 'sum' if key == 'Media' else 'mean' for key, value in st.session_state['category_dict'].items()  for item in value if item not in ['date','Panel_1']}

with st.expander('**Reponse Metric Analysis**'):
    
    if len(selected_panels)>0:
        st.session_state['Cleaned_data_panel']=st.session_state['cleaned_data'][st.session_state['cleaned_data']['Panel_1'].isin(selected_panels)]
 
        st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].groupby(by='date').agg(aggregation_dict)
        st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].reset_index()
    else:
        st.session_state['Cleaned_data_panel']=st.session_state['cleaned_data'].groupby(by='date').agg(aggregation_dict)
        st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].reset_index()


    fig=line_plot_target(st.session_state['Cleaned_data_panel'], target=target_column, title=f'{target_column} Over Time')
    st.plotly_chart(fig, use_container_width=True)


    media_channel=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Media'])
    # st.write(media_channel)

    Non_media_variables=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Exogenous' or key=='Internal'])


    st.markdown('### Annual Data Summary')
    st.dataframe(summary(st.session_state['Cleaned_data_panel'], media_channel+[target_column], spends=None,Target=True), use_container_width=True)

    if st.checkbox('Show raw data'):
        st.write(pd.concat([pd.to_datetime(st.session_state['Cleaned_data_panel']['date']).dt.strftime('%m/%d/%Y'),st.session_state['Cleaned_data_panel'].select_dtypes(np.number).applymap(format_numbers)],axis=1))
col1 = st.columns(1)

if "selected_feature" not in st.session_state:
    st.session_state['selected_feature']=None


def generate_report_with_target(channel_data, target_feature):
        report = sv.analyze([channel_data, "Dataset"], target_feat=target_feature)
        temp_dir = tempfile.mkdtemp()
        report_path = os.path.join(temp_dir, "report.html")
        report.show_html(filepath=report_path, open_browser=False)  # Generate the report as an HTML file
        return report_path

def generate_profile_report(df):
    pr = df.profile_report()
    temp_dir = tempfile.mkdtemp()
    report_path = os.path.join(temp_dir, "report.html")
    pr.to_file(report_path)
    return report_path


#st.header()
with st.expander('Univariate and Bivariate Report'):
    eda_columns=st.columns(2)
    with eda_columns[0]:
        if st.button('Generate Profile Report',help='Univariate report which inlcudes all statistical analysis'):
            with st.spinner('Generating Report'):     
                report_file = generate_profile_report(st.session_state['Cleaned_data_panel'])

                if os.path.exists(report_file):
                    with open(report_file, 'rb') as f:
                        st.success('Report Generated')
                        st.download_button(
                            label="Download EDA Report",
                            data=f.read(),
                            file_name="pandas_profiling_report.html",
                            mime="text/html"
                        )
                else:
                    st.warning("Report generation failed. Unable to find the report file.")

with eda_columns[1]:
    if st.button('Generate Sweetviz Report',help='Bivariate report for selected response metric'):
       with st.spinner('Generating Report'):     
            report_file = generate_report_with_target(st.session_state['Cleaned_data_panel'], target_column)

            if os.path.exists(report_file):
                with open(report_file, 'rb') as f:
                    st.success('Report Generated')
                    st.download_button(
                        label="Download EDA Report",
                        data=f.read(),
                        file_name="report.html",
                        mime="text/html"
                    )
            else:
                st.warning("Report generation failed. Unable to find the report file.")
            


#st.warning('Work in Progress')
with st.expander('Media Variables Analysis'):
# Get the selected feature
    st.session_state["selected_feature"]= st.selectbox('Select media', [col for col in media_channel if 'cost' not in col.lower() and 'spend' not in col.lower()])

    # Filter spends features based on the selected feature
    spends_features = [col for col in st.session_state['Cleaned_data_panel'].columns if any(keyword in col.lower() for keyword in ['cost', 'spend'])]
    spends_feature = [col for col in spends_features if re.split(r'_cost|_spend', col.lower())[0] in st.session_state["selected_feature"]]

    if 'validation' not in st.session_state:
        st.session_state['validation']=[]


    val_variables=[col for col in media_channel if col!='date']
    if len(spends_feature)==0:  
        st.warning('No spends varaible available for the selected metric in data') 
        
    else:
        fig_row1 = line_plot(st.session_state['Cleaned_data_panel'], x_col='date', y1_cols=[st.session_state["selected_feature"]], y2_cols=[target_column], title=f'Analysis of {st.session_state["selected_feature"]} and {[target_column][0]} Over Time')
        st.plotly_chart(fig_row1, use_container_width=True)
        st.markdown('### Summary')
        st.dataframe(summary(st.session_state['cleaned_data'],[st.session_state["selected_feature"]],spends=spends_feature[0]),use_container_width=True)

        cols2=st.columns(2)
        with cols2[0]:
            if st.button('Validate'):
                st.session_state['validation'].append(st.session_state["selected_feature"])
        with cols2[1]:
            if st.checkbox('Validate all'):
                st.session_state['validation'].extend(val_variables)
                st.success('All media variables are validated ✅')

        if len(set(st.session_state['validation']).intersection(val_variables))!=len(val_variables):
            validation_data=pd.DataFrame({'Validate':[True if col in st.session_state['validation'] else False for col in val_variables],
                                        'Variables':val_variables
                                        })
            cols3=st.columns([1,30])
            with cols3[1]:
                validation_df=st.data_editor(validation_data,
                                            # column_config={
                                            # 'Validate':st.column_config.CheckboxColumn(wi)
                                            
                                            # },
                                            column_config={
                                                "Validate": st.column_config.CheckboxColumn(
                                                    default=False,
                                                    width=100,
                                                ),
                                                'Variables':st.column_config.TextColumn(
                                                    width=1000

                                                )
                                                },hide_index=True)

                selected_rows = validation_df[validation_df['Validate']==True]['Variables']

                #st.write(selected_rows)

                st.session_state['validation'].extend(selected_rows)

                not_validated_variables = [col for col in val_variables if col not in st.session_state["validation"]]
                if not_validated_variables: 
                    not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}'
                    st.warning(not_validated_message)



with st.expander('Non Media Variables Analysis'):
    selected_columns_row4 = st.selectbox('Select Channel',Non_media_variables,index=1)
    #     # Create the dual-axis line plot
    fig_row4 = line_plot(st.session_state['Cleaned_data_panel'], x_col='date', y1_cols=[selected_columns_row4], y2_cols=[target_column], title=f'Analysis of {selected_columns_row4} and {target_column} Over Time')
    st.plotly_chart(fig_row4, use_container_width=True)
    selected_non_media=selected_columns_row4
    sum_df = st.session_state['Cleaned_data_panel'][['date', selected_non_media,target_column]]
    sum_df['Year']=pd.to_datetime(st.session_state['Cleaned_data_panel']['date']).dt.year
    #st.dataframe(df)
    #st.dataframe(sum_df.head(2))
    sum_df=sum_df.groupby('Year').agg('sum')
    sum_df.loc['Grand Total']=sum_df.sum()         
    sum_df=sum_df.applymap(format_numbers) 
    sum_df.fillna('-',inplace=True)
    sum_df=sum_df.replace({"0.0":'-','nan':'-'})
    st.markdown('### Summary')    
    st.dataframe(sum_df,use_container_width=True)


with st.expander('Correlation Analysis'):
    options = list(st.session_state['Cleaned_data_panel'].select_dtypes(np.number).columns)

    # selected_options = []
    # num_columns = 4
    # num_rows = -(-len(options) // num_columns)  # Ceiling division to calculate rows

    # # Create a grid of checkboxes
    # st.header('Select Features for Correlation Plot')
    # tick=False
    # if st.checkbox('Select all'):
    #     tick=True
    # selected_options = []
    # for row in range(num_rows):
    #     cols = st.columns(num_columns)
    #     for col in cols:
    #         if options:
    #             option = options.pop(0) 
    #             selected = col.checkbox(option,value=tick)
    #             if selected:
    #                 selected_options.append(option)
    # # Display selected options

    selected_options=st.multiselect('Select Variables For correlation plot',[var for var in options if var!= target_column],default=options[3])

    st.pyplot(correlation_plot(st.session_state['Cleaned_data_panel'],selected_options,target_column))