Update data_processor.py

data_processor.py

@@ -1,3 +1,235 @@
+# import re
+# import pandas as pd
+# import os
+# from huggingface_hub import InferenceClient
+
+# class DataProcessor:
+#     INTERVENTION_COLUMN_OPTIONS = [
+#         'Did the intervention happen today?',
+#         'Did the intervention take place today?'
+#     ]
+#     YES_RESPONSES = ['yes', 'assessment day']  # Added this line
+#     ENGAGED_STR = 'Engaged'
+#     PARTIALLY_ENGAGED_STR = 'Partially Engaged'
+#     NOT_ENGAGED_STR = 'Not Engaged'
+
+#     def __init__(self, student_metrics_df=None):
+#         self.hf_api_key = os.getenv('HF_API_KEY')
+#         if not self.hf_api_key:
+#             raise ValueError("HF_API_KEY not set in environment variables")
+#         self.client = InferenceClient(api_key=self.hf_api_key)
+#         self.student_metrics_df = student_metrics_df
+#         self.intervention_column = None  # Will be set when processing data
+
+#     def read_excel(self, uploaded_file):
+#         return pd.read_excel(uploaded_file)
+
+#     def format_session_data(self, df):
+#         date_column = next((col for col in df.columns if col in ["Date of Session", "Date"]), None)
+#         if date_column:
+#             df[date_column] = pd.to_datetime(df[date_column], errors='coerce').dt.date
+#         else:
+#             print("Warning: Neither 'Date of Session' nor 'Date' column found in the dataframe.")
+        
+#         df['Timestamp'] = self.safe_convert_to_datetime(df['Timestamp'], '%I:%M %p')
+#         df['Session Start Time'] = self.safe_convert_to_time(df['Session Start Time'], '%I:%M %p')
+#         df['Session End Time'] = self.safe_convert_to_time(df['Session End Time'], '%I:%M %p')
+#         return df
+
+#     def safe_convert_to_time(self, series, format_str='%I:%M %p'):
+#         try:
+#             converted = pd.to_datetime(series, format='%H:%M:%S', errors='coerce')
+#             if format_str:
+#                 return converted.dt.strftime(format_str)
+#             return converted
+#         except Exception as e:
+#             print(f"Error converting series to time: {e}")
+#             return series
+
+#     def safe_convert_to_datetime(self, series, format_str=None):
+#         try:
+#             converted = pd.to_datetime(series, errors='coerce')
+#             if format_str:
+#                 return converted.dt.strftime(format_str)
+#             return converted
+#         except Exception as e:
+#             print(f"Error converting series to datetime: {e}")
+#             return series
+
+#     def replace_student_names_with_initials(self, df):
+#         updated_columns = []
+#         for col in df.columns:
+#             if 'Student Attendance' in col:
+#                 # Search for the last occurrence of text within square brackets at the end of the string
+#                 match = re.search(r'\[(.+?)\]$', col)
+#                 if not match:
+#                     # Handle cases where the closing bracket might be missing
+#                     match = re.search(r'\[(.+)$', col)
+#                 if match:
+#                     name = match.group(1).strip()
+#                     # Remove any trailing closing bracket if it wasn't matched earlier
+#                     name = name.rstrip(']')
+#                     # Get initials
+#                     initials = ''.join([part[0] for part in name.strip().split()])
+#                     updated_col = f'Student Attendance [{initials}]'
+#                     updated_columns.append(updated_col)
+#                 else:
+#                     # If no match is found, keep the column name as is
+#                     updated_columns.append(col)
+#             else:
+#                 updated_columns.append(col)
+#         df.columns = updated_columns
+#         return df
+
+
+#     def find_intervention_column(self, df):
+#         for column in self.INTERVENTION_COLUMN_OPTIONS:
+#             if column in df.columns:
+#                 self.intervention_column = column
+#                 return column
+#         raise ValueError("No intervention column found in the dataframe.")
+
+#     def get_intervention_column(self, df):
+#         if self.intervention_column is None:
+#             self.intervention_column = self.find_intervention_column(df)
+#         return self.intervention_column
+
+#     def compute_intervention_statistics(self, df):
+#         intervention_column = self.get_intervention_column(df)
+#         total_days = len(df)
+#         sessions_held = df[intervention_column].str.strip().str.lower().isin(self.YES_RESPONSES).sum()  # Modified line
+#         intervention_frequency = (sessions_held / total_days) * 100 if total_days > 0 else 0
+#         return pd.DataFrame({
+#             'Intervention Dosage (%)': [round(intervention_frequency, 0)],
+#             'Intervention Sessions Held': [sessions_held],
+#             'Intervention Sessions Not Held': [total_days - sessions_held],
+#             'Total Number of Days Available': [total_days]
+#         })
+
+#     def classify_engagement(self, engagement_str):
+#         engagement_str = str(engagement_str).lower()
+#         if engagement_str.startswith(self.ENGAGED_STR.lower()):
+#             return self.ENGAGED_STR
+#         elif engagement_str.startswith(self.PARTIALLY_ENGAGED_STR.lower()):
+#             return self.PARTIALLY_ENGAGED_STR
+#         elif engagement_str.startswith(self.NOT_ENGAGED_STR.lower()):
+#             return self.NOT_ENGAGED_STR
+#         else:
+#             return 'Unknown'
+    
+#     def compute_student_metrics(self, df):
+#         intervention_column = self.get_intervention_column(df)
+#         intervention_df = df[df[intervention_column].str.strip().str.lower().isin(self.YES_RESPONSES)]
+#         intervention_sessions_held = len(intervention_df)
+#         student_columns = [col for col in df.columns if col.startswith('Student Attendance')]
+    
+#         student_metrics = {}
+#         for col in student_columns:
+#             student_name = col.replace('Student Attendance [', '').replace(']', '').strip()
+#             student_data = intervention_df[[col]].copy()
+#             student_data[col] = student_data[col].fillna('Absent')
+    
+#             # Classify each entry
+#             student_data['Engagement'] = student_data[col].apply(self.classify_engagement)
+    
+#             # Calculate attendance
+#             attendance_values = student_data['Engagement'].apply(
+#                 lambda x: 1 if x in [self.ENGAGED_STR, self.PARTIALLY_ENGAGED_STR, self.NOT_ENGAGED_STR] else 0
+#             )
+    
+#             sessions_attended = attendance_values.sum()
+#             attendance_pct = (sessions_attended / intervention_sessions_held * 100) if intervention_sessions_held > 0 else 0
+#             attendance_pct = round(attendance_pct)
+    
+#             # Engagement counts (excluding 'Absent')
+#             engagement_counts = {
+#                 self.ENGAGED_STR: 0,
+#                 self.PARTIALLY_ENGAGED_STR: 0,
+#                 self.NOT_ENGAGED_STR: 0
+#             }
+    
+#             # Count the engagement types, excluding 'Absent'
+#             for x in student_data['Engagement']:
+#                 if x in engagement_counts:
+#                     engagement_counts[x] += 1
+#                 # 'Absent' is not counted in engagement_counts
+    
+#             total_present_sessions = sum(engagement_counts.values())
+    
+#             engaged_pct = (
+#                 (engagement_counts[self.ENGAGED_STR] / total_present_sessions * 100)
+#                 if total_present_sessions > 0 else 0
+#             )
+#             engaged_pct = round(engaged_pct)
+    
+#             partially_engaged_pct = (
+#                 (engagement_counts[self.PARTIALLY_ENGAGED_STR] / total_present_sessions * 100)
+#                 if total_present_sessions > 0 else 0
+#             )
+#             partially_engaged_pct = round(partially_engaged_pct)
+    
+#             not_engaged_pct = (
+#                 (engagement_counts[self.NOT_ENGAGED_STR] / total_present_sessions * 100)
+#                 if total_present_sessions > 0 else 0
+#             )
+#             not_engaged_pct = round(not_engaged_pct)
+    
+#             # Engagement percentage is based on Engaged and Partially Engaged sessions
+#             engagement_pct = (
+#                 ((engagement_counts[self.ENGAGED_STR] + engagement_counts[self.PARTIALLY_ENGAGED_STR]) / total_present_sessions * 100)
+#                 if total_present_sessions > 0 else 0
+#             )
+#             engagement_pct = round(engagement_pct)
+    
+#             # Absent percentage (for reference, not used in engagement calculation)
+#             absent_sessions = student_data['Engagement'].value_counts().get('Absent', 0)
+#             absent_pct = (absent_sessions / intervention_sessions_held * 100) if intervention_sessions_held > 0 else 0
+#             absent_pct = round(absent_pct)
+    
+#             # Determine if the student attended ≥ 90% of sessions
+#             attended_90 = "Yes" if attendance_pct >= 90 else "No"
+    
+#             # Determine if the student was engaged ≥ 80% of the time
+#             engaged_80 = "Yes" if engagement_pct >= 80 else "No"
+    
+#             # Store metrics
+#             student_metrics[student_name] = {
+#                 'Attended ≥ 90%': attended_90,
+#                 'Engagement ≥ 80%': engaged_80,
+#                 'Attendance (%)': attendance_pct,
+#                 'Engagement (%)': engagement_pct,
+#                 f'{self.ENGAGED_STR} (%)': engaged_pct,
+#                 f'{self.PARTIALLY_ENGAGED_STR} (%)': partially_engaged_pct,
+#                 f'{self.NOT_ENGAGED_STR} (%)': not_engaged_pct,
+#                 'Absent (%)': absent_pct
+#             }
+    
+#         # Create a DataFrame from student_metrics
+#         student_metrics_df = pd.DataFrame.from_dict(student_metrics, orient='index').reset_index()
+#         student_metrics_df.rename(columns={'index': 'Student'}, inplace=True)
+#         return student_metrics_df
+    
+#     def compute_average_metrics(self, student_metrics_df):
+#         # Calculate the attendance and engagement average percentages across students
+#         attendance_avg_stats = student_metrics_df['Attendance (%)'].mean()  # Average attendance percentage
+#         engagement_avg_stats = student_metrics_df['Engagement (%)'].mean()  # Average engagement percentage
+        
+#         # Round the averages to whole numbers
+#         attendance_avg_stats = round(attendance_avg_stats)
+#         engagement_avg_stats = round(engagement_avg_stats)
+        
+#         return attendance_avg_stats, engagement_avg_stats
+    
+#     def evaluate_student(self, row, attendance_threshold=90, engagement_threshold=80):
+#         if row["Attended ≥ 90%"] == "No":
+#             return "Address Attendance"
+#         elif row["Engagement ≥ 80%"] == "No":
+#             return "Address Engagement"
+#         else:
+#             return "Consider barriers, fidelity, and progress monitoring"
+
+
+
 import re
 import pandas as pd
 import os
@@ -81,7 +313,6 @@ class DataProcessor:
         df.columns = updated_columns
         return df
 
-
     def find_intervention_column(self, df):
         for column in self.INTERVENTION_COLUMN_OPTIONS:
             if column in df.columns:
@@ -117,83 +348,6 @@ class DataProcessor:
         else:
             return 'Unknown'
     
-    # def compute_student_metrics(self, df):
-    #     intervention_column = self.get_intervention_column(df)
-    #     intervention_df = df[df[intervention_column].str.strip().str.lower().isin(self.YES_RESPONSES)]  # Modified line
-    #     intervention_sessions_held = len(intervention_df)
-    #     student_columns = [col for col in df.columns if col.startswith('Student Attendance')]
-    
-    #     student_metrics = {}
-    #     for col in student_columns:
-    #         student_name = col.replace('Student Attendance [', '').replace(']', '').strip()
-    #         student_data = intervention_df[[col]].copy()
-    #         student_data[col] = student_data[col].fillna('Absent')
-    
-    #         attendance_values = student_data[col].apply(lambda x: 1 if self.classify_engagement(x) in [
-    #             self.ENGAGED_STR,
-    #             self.PARTIALLY_ENGAGED_STR,
-    #             self.NOT_ENGAGED_STR
-    #         ] else 0)
-    
-    #         sessions_attended = attendance_values.sum()
-    #         attendance_pct = (sessions_attended / intervention_sessions_held * 100) if intervention_sessions_held > 0 else 0
-    #         attendance_pct = round(attendance_pct)
-    
-    #         engagement_counts = {
-    #             self.ENGAGED_STR: 0,
-    #             self.PARTIALLY_ENGAGED_STR: 0,
-    #             self.NOT_ENGAGED_STR: 0,
-    #             'Absent': 0
-    #         }
-    
-    #         for x in student_data[col]:
-    #             classified_engagement = self.classify_engagement(x)
-    #             if classified_engagement in engagement_counts:
-    #                 engagement_counts[classified_engagement] += 1
-    #             else:
-    #                 engagement_counts['Absent'] += 1  # Count as Absent if not engaged
-    
-    #         total_sessions = sum(engagement_counts.values())
-            
-    #         engaged_pct = (engagement_counts[self.ENGAGED_STR] / total_sessions * 100) if total_sessions > 0 else 0
-    #         engaged_pct = round(engaged_pct)
-    
-    #         partially_engaged_pct = (engagement_counts[self.PARTIALLY_ENGAGED_STR] / total_sessions * 100) if total_sessions > 0 else 0
-    #         partially_engaged_pct = round(partially_engaged_pct)
-    
-    #         not_engaged_pct = (engagement_counts[self.NOT_ENGAGED_STR] / total_sessions * 100) if total_sessions > 0 else 0
-    #         not_engaged_pct = round(not_engaged_pct)
-    
-    #         absent_pct = (engagement_counts['Absent'] / total_sessions * 100) if total_sessions > 0 else 0
-    #         absent_pct = round(absent_pct)
-    
-    #         # Engagement percentage is based on Engaged and Partially Engaged sessions
-    #         engagement_pct = ((engagement_counts[self.ENGAGED_STR] + engagement_counts[self.PARTIALLY_ENGAGED_STR]) / total_sessions * 100) if total_sessions > 0 else 0
-    #         engagement_pct = round(engagement_pct)
-    
-    #         # Determine if the student attended ≥ 90% of sessions
-    #         attended_90 = "Yes" if attendance_pct >= 90 else "No"
-    
-    #         # Determine if the student was engaged ≥ 80% of the time
-    #         engaged_80 = "Yes" if engagement_pct >= 80 else "No"
-    
-    #         # Store metrics in the required order
-    #         student_metrics[student_name] = {
-    #             'Attended ≥ 90%': attended_90,
-    #             'Engagement ≥ 80%': engaged_80,
-    #             'Attendance (%)': attendance_pct,
-    #             'Engagement (%)': engagement_pct,
-    #             f'{self.ENGAGED_STR} (%)': engaged_pct,
-    #             f'{self.PARTIALLY_ENGAGED_STR} (%)': partially_engaged_pct,
-    #             f'{self.NOT_ENGAGED_STR} (%)': not_engaged_pct,
-    #             'Absent (%)': absent_pct
-    #         }
-    
-    #     # Create a DataFrame from student_metrics
-    #     student_metrics_df = pd.DataFrame.from_dict(student_metrics, orient='index').reset_index()
-    #     student_metrics_df.rename(columns={'index': 'Student'}, inplace=True)
-    #     return student_metrics_df
-
     def compute_student_metrics(self, df):
         intervention_column = self.get_intervention_column(df)
         intervention_df = df[df[intervention_column].str.strip().str.lower().isin(self.YES_RESPONSES)]
@@ -274,7 +428,7 @@ class DataProcessor:
                 'Attended ≥ 90%': attended_90,
                 'Engagement ≥ 80%': engaged_80,
                 'Attendance (%)': attendance_pct,
-                'Engagement (%)': engagement_pct,
+                # 'Engagement (%)': engagement_pct, REMOVED REMOVED
                 f'{self.ENGAGED_STR} (%)': engaged_pct,
                 f'{self.PARTIALLY_ENGAGED_STR} (%)': partially_engaged_pct,
                 f'{self.NOT_ENGAGED_STR} (%)': not_engaged_pct,
@@ -289,7 +443,8 @@ class DataProcessor:
     def compute_average_metrics(self, student_metrics_df):
         # Calculate the attendance and engagement average percentages across students
         attendance_avg_stats = student_metrics_df['Attendance (%)'].mean()  # Average attendance percentage
-        engagement_avg_stats = student_metrics_df['Engagement (%)'].mean()  # Average engagement percentage
+        # engagement_avg_stats = student_metrics_df['Engagement (%)'].mean()  # Average engagement percentage REMOVED REMOVED
+        engagement_avg_stats = student_metrics_df[f'{self.ENGAGED_STR} (%)'].mean()  # Average engagement percentage
         
         # Round the averages to whole numbers
         attendance_avg_stats = round(attendance_avg_stats)