automated progress bar

components/review_dashboard_page.py

@@ -12,6 +12,7 @@ from config import conf
 from utils.database import get_db
 from data.models import Annotation, TTSData, Annotator, Validation
 from data.repository.annotator_workload_repo import AnnotatorWorkloadRepo
+from utils.user_progress import get_user_progress, update_user_progress, get_next_unreviewed_annotation, get_annotations_from_position
 
 log = Logger()
 LOADER = CloudServerAudioLoader(conf.FTP_URL)
@@ -302,7 +303,7 @@ class ReviewDashboardPage:
                 log.warning(f"No target annotator found for reviewer {username}")
                 return [], 0, "", "", "", "", "", "", "", "", gr.update(value=None, autoplay=False), gr.update(visible=False, value=""), False, gr.update(value="❌ Reject")
 
-            # Load ALL annotations from target annotator to properly implement "continue from where left off"
+            # Load annotations from target annotator with PROGRESS-AWARE LOADING
             with get_db() as db:
                 # Get target annotator's ID
                 target_annotator_obj = db.query(Annotator).filter_by(name=target_annotator).first()
@@ -312,70 +313,56 @@ class ReviewDashboardPage:
                 
                 log.info(f"Found target annotator with ID: {target_annotator_obj.id}")
 
-                # Load ALL annotations to properly implement resume functionality
-                # This is similar to phase 1 dashboard approach
-                all_annotations_query = db.query(
-                    Annotation,
-                    TTSData.filename,
-                    TTSData.sentence
-                ).join(
-                    TTSData, Annotation.tts_data_id == TTSData.id
-                ).filter(
+                # Get the next unreviewed annotation position for this user
+                next_annotation_id, next_position = get_next_unreviewed_annotation(user_id, target_annotator_obj.id)
+                
+                if next_annotation_id is None:
+                    # All annotations have been reviewed
+                    total_count = db.query(Annotation).filter(
+                        Annotation.annotator_id == target_annotator_obj.id
+                    ).count()
+                    return [], 0, f"🎉 **All Done!** You have reviewed all {total_count} annotations for {target_annotator}.", "", "", "", "", "", "", "", gr.update(value=None, autoplay=False), gr.update(visible=False, value=""), False, gr.update(value="❌ Reject")
+
+                # Load annotations starting from the next unreviewed position
+                INITIAL_BATCH_SIZE = 10  # Load 10 items starting from where user left off
+                annotations_data = get_annotations_from_position(
+                    target_annotator_obj.id, 
+                    next_position, 
+                    INITIAL_BATCH_SIZE
+                )
+                
+                # Get total count for progress info
+                total_count = db.query(Annotation).filter(
                     Annotation.annotator_id == target_annotator_obj.id
-                ).order_by(Annotation.id)
+                ).count()
 
-                all_results = all_annotations_query.all()
-                total_count = len(all_results)
+                log.info(f"Progress-aware load: Starting from position {next_position}, loaded {len(annotations_data)} annotations out of {total_count} total for target annotator ID {target_annotator_obj.id}")
 
-                log.info(f"Loaded {total_count} annotations for target annotator ID {target_annotator_obj.id}")
-
-                # Process all items and determine validation status immediately
+                # Process items with minimal data - validation status will be loaded on-demand
                 items = []
-                for annotation, filename, sentence in all_results:
-                    # Check if annotation is deleted
+                for annotation, filename, sentence in annotations_data:
+                    # Check if annotation is deleted (minimal processing)
                     is_deleted = not annotation.annotated_sentence or annotation.annotated_sentence.strip() == ""
                     annotated_sentence_display = "[DELETED ANNOTATION]" if is_deleted else annotation.annotated_sentence
                     
-                    # Get validation status immediately (not on-demand)
-                    validation_status, is_deleted_updated = get_validation_status_for_item(db, annotation.id, user_id, annotation)
-                    
                     items.append({
                         "annotation_id": annotation.id,
                         "tts_id": annotation.tts_data_id,
                         "filename": filename,
                         "sentence": sentence,
                         "annotated_sentence": annotated_sentence_display,
-                        "is_deleted": is_deleted_updated,
+                        "is_deleted": is_deleted,
                         "annotated_at": annotation.annotated_at.isoformat() if annotation.annotated_at else "",
-                        "validation_status": validation_status,
-                        "validation_loaded": True  # Already loaded
+                        "validation_status": "Loading...",  # Will be loaded on-demand
+                        "validation_loaded": False  # Track if validation status has been loaded
                     })
 
-                # --- Resume Logic: Find first unreviewed item (similar to phase 1 dashboard) ---
-                initial_idx = 0
+                # Set initial display - start from the first item (which should be unreviewed)
                 if items:
-                    first_unreviewed_idx = -1
-                    for i, item_data in enumerate(items):
-                        if item_data["validation_status"] == "Not Reviewed":
-                            first_unreviewed_idx = i
-                            break
+                    initial_item = items[0]  # Always start from the first item in the loaded batch
+                    review_info_text = f"🔍 **Phase 2 Review Mode** - Continuing from where you left off. Loaded {len(items)} items starting from position {next_position + 1} of {total_count} total items."
                     
-                    if first_unreviewed_idx != -1:
-                        initial_idx = first_unreviewed_idx
-                        log.info(f"Resuming at first unreviewed item, index: {initial_idx} (ID: {items[initial_idx]['tts_id']})")
-                    else:  # All items are reviewed
-                        initial_idx = len(items) - 1
-                        log.info(f"All items reviewed, starting at last item, index: {initial_idx} (ID: {items[initial_idx]['tts_id']})")
-                else:  # No items assigned
-                    initial_idx = 0
-                    log.info("No items assigned to user, starting at index 0.") 
-                
-                # Set initial display
-                if items:
-                    initial_item = items[initial_idx]
-                    review_info_text = f"🔍 **Phase 2 Review Mode** - Reviewing assigned annotations. Loaded {len(items)} items."
                     # Ensure correct order of return values for 14 outputs
-                    # items, idx, review_info, tts_id, filename, sentence, ann_sentence, annotated_at, validation_status, annotator_placeholder, audio_update, rejection_reason_update, rejection_mode_reset, btn_reject_update
                     rejection_reason_val = ""
                     rejection_visible_val = False
                     if initial_item["validation_status"].startswith("Rejected"):
@@ -387,7 +374,7 @@ class ReviewDashboardPage:
                     
                     return (
                         items, 
-                        initial_idx, 
+                        0,  # Always start from index 0 of the loaded batch
                         review_info_text,
                         str(initial_item["tts_id"]), 
                         initial_item["filename"], 
@@ -405,13 +392,6 @@ class ReviewDashboardPage:
                     # Ensure correct order and number of return values for empty items (14 outputs)
                     return [], 0, f"🔍 **Phase 2 Review Mode** - No annotations found for review.", "", "", "", "", "", "", "", gr.update(value=None, autoplay=False), gr.update(visible=False, value=""), False, gr.update(value="❌ Reject")
 
-                # except Exception as e:
-                #     log.error(f"Error loading review items: {e}")
-                #     sentry_sdk.capture_exception(e)
-                #     gr.Error(f"Failed to load review data: {e}")
-                #     # Ensure correct order and number of return values for error case (14 outputs)
-                #     return [], 0, "", "", "", "", "", "", "", "", gr.update(value=None, autoplay=False), gr.update(visible=False, value=""), False, gr.update(value="❌ Reject")
-
         def show_current_review_item_fn(items, idx, session):
             if not items or idx >= len(items) or idx < 0:
                 # tts_id, filename, sentence, ann_sentence, annotated_at, validation_status, annotator_name_placeholder, audio_update, rejection_reason_update, rejection_mode_reset, btn_reject_update
@@ -419,7 +399,28 @@ class ReviewDashboardPage:
             
             current_item = items[idx]
             
-            # Validation status is already loaded, no need for on-demand loading
+            # Load validation status on-demand if not already loaded
+            if not current_item.get("validation_loaded", False):
+                user_id = session.get("user_id")
+                if user_id:
+                    with get_db() as db:
+                        try:
+                            # Get the full annotation object for validation processing
+                            annotation_obj = db.query(Annotation).filter_by(id=current_item["annotation_id"]).first()
+                            if annotation_obj:
+                                validation_status, is_deleted = get_validation_status_for_item(db, current_item["annotation_id"], user_id, annotation_obj)
+                                current_item["validation_status"] = validation_status
+                                current_item["is_deleted"] = is_deleted
+                                current_item["validation_loaded"] = True
+                                
+                                # Update displayed annotation if deleted
+                                if is_deleted:
+                                    current_item["annotated_sentence"] = "[DELETED ANNOTATION]"
+                                
+                                log.info(f"Loaded validation status for item {idx}: {validation_status}")
+                        except Exception as e:
+                            log.error(f"Error loading validation status for item {idx}: {e}")
+                            current_item["validation_status"] = "Error loading status"
             
             rejection_reason = ""
             rejection_visible = False
@@ -452,19 +453,29 @@ class ReviewDashboardPage:
         def update_review_info_fn(items, total_count):
             """Update the review info banner with current loaded items count"""
             if items:
-                return f"🔍 **Phase 2 Review Mode** - Reviewing assigned annotations. Loaded {len(items)} items."
+                return f"🔍 **Phase 2 Review Mode** - Reviewing assigned annotations. Loaded {len(items)} of {total_count} total items."
             else:
                 return f"🔍 **Phase 2 Review Mode** - No annotations found for review."
 
         def navigate_and_load_fn(items, current_idx, direction, session):
-            """Simple navigation function - all items are already loaded"""
+            """Combined navigation and loading function"""
             if not items:
                 return items, 0, ""
             
             # Navigate
             if direction == "next":
                 new_idx = min(current_idx + 1, len(items) - 1)
-                return items, new_idx, ""  # No review info update needed
+                # Only load more items when user reaches the LAST item of a batch
+                should_load_more = (new_idx == len(items) - 1 and len(items) % 5 == 0)
+                if should_load_more:
+                    log.info(f"User reached end of loaded items ({new_idx}/{len(items)}), will load more items")
+                    # Load more items
+                    updated_items, total_count = load_more_items_fn(items, session, current_batch_size=10)
+                    # Update review info with new count
+                    review_info = update_review_info_fn(updated_items, total_count)
+                    return updated_items, new_idx, review_info
+                else:
+                    return items, new_idx, ""  # No review info update needed
             else:  # prev
                 new_idx = max(current_idx - 1, 0)
                 return items, new_idx, ""  # No review info update needed
@@ -509,6 +520,32 @@ class ReviewDashboardPage:
                 db.commit()
                 log.info(f"Validation saved successfully for annotation_id: {annotation_id}")
                 
+                # Update user progress tracking
+                try:
+                    # Find target annotator for this user
+                    username = session.get("username")
+                    target_annotator = None
+                    for annotator_name, reviewer_name in conf.REVIEW_MAPPING.items():
+                        if reviewer_name == username:
+                            target_annotator = annotator_name
+                            break
+                    
+                    if target_annotator:
+                        target_annotator_obj = db.query(Annotator).filter_by(name=target_annotator).first()
+                        if target_annotator_obj:
+                            # Calculate the current position in the review list
+                            current_position = db.query(Annotation).filter(
+                                Annotation.annotator_id == target_annotator_obj.id,
+                                Annotation.id <= annotation_id
+                            ).count() - 1  # Convert to 0-based index
+                            
+                            # Update user progress
+                            update_user_progress(user_id, target_annotator_obj.id, annotation_id, current_position)
+                            log.info(f"Updated user progress: user {user_id} at position {current_position} for annotation {annotation_id}")
+                except Exception as e:
+                    log.warning(f"Failed to update user progress: {e}")
+                    # Don't fail the validation save if progress tracking fails
+                
                 items[idx]["validation_status"] = "Approved" if approved else f"Rejected ({rejection_reason})" if rejection_reason else "Rejected"
                 
                 # Show rejection reason input only if rejected, otherwise hide and clear
@@ -578,7 +615,73 @@ class ReviewDashboardPage:
             #     gr.Warning(f"Invalid Data ID format: {target_data_id}")
             return current_idx
 
-
+        def load_more_items_fn(items, session, current_batch_size=10):
+            """Load more items when user needs them (pagination support)"""
+            user_id = session.get("user_id")
+            username = session.get("username")
+            
+            if not user_id or not username:
+                return items, 0  # Return existing items if no user session
+                
+            # Find target annotator
+            target_annotator = None
+            for annotator_name, reviewer_name in conf.REVIEW_MAPPING.items():
+                if reviewer_name == username:
+                    target_annotator = annotator_name
+                    break
+                    
+            if not target_annotator:
+                return items, 0
+                
+            with get_db() as db:
+                target_annotator_obj = db.query(Annotator).filter_by(name=target_annotator).first()
+                if not target_annotator_obj:
+                    return items, 0
+                    
+                # Get total count for updated review info
+                total_count = db.query(Annotation).filter(
+                    Annotation.annotator_id == target_annotator_obj.id
+                ).count()
+                    
+                # Load next batch starting from where we left off
+                offset = len(items)
+                
+                # FAST LOADING: Use same strategy as initial load - simple query without complex JOINs
+                query = db.query(
+                    Annotation,
+                    TTSData.filename,
+                    TTSData.sentence
+                ).join(
+                    TTSData, Annotation.tts_data_id == TTSData.id
+                ).filter(
+                    Annotation.annotator_id == target_annotator_obj.id
+                ).order_by(Annotation.id).offset(offset).limit(current_batch_size)
+                
+                results = query.all()
+                
+                # Process new items with minimal data - validation status loaded on-demand
+                new_items = []
+                for annotation, filename, sentence in results:
+                    # Check if annotation is deleted (minimal processing)
+                    is_deleted = not annotation.annotated_sentence or annotation.annotated_sentence.strip() == ""
+                    annotated_sentence_display = "[DELETED ANNOTATION]" if is_deleted else annotation.annotated_sentence
+                    
+                    new_items.append({
+                        "annotation_id": annotation.id,
+                        "tts_id": annotation.tts_data_id,
+                        "filename": filename,
+                        "sentence": sentence,
+                        "annotated_sentence": annotated_sentence_display,
+                        "is_deleted": is_deleted,
+                        "annotated_at": annotation.annotated_at.isoformat() if annotation.annotated_at else "",
+                        "validation_status": "Loading...",  # Will be loaded on-demand
+                        "validation_loaded": False  # Track if validation status has been loaded
+                    })
+                
+                # Combine with existing items
+                all_items = items + new_items
+                log.info(f"Loaded {len(new_items)} more items, total now: {len(all_items)}")
+                return all_items, total_count
 
         # Output definitions
         review_display_outputs = [

data/models.py

@@ -158,4 +158,23 @@ class Validation(Base):
     validated_at = Column(DateTime, nullable=False)
 
     annotation = relationship("Annotation")
-    validator = relationship("Annotator", foreign_keys=[validator_id])  # Fixed: should reference Annotator
+    validator = relationship("Annotator", foreign_keys=[validator_id])  # Fixed: should reference Annotator
+
+
+# --------------------------------------------------------------------------- #
+#                                 UserProgress                                #
+# --------------------------------------------------------------------------- #
+class UserProgress(Base):
+    __tablename__ = "user_progress"
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey("annotators.id"), nullable=False)
+    target_annotator_id = Column(Integer, ForeignKey("annotators.id"), nullable=False)
+    last_reviewed_annotation_id = Column(Integer, ForeignKey("annotations.id"), nullable=True)
+    last_position = Column(Integer, default=0)  # Position in the review list
+    updated_at = Column(DateTime, nullable=False)
+    
+    # Relationships
+    user = relationship("Annotator", foreign_keys=[user_id])
+    target_annotator = relationship("Annotator", foreign_keys=[target_annotator_id])
+    last_reviewed_annotation = relationship("Annotation", foreign_keys=[last_reviewed_annotation_id])

utils/database.py

@@ -21,20 +21,29 @@ log = Logger()
 
 def get_db_engine():
     """Create DB engine with error handling for HF Spaces"""
-    # try:
-    engine = create_engine(
-        conf.db_url,
-        pool_pre_ping=True,
-        pool_size=5,
-        max_overflow=10,
-        connect_args={"connect_timeout": 10},
-    )
-    log.info("Database engine created successfully")
-    return engine
-    # except Exception as e:
-    #     log.error(f"Failed to create database engine: {e}")
-    #     sentry_sdk.capture_exception(e)
-    #     raise
+    try:
+        # SQLite doesn't support some MySQL-specific parameters
+        if conf.db_url.startswith("sqlite"):
+            engine = create_engine(
+                conf.db_url,
+                pool_pre_ping=True,
+                pool_size=5,
+                max_overflow=10,
+            )
+        else:
+            engine = create_engine(
+                conf.db_url,
+                pool_pre_ping=True,
+                pool_size=5,
+                max_overflow=10,
+                connect_args={"connect_timeout": 10},
+            )
+        log.info("Database engine created successfully")
+        return engine
+    except Exception as e:
+        log.error(f"Failed to create database engine: {e}")
+        sentry_sdk.capture_exception(e)
+        raise
 
 
 engine = get_db_engine()