review_submission.py

import json
import os
from datetime import datetime
import glob
from huggingface_hub import HfApi

REPO_ID = "snap-stanford/stark-leaderboard"

class ForumPost:
    def __init__(self, message: str, timestamp: str, post_type: str):
        self.message = message
        self.timestamp = timestamp
        self.post_type = post_type

def load_forum_posts(forum_file="submissions/forum_posts.json"):
    """Load forum posts from local file"""
    try:
        if os.path.exists(forum_file):
            with open(forum_file, 'r') as f:
                posts_data = json.load(f)
                return [ForumPost(**post) for post in posts_data]
        return []
    except Exception as e:
        print(f"Error loading forum posts: {e}")
        return []

def save_forum_posts(posts, forum_file="submissions/forum_posts.json"):
    """Save forum posts to local file"""
    try:
        posts_data = [
            {
                "message": post.message,
                "timestamp": post.timestamp,
                "post_type": post.post_type
            }
            for post in posts
        ]
        os.makedirs(os.path.dirname(forum_file), exist_ok=True)
        with open(forum_file, 'w') as f:
            json.dump(posts_data, f, indent=4)
    except Exception as e:
        print(f"Error saving forum posts: {e}")

def add_status_update(posts, method_name: str, new_status: str):
    """Add a status update post"""
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    emoji = "✅" if new_status == "approved" else "❌"
    message = f"{emoji} Status update: {method_name} has been {new_status}"
    posts.append(ForumPost(message, timestamp, "status_update"))
    save_forum_posts(posts)

def review_pending_submissions(submissions_dir: str = "submissions"):
    """Review all pending submissions and allow admin to approve/reject them."""
    try:
        # Load forum posts
        forum_posts = load_forum_posts()
        
        # Find all latest.json files
        latest_files = glob.glob(os.path.join(submissions_dir, "**", "latest.json"), recursive=True)
        
        if not latest_files:
            print("No submissions found.")
            return
        
        changes_made = []
        
        # Process each submission
        for latest_file in latest_files:
            try:
                with open(latest_file, 'r') as f:
                    latest_info = json.load(f)
                
                if latest_info.get('status') != 'pending_review':
                    continue
                    
                folder_path = os.path.dirname(latest_file)
                timestamp = latest_info.get('latest_submission')
                if not timestamp:
                    print(f"No timestamp found in {latest_file}, skipping...")
                    continue
                
                metadata_file = os.path.join(folder_path, f"metadata_{timestamp}.json")
                try:
                    with open(metadata_file, 'r') as f:
                        metadata = json.load(f)
                except Exception as e:
                    print(f"Could not read metadata for {latest_file}, skipping...")
                    continue
                
                # Display submission details
                print("\n" + "="*80)
                print(f"Submission Details:")
                print(f"Method Name: {metadata.get('Method Name')}")
                print(f"Team Name: {metadata.get('Team Name')}")
                print(f"Dataset: {metadata.get('Dataset')}")
                print(f"Split: {metadata.get('Split')}")
                print(f"Contact Email(s): {metadata.get('Contact Email(s)')}")
                print(f"Code Repository: {metadata.get('Code Repository')}")
                print("\nModel Description:")
                print(metadata.get('Model Description', 'No description provided'))
                print("\nResults:")
                if 'results' in metadata:
                    for metric, value in metadata['results'].items():
                        print(f"{metric}: {value}")
                print("\nSubmission Date:", metadata.get('submission_date', 'Unknown'))
                print("="*80)
                
                # Get admin decision
                while True:
                    decision = input("\nApprove this submission? (y/n/skip/quit): ").lower()
                    if decision in ['y', 'n', 'skip', 'quit']:
                        break
                    print("Invalid input. Please enter 'y', 'n', 'skip', or 'quit'")
                
                if decision == 'quit':
                    print("Exiting review process...")
                    break
                    
                if decision == 'skip':
                    continue
                    
                # Update status
                new_status = 'approved' if decision == 'y' else 'rejected'
                review_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                
                # Update latest.json
                latest_info['status'] = new_status
                latest_info['reviewed_at'] = review_timestamp
                with open(latest_file, 'w') as f:
                    json.dump(latest_info, f, indent=4)
                changes_made.append(latest_file)
                
                # Update metadata
                metadata['status'] = new_status
                metadata['reviewed_at'] = review_timestamp
                with open(metadata_file, 'w') as f:
                    json.dump(metadata, f, indent=4)
                changes_made.append(metadata_file)
                
                # Add forum post
                add_status_update(forum_posts, metadata.get('Method Name'), new_status)
                changes_made.append("submissions/forum_posts.json")
                
                print(f"Submission {new_status}")
                
            except Exception as e:
                print(f"Error processing {latest_file}: {e}")
                continue
        
        if changes_made:
            print("\nThe following files have been modified:")
            for file in sorted(set(changes_made)):
                print(f"- {file}")
            print("\nPlease push these changes to HuggingFace manually.")
            
    except Exception as e:
        print(f"Error during review process: {str(e)}")

if __name__ == "__main__":
    print("Starting submission review process...")
    review_pending_submissions()
    print("\nReview process completed.")