sync_data.sh

#!/bin/bash

if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
    echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
    exec uvicorn app.main:app --host 0.0.0.0 --port 7860
    exit 0
fi

export HUGGING_FACE_HUB_TOKEN=$HF_TOKEN

restore_latest() {
    echo "Restoring latest backup..."
    python3 -c "
from huggingface_hub import HfApi
import os
api = HfApi()
files = api.list_repo_files(repo_id='${DATASET_ID}', repo_type='dataset')
backup_files = sorted([f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')])
if backup_files:
    latest = backup_files[-1]
    api.hf_hub_download(repo_id='${DATASET_ID}', filename=latest, repo_type='dataset', local_dir='/tmp')
    os.system(f'tar -xzf /tmp/{latest} -C /app/app')
    os.remove(f'/tmp/{latest}')
    print(f'Restored from {latest}')
else:
    print('No backup found')
"
}

sync_data() {
    while true; do
        echo "Starting backup at $(date)"
        cd /app
        timestamp=$(date +%Y%m%d_%H%M%S)
        backup_file="backup_${timestamp}.tar.gz"
        if [ -d "app" ] && [ "$(ls -A app 2>/dev/null)" ]; then
            tar -czf "/tmp/${backup_file}" app/
            python3 -c "
from huggingface_hub import HfApi
api = HfApi()
api.upload_file(path_or_fileobj='/tmp/${backup_file}', path_in_repo='${backup_file}', repo_id='${DATASET_ID}', repo_type='dataset')
print('Backup uploaded successfully')
backup_files = sorted([f for f in api.list_repo_files('${DATASET_ID}', repo_type='dataset') if f.startswith('backup_')])
for old_backup in backup_files[:-50]:
    api.delete_file(path_in_repo=old_backup, repo_id='${DATASET_ID}', repo_type='dataset')
    print(f'Deleted old backup: {old_backup}')
"
            rm -f "/tmp/${backup_file}"
        else
            echo "No data to backup"
        fi
        sleep ${SYNC_INTERVAL:-7200}
    done
}

(
    restore_latest
    sync_data &
    exec uvicorn app.main:app --host 0.0.0.0 --port 7860
) 2>&1 | tee -a /app/data/backup.log