sync_data.sh

#!/bin/bash

if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
    echo "Starting without backup - missing HF_TOKEN or DATASET_ID"
    exec uvicorn app.main:app --host 0.0.0.0 --port 7860
    exit 0
fi

export HUGGING_FACE_HUB_TOKEN=$HF_TOKEN

restore_latest() {
    echo "正在检查是否有可用备份..."
    python3 -c "
from huggingface_hub import HfApi
import os

api = HfApi()
files = api.list_repo_files('${DATASET_ID}', repo_type='dataset')
backup_files = sorted([f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')])

if backup_files:
    latest = backup_files[-1]
    print(f'发现备份文件: {latest}, 正在下载...')
    api.hf_hub_download(repo_id='${DATASET_ID}', filename=latest, repo_type='dataset', local_dir='/tmp')
    # 解压文件并禁止恢复原始权限和所有者，避免容器内权限错误
    result = os.system(f'tar --no-same-owner --no-same-permissions -xzf /tmp/{latest} -C /app')
    if result == 0:
        print(f'成功从 {latest} 恢复备份数据')
    else:
        print(f'解压备份文件 {latest} 时出现问题，请检查相关日志')
    os.remove(f'/tmp/{latest}')
else:
    print('未找到备份文件，已跳过数据恢复步骤')
"
}



sync_data() {
    echo "Waiting ${SYNC_INTERVAL:-7200} seconds before first backup..."
    sleep ${SYNC_INTERVAL:-7200}

    while true; do
        echo "Starting backup at $(date)"
        cd /app
        timestamp=$(date +%Y%m%d_%H%M%S)
        backup_file="backup_${timestamp}.tar.gz"

        if [ -d "app" ] && [ "$(ls -A app 2>/dev/null)" ]; then
            tar -czf "/tmp/${backup_file}" app/
            python3 -c "
from huggingface_hub import HfApi
api = HfApi()
api.upload_file('/tmp/${backup_file}', '${backup_file}', '${DATASET_ID}', repo_type='dataset')
print('Backup uploaded successfully')
backup_files = sorted([f for f in api.list_repo_files('${DATASET_ID}', repo_type='dataset') if f.startswith('backup_')])
for old_backup in backup_files[:-50]:
    api.delete_file(path_in_repo=old_backup, repo_id='${DATASET_ID}', repo_type='dataset')
    print(f'Deleted old backup: {old_backup}')
"
            rm -f "/tmp/${backup_file}"
        else
            echo "No data to backup"
        fi
        echo "Next backup in ${SYNC_INTERVAL:-7200} seconds..."
        sleep ${SYNC_INTERVAL:-7200}
    done
}

(
    restore_latest
    sync_data &
    exec uvicorn app.main:app --host 0.0.0.0 --port 7860
) 2>&1 | tee -a /app/data/backup.log