File size: 1,991 Bytes
398ee84 9506a5d 398ee84 2ebc237 9506a5d 2ebc237 9506a5d 398ee84 9506a5d 398ee84 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
#!/bin/bash
if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
exec uvicorn app.main:app --host 0.0.0.0 --port 7860
exit 0
fi
export HUGGING_FACE_HUB_TOKEN=$HF_TOKEN
restore_latest() {
echo "Restoring latest backup..."
python3 -c "
from huggingface_hub import HfApi
import os
api = HfApi()
files = api.list_repo_files(repo_id='${DATASET_ID}', repo_type='dataset')
backup_files = sorted([f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')])
if backup_files:
latest = backup_files[-1]
api.hf_hub_download(repo_id='${DATASET_ID}', filename=latest, repo_type='dataset', local_dir='/tmp')
os.system(f'tar -xzf /tmp/{latest} -C /app/app')
os.remove(f'/tmp/{latest}')
print(f'Restored from {latest}')
else:
print('No backup found')
"
}
sync_data() {
while true; do
echo "Starting backup at $(date)"
cd /app
timestamp=$(date +%Y%m%d_%H%M%S)
backup_file="backup_${timestamp}.tar.gz"
if [ -d "app" ] && [ "$(ls -A app 2>/dev/null)" ]; then
tar -czf "/tmp/${backup_file}" app/
python3 -c "
from huggingface_hub import HfApi
api = HfApi()
api.upload_file(path_or_fileobj='/tmp/${backup_file}', path_in_repo='${backup_file}', repo_id='${DATASET_ID}', repo_type='dataset')
print('Backup uploaded successfully')
backup_files = sorted([f for f in api.list_repo_files('${DATASET_ID}', repo_type='dataset') if f.startswith('backup_')])
for old_backup in backup_files[:-50]:
api.delete_file(path_in_repo=old_backup, repo_id='${DATASET_ID}', repo_type='dataset')
print(f'Deleted old backup: {old_backup}')
"
rm -f "/tmp/${backup_file}"
else
echo "No data to backup"
fi
sleep ${SYNC_INTERVAL:-7200}
done
}
(
restore_latest
sync_data &
exec uvicorn app.main:app --host 0.0.0.0 --port 7860
) 2>&1 | tee -a /app/data/backup.log
|