Create sync_data.sh

sync_data.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# 检查环境变量
+if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
+    echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
+    exec uvicorn app.main:app --host 0.0.0.0 --port 7860
+    exit 0
+fi
+
+# 登录HuggingFace (使用环境变量方式避免交互问题)
+export HUGGING_FACE_HUB_TOKEN=$HF_TOKEN
+
+# 确保data目录存在
+mkdir -p /app/data
+
+# 同步函数
+sync_data() {
+    while true; do
+     echo "Starting sync process at $(date)"
+     
+     # 创建临时压缩文件
+     cd /app
+        timestamp=$(date +%Y%m%d_%H%M%S)
+    backup_file="backup_${timestamp}.tar.gz"
+    
+    # 检查data目录是否存在且不为空
+     if [ -d "data" ] && [ "$(ls -A data 2>/dev/null)" ]; then
+   tar -czf "/tmp/${backup_file}" data/
+   
+      # 上传备份并管理历史备份
+      python3 -c "
+from huggingface_hub import HfApi
+import os
+def manage_backups(api, repo_id, max_files=50):
+    files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
+    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
+  backup_files.sort()
+    
+    if len(backup_files) >= max_files:
+     files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
+      for file_to_delete in files_to_delete:
+   try:
+    api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
+    print(f'Deleted old backup: {file_to_delete}')
+      except Exception as e:
+    print(f'Error deleting {file_to_delete}: {str(e)}')
+try:
+  api = HfApi()
+    api.upload_file(
+     path_or_fileobj='/tmp/${backup_file}',
+     path_in_repo='${backup_file}',
+     repo_id='${DATASET_ID}',
+      repo_type='dataset'
+    )
+    print('Backup uploaded successfully')
+    
+    manage_backups(api, '${DATASET_ID}')
+except Exception as e:
+    print(f'Backup failed: {str(e)}')
+"
+      # 清理临时文件
+      rm -f "/tmp/${backup_file}"
+   echo "Backup completed"
+     else
+         echo "No data to backup or data directory not found"
+     fi
+     
+     # 设置同步间隔
+     SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
+     echo "Next sync in ${SYNC_INTERVAL} seconds..."
+        sleep $SYNC_INTERVAL
+    done
+}
+
+# 恢复函数
+restore_latest() {
+    echo "Attempting to restore latest backup..."
+    python3 -c "
+try:
+    from huggingface_hub import HfApi
+    import os
+    
+    api = HfApi()
+    files = api.list_repo_files('${DATASET_ID}', repo_type='dataset')
+    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
+    
+    if backup_files:
+     latest = sorted(backup_files)[-1]
+        api.hf_hub_download(
+       repo_id='${DATASET_ID}',
+      filename=latest,
+      repo_type='dataset',
+         local_dir='/tmp'
+        )
+        os.system(f'tar -xzf /tmp/{latest} -C /app')
+     os.remove(f'/tmp/{latest}')
+     print(f'Restored from {latest}')
+    else:
+     print('No backup found')
+except Exception as e:
+    print(f'Restore failed: {str(e)}')
+"
+}
+
+# 主程序
+(
+    # 安装huggingface_hub库
+    pip install --quiet huggingface_hub
+    
+  # 尝试恢复
+    restore_latest
+    
+    # 启动同步进程
+    sync_data &
+    
+    # 启动主应用(适配DeepClaude的启动命令)
+    exec uvicorn app.main:app --host 0.0.0.0 --port 7860
+) 2>&1 | tee -a /app/data/backup.log