Update sync_data.sh

sync_data.sh

@@ -1,118 +1,61 @@
 #!/bin/bash
 
-# 检查环境变量
 if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
     echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
     exec uvicorn app.main:app --host 0.0.0.0 --port 7860
     exit 0
 fi
 
-# 登录HuggingFace (使用环境变量方式避免交互问题)
 export HUGGING_FACE_HUB_TOKEN=$HF_TOKEN
 
-# 确保data目录存在
-mkdir -p /app/data
-
-# 同步函数
-sync_data() {
-    while true; do
-     echo "Starting sync process at $(date)"
-     
-     # 创建临时压缩文件
-     cd /app
-        timestamp=$(date +%Y%m%d_%H%M%S)
-    backup_file="backup_${timestamp}.tar.gz"
-    
-    # 检查data目录是否存在且不为空
-     if [ -d "data" ] && [ "$(ls -A data 2>/dev/null)" ]; then
-   tar -czf "/tmp/${backup_file}" data/
-   
-      # 上传备份并管理历史备份
-      python3 -c "
-from huggingface_hub import HfApi
-import os
-def manage_backups(api, repo_id, max_files=50):
-    files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
-    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
-  backup_files.sort()
-    
-    if len(backup_files) >= max_files:
-     files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
-      for file_to_delete in files_to_delete:
-   try:
-    api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
-    print(f'Deleted old backup: {file_to_delete}')
-      except Exception as e:
-    print(f'Error deleting {file_to_delete}: {str(e)}')
-try:
-  api = HfApi()
-    api.upload_file(
-     path_or_fileobj='/tmp/${backup_file}',
-     path_in_repo='${backup_file}',
-     repo_id='${DATASET_ID}',
-      repo_type='dataset'
-    )
-    print('Backup uploaded successfully')
-    
-    manage_backups(api, '${DATASET_ID}')
-except Exception as e:
-    print(f'Backup failed: {str(e)}')
-"
-      # 清理临时文件
-      rm -f "/tmp/${backup_file}"
-   echo "Backup completed"
-     else
-         echo "No data to backup or data directory not found"
-     fi
-     
-     # 设置同步间隔
-     SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
-     echo "Next sync in ${SYNC_INTERVAL} seconds..."
-        sleep $SYNC_INTERVAL
-    done
-}
-
 restore_latest() {
-    echo "Attempting to restore latest backup..."
+    echo "Restoring latest backup..."
     python3 -c "
 from huggingface_hub import HfApi
 import os
+api = HfApi()
+files = api.list_repo_files(repo_id='${DATASET_ID}', repo_type='dataset')
+backup_files = sorted([f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')])
+if backup_files:
+    latest = backup_files[-1]
+    api.hf_hub_download(repo_id='${DATASET_ID}', filename=latest, repo_type='dataset', local_dir='/tmp')
+    os.system(f'tar -xzf /tmp/{latest} -C /app/app')
+    os.remove(f'/tmp/{latest}')
+    print(f'Restored from {latest}')
+else:
+    print('No backup found')
+"
+}
 
-try:
-    api = HfApi()
-    files = api.list_repo_files('${DATASET_ID}', repo_type='dataset')
-    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
+sync_data() {
+    while true; do
+        echo "Starting backup at $(date)"
+        cd /app
+        timestamp=$(date +%Y%m%d_%H%M%S)
+        backup_file="backup_${timestamp}.tar.gz"
 
-    if backup_files:
-        latest = sorted(backup_files)[-1]
-        api.hf_hub_download(
-            repo_id='${DATASET_ID}',
-            filename=latest,
-            repo_type='dataset',
-            local_dir='/tmp'
-        )
-        os.system(f'tar -xzf /tmp/{latest} -C /app')
-        os.remove(f'/tmp/{latest}')
-        print(f'Restored from {latest}')
-    else:
-        print('No backup found')
-except Exception as e:
-    print(f'Restore failed: {str(e)}')
+        if [ -d "app" ] && [ "$(ls -A app 2>/dev/null)" ]; then
+            tar -czf "/tmp/${backup_file}" app/
+            python3 -c "
+from huggingface_hub import HfApi
+api = HfApi()
+api.upload_file(path_or_fileobj='/tmp/${backup_file}', path_in_repo='${backup_file}', repo_id='${DATASET_ID}', repo_type='dataset')
+print('Backup uploaded successfully')
+backup_files = sorted([f for f in api.list_repo_files('${DATASET_ID}', repo_type='dataset') if f.startswith('backup_')])
+for old_backup in backup_files[:-50]:
+    api.delete_file(path_in_repo=old_backup, repo_id='${DATASET_ID}', repo_type='dataset')
+    print(f'Deleted old backup: {old_backup}')
 "
+            rm -f "/tmp/${backup_file}"
+        else
+            echo "No data to backup"
+        fi
+        sleep ${SYNC_INTERVAL:-7200}
+    done
 }
 
-
-# 主程序
 (
-    # 安装huggingface_hub库
-    pip install --quiet huggingface_hub
-    
-  # 尝试恢复
     restore_latest
-    
-    # 启动同步进程
     sync_data &
-    
-    # 启动主应用(适配DeepClaude的启动命令)
     exec uvicorn app.main:app --host 0.0.0.0 --port 7860
 ) 2>&1 | tee -a /app/data/backup.log