Create sync_data.sh

sync_data.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# 检查环境变量
+if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
+    echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
+    exec python main.py
+fi
+
+# 登录HuggingFace (使用环境变量方式避免交互问题)
+export HUGGING_FACE_HUB_TOKEN=$HF_TOKEN
+
+# 同步函数
+sync_data() {
+    while true; do
+        echo "Starting sync process at $(date)"
+        
+        # 创建临时压缩文件
+        cd /app
+        timestamp=$(date +%Y%m%d_%H%M%S)
+        backup_file="backup_${timestamp}.tar.gz"
+        
+        tar -czf "/tmp/${backup_file}" data/
+        
+        python3 -c "
+from huggingface_hub import HfApi
+import os
+def manage_backups(api, repo_id, max_files=50):
+    files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
+    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
+    backup_files.sort()
+    
+    if len(backup_files) >= max_files:
+        files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
+        for file_to_delete in files_to_delete:
+            try:
+                api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
+                print(f'Deleted old backup: {file_to_delete}')
+            except Exception as e:
+                print(f'Error deleting {file_to_delete}: {str(e)}')
+try:
+    api = HfApi()
+    api.upload_file(
+        path_or_fileobj='/tmp/${backup_file}',
+        path_in_repo='${backup_file}',
+        repo_id='${DATASET_ID}',
+        repo_type='dataset'
+    )
+    print('Backup uploaded successfully')
+    
+    manage_backups(api, '${DATASET_ID}')
+except Exception as e:
+    print(f'Backup failed: {str(e)}')
+"
+        # 清理临时文件
+        rm -f "/tmp/${backup_file}"
+        
+        # 设置同步间隔
+        SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
+        echo "Next sync in ${SYNC_INTERVAL} seconds..."
+        sleep $SYNC_INTERVAL
+    done
+}
+
+# 恢复函数
+restore_latest() {
+    echo "Attempting to restore latest backup..."
+    python3 -c "
+try:
+    from huggingface_hub import HfApi
+    import os
+    
+    api = HfApi()
+    files = api.list_repo_files('${DATASET_ID}', repo_type='dataset')
+    backup_files = [f for f in files if f.startswith('backup_') and f.endswith('.tar.gz')]
+    
+    if backup_files:
+        latest = sorted(backup_files)[-1]
+        api.hf_hub_download(
+            repo_id='${DATASET_ID}',
+            filename=latest,
+            repo_type='dataset',
+            local_dir='/tmp'
+        )
+        os.system(f'tar -xzf /tmp/{latest} -C /app')
+        os.remove(f'/tmp/{latest}')
+        print(f'Restored from {latest}')
+    else:
+        print('No backup found')
+except Exception as e:
+    print(f'Restore failed: {str(e)}')
+"
+}
+
+# 主程序
+(
+    # 尝试恢复
+    restore_latest
+    
+    # 启动同步进程
+    sync_data &
+    
+    # 启动主应用
+    exec python main.py
+) 2>&1 | tee -a /app/data/backup.log