Update sync_data.sh

sync_data.sh

@@ -2,18 +2,28 @@
 
 # 检查必要的环境变量
 if [ -z "$WEBDAV_URL" ] || [ -z "$WEBDAV_USERNAME" ] || [ -z "$WEBDAV_PASSWORD" ]; then
-    echo "缺少必要的环境变量 WEBDAV_URL、WEBDAV_USERNAME 或 WEBDAV_PASSWORD"
+    echo "缺少必要的环境变量: WEBDAV_URL、WEBDAV_USERNAME 或 WEBDAV_PASSWORD"
     exit 1
 fi
 
 if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
-    echo "缺少必要的环境变量 HF_TOKEN 或 DATASET_ID"
+    echo "缺少必要的环境变量: HF_TOKEN 或 DATASET_ID"
     exit 1
 fi
 
 # 创建数据目录
 mkdir -p ./data
 
+# 定义哈希计算函数
+get_file_hash() {
+    local file_path="$1"
+    if [ -f "$file_path" ]; then
+        md5sum "$file_path" | awk '{print $1}'
+    else
+        echo "文件不存在"
+    fi
+}
+
 # 创建 Hugging Face 同步脚本
 cat > /tmp/hf_sync.py << 'EOL'
 from huggingface_hub import HfApi
@@ -21,234 +31,206 @@ import sys
 import os
 
 def manage_backups(api, repo_id, max_files=50):
-  """管理备份文件，保留最新的max_files个文件"""
-  files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
+    """管理备份文件，保留最新的max_files个文件"""
+    files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
     backup_files = [f for f in files if f.startswith('webui_backup_') and f.endswith('.db')]
-
+    
     # 按日期分组文件（从文件名中提取日期）
     backup_by_date = {}
     for file in backup_files:
-    try:
-   # 从文件名格式 webui_backup_YYYYMMDD.db 中提取日期部分
-       date_part = file.split('_')[2].split('.')[0]  # 提取YYYYMMDD部分
-      backup_by_date[date_part] = file
-     except:
-    # 如果文件名格式不符，则跳过
-   continue
-  
-    # 日期排序并仅保留最新max_files个文件
-    sorted_dates = sorted(backup_by_date.keys())
+        try:
+            date_part = file.split('_')[2].split('.')[0]
+            backup_by_date[date_part] = file
+        except:
+            continue
+    
+    # 保留最新的max_files个文件
+    sorted_dates = sorted(backup_by_date.keys(), reverse=True)
     if len(sorted_dates) > max_files:
-     dates_to_delete = sorted_dates[:(len(sorted_dates) - max_files)]
-     for date in dates_to_delete:
-   try:
-    api.delete_file(path_in_repo=backup_by_date[date], repo_id=repo_id, repo_type="dataset")
-    print(f'已删除旧备份: {backup_by_date[date]}')
-      except Exception as e:
-       print(f'删除 {backup_by_date[date]} 时出错: {str(e)}')
+        files_to_delete = [backup_by_date[date] for date in sorted_dates[max_files:]]
+        for file in files_to_delete:
+            api.delete_file(path_in_repo=file, repo_id=repo_id, repo_type="dataset")
+            print(f"已删除旧备份: {file}")
 
 def upload_backup(file_path, file_name, token, repo_id):
     """上传备份文件到Hugging Face"""
     api = HfApi(token=token)
     try:
-     # 检查同名文件是否已存在
-     files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-    if file_name in files:
-      # 上传前删除同名文件
-      api.delete_file(path_in_repo=file_name, repo_id=repo_id, repo_type="dataset")
-      print(f"已删除同名文件: {file_name}")
-    
-      api.upload_file(
-      path_or_fileobj=file_path,
-      path_in_repo=file_name,
-      repo_id=repo_id,
-      repo_type="dataset"
-     )
-     print(f"成功上传 {file_name}")
-     manage_backups(api, repo_id)
+        # 删除同名文件（如有）
+        files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
+        if file_name in files:
+            api.delete_file(path_in_repo=file_name, repo_id=repo_id, repo_type="dataset")
+            print(f"已删除同名文件: {file_name}")
+        
+        # 上传新文件
+        api.upload_file(
+            path_or_fileobj=file_path,
+            path_in_repo=file_name,
+            repo_id=repo_id,
+            repo_type="dataset"
+        )
+        print(f"成功上传: {file_name}")
+        manage_backups(api, repo_id)
     except Exception as e:
-     print(f"文件上传出错: {str(e)}")
+        print(f"上传失败: {str(e)}")
 
-# 下载最新备份
 def download_latest_backup(token, repo_id):
+    """从Hugging Face下载最新备份"""
+    api = HfApi(token=token)
     try:
-     api = HfApi(token=token)
-     files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-     backup_files = [f for f in files if f.startswith('webui_backup_') and f.endswith('.db')]
-     if not backup_files:
-      print("未找到备份文件")
-   return False
-   
-     latest_backup = sorted(backup_files)[-1]
-    file = api.hf_hub_download(
-      repo_id=repo_id,
-      filename=latest_backup,
-      repo_type="dataset"
-     )
-   
-     if file and os.path.exists(file):
-        os.makedirs('./data', exist_ok=True)
-        os.system(f'cp "{file}" ./data/webui.db')
-      print(f"成功从 {latest_backup} 恢复备份")
-      return True
-      
-        return False
+        files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
+        backup_files = [f for f in files if f.startswith('webui_backup_') and f.endswith('.db')]
+        if not backup_files:
+            return False
+        
+        # 找到最新的文件（按日期排序）
+        latest_file = max(backup_files, key=lambda x: x.split('_')[2].split('.')[0])
+        file_path = api.hf_hub_download(
+            repo_id=repo_id,
+            filename=latest_file,
+            repo_type="dataset"
+        )
+        
+        if file_path and os.path.exists(file_path):
+            os.makedirs('./data', exist_ok=True)
+            os.system(f'cp "{file_path}" ./data/webui.db')
+            print(f"成功从Hugging Face恢复: {latest_file}")
+            return True
+        else:
+            return False
     except Exception as e:
-        print(f"下载备份时出错: {str(e)}")
+        print(f"下载失败: {str(e)}")
         return False
 
 if __name__ == "__main__":
     action = sys.argv[1]
     token = sys.argv[2]
     repo_id = sys.argv[3]
-  
+    
     if action == "upload":
-     file_path = sys.argv[4]
-     file_name = sys.argv[5]
-     upload_backup(file_path, file_name, token, repo_id)
+        file_path = sys.argv[4]
+        file_name = sys.argv[5]
+        upload_backup(file_path, file_name, token, repo_id)
     elif action == "download":
-      download_latest_backup(token, repo_id)
+        download_latest_backup(token, repo_id)
 EOL
 
-# 首次启动时的数据恢复策略
-echo "开始初始化数据恢复..."
-
-# 打印 WebDAV 信息（隐藏密码）
+# 初始化数据恢复策略
+echo "初始化数据恢复..."
 echo "WebDAV URL: $WEBDAV_URL"
 echo "WebDAV 用户名: $WEBDAV_USERNAME"
-masked_password=$(echo $WEBDAV_PASSWORD | sed 's/./*/g')
-echo "WebDAV 密码: $masked_password"
-
-# 首先尝试从 WebDAV 恢复最新文件
-echo "正在尝试从 WebDAV 获取文件列表..."
-echo "PROPFIND 请求: $WEBDAV_URL/openwebui/"
-
-# 获取并打印 WebDAV 目录内容
-webdav_list_output=$(curl -v -X PROPFIND --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" -H "Depth: 1" "$WEBDAV_URL/openwebui/" 2>&1)
-echo "WebDAV 目录内容响应:"
-echo "$webdav_list_output"
+echo "WebDAV 密码: $(echo $WEBDAV_PASSWORD | sed 's/./*/g')"
 
-# 获取 WebDAV 目录中的文件列表并找出最新的备份文件
-webdav_files=$(echo "$webdav_list_output" | grep -o '<d:href>[^<]*webui_[0-9]\{4\}[0-9]\{2\}[0-9]\{2\}\.db</d:href>' | sed 's/<d:href>//g' | sed 's/<\/d:href>//g')
+# 尝试从 WebDAV 恢复最新文件
+echo "尝试从 WebDAV 获取文件列表..."
+webdav_files=$(curl -s -X PROPFIND --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" -H "Depth: 1" "$WEBDAV_URL/openwebui/" | grep '<d:href>' | grep 'webui_[0-9]\{8\}.db' | sed 's|</?d:href>||g')
 
 if [ -n "$webdav_files" ]; then
-  # 找出最新的文件 (按文件名排序，取最后一个)
-  latest_file=$(echo "$webdav_files" | sort | tail -n 1)
-    echo "找到最新的 WebDAV 备份文件: $latest_file"
-  
-  # 构建并打印完整下载链接
-  download_url="$WEBDAV_URL$latest_file"
-    echo "下载 URL: $download_url"
-  
-    # 下载最新文件
-    curl -v -L --fail --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$download_url" -o "./data/webui.db" && {
-     echo "成功从 WebDAV 下载最新数据库文件: $latest_file"
-   
-     # 上传到 HuggingFace
-     DATE_FORMAT=$(date +%Y%m%d)
-  backup_file="webui_backup_${DATE_FORMAT}.db"
-    cp ./data/webui.db "/tmp/${backup_file}"
-     echo "正在上传初始备份到 Hugging Face..."
-     python /tmp/hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
-     rm -f "/tmp/${backup_file}"
+    latest_file=$(echo "$webdav_files" | sort -r | head -n 1)
+    download_url="$WEBDAV_URL/openwebui/$latest_file"
+    curl -L -o "./data/webui.db" --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$download_url" && {
+        echo "成功从 WebDAV 下载最新数据库: $latest_file"
     } || {
-     echo "从 WebDAV 下载最新文件失败，尝试从 Hugging Face 恢复..."
-  python /tmp/hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}" || {
-   echo "所有恢复方式均失败，将使用空数据库开始"
-      }
+        echo "WebDAV 下载失败，尝试从 Hugging Face 恢复..."
+        python /tmp/hf_sync.py download "$HF_TOKEN" "$DATASET_ID"
     }
 else
-  echo "WebDAV 中没有找到符合格式的备份文件，尝试从 Hugging Face 恢复..."
-  python /tmp/hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}" || {
-    echo "从 Hugging Face 恢复失败，将使用空数据库开始"
+    echo "WebDAV 无有效备份，尝试从 Hugging Face 恢复..."
+    python /tmp/hf_sync.py download "$HF_TOKEN" "$DATASET_ID" || {
+        echo "所有恢复失败，创建空数据库..."
+        touch ./data/webui.db
     }
 fi
 
-# 定义 WebDAV 同步函数
-webdav_sync_() {
-    SYNC_INTERVAL=${SYNC_INTERVAL:-7200}  # 默认间隔时间为 7200 秒
-    echo "WebDAV 同步进程启动，等待 ${SYNC_INTERVAL} 秒后开始同步..."
-    sleep $SYNC_INTERVAL
-  
-    while true; do
-    echo "开始 WebDAV 同步 $(date)"
-   
-     # 检查数据库文件是否存在
-     if [ -f "./data/webui.db" ]; then
-   # 生成当天的日期文件名（YYYYMMDD格式）
-     DATE_FORMAT=$(date +%Y%m%d)
-     FILE="webui_${DATE_FORMAT}.db"
-      echo "同步到 WebDAV..."
-      upload_url="$WEBDAV_URL/openwebui/$FILE"
-      echo "上传 URL: $upload_url"
+# WebDAV 同步函数（仅上传变化文件）
+webdav_sync() {
+    SYNC_INTERVAL=${SYNC_INTERVAL:-7200}  # 默认 2 小时
+    echo "WebDAV 同步启动，间隔: ${SYNC_INTERVAL} 秒"
     
-   # 上传以日期命名的数据库文件
-      curl -v -L -T "./data/webui.db" --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$upload_url" && {
-       echo "WebDAV 上传成功: $FILE"
-  
-       # 删除较旧的文件以管理空间
-    echo "获取 WebDAV 目录以清理旧文件..."
-       webdav_files_list=$(curl -s -X PROPFIND --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" -H "Depth: 1" "$WEBDAV_URL/openwebui/" | grep -o '<d:href>[^<]*webui_[0-9]\{8\}\.db</d:href>' | sed 's/<d:href>//g' | sed 's/<\/d:href>//g')
-     
-    # 保留最新的10个文件
-    if [ -n "$webdav_files_list" ]; then
-        files_to_keep=$(echo "$webdav_files_list" | sort | tail -n 10)
-   
-     # 删除较旧的文件
-     for old_file in $webdav_files_list; do
-   if ! echo "$files_to_keep" | grep -q "$old_file"; then
-    echo "删除旧文件: $old_file"
-       curl -X DELETE --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$WEBDAV_URL$old_file"
-      fi
-     done
-    fi
-  
-         } || {
-    echo "WebDAV 上传失败，等待重试..."
-        sleep 10
-        echo "重试上传 URL: $upload_url"
-       curl -v -L -T "./data/webui.db" --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$upload_url" || {
-     echo "重试失败，放弃上传。"
-    }
-   }
-     else
-   echo "未找到 webui.db 文件，跳过 WebDAV 同步"
-     fi
-   
-     # 等待下一次同步间隔
-     echo "WebDAV 同步完成，下次同步将在 ${SYNC_INTERVAL} 秒后进行..."
-     sleep $SYNC_INTERVAL
+    while true; do
+        echo "开始 WebDAV 同步: $(date)"
+        
+        if [ -f "./data/webui.db" ]; then
+            # 生成文件名（包含年月日）
+            current_date=$(date +'%Y%m%d')
+            file_name="webui_${current_date}.db"
+            upload_url="$WEBDAV_URL/openwebui/${file_name}"
+            
+            # 计算本地文件哈希
+            local_hash=$(get_file_hash "./data/webui.db")
+            
+            # 获取远程文件哈希（通过临时下载）
+            remote_temp="/tmp/webui_remote.db"
+            curl -s -o "$remote_temp" --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$upload_url" > /dev/null 2>&1
+            remote_hash=$(get_file_hash "$remote_temp")
+            rm -f "$remote_temp"
+            
+            if [ "$local_hash" == "$remote_hash" ]; then
+                echo "文件未变化，跳过 WebDAV 上传"
+            else
+                echo "检测到文件变化，开始上传到 WebDAV..."
+                curl -L -T "./data/webui.db" --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$upload_url" && {
+                    echo "WebDAV 上传成功: $file_name"
+                    
+                    # 更新主文件（覆盖 webui.db）
+                    main_url="$WEBDAV_URL/openwebui/webui.db"
+                    curl -L -T "./data/webui.db" --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$main_url" && {
+                        echo "主文件更新成功"
+                    } || {
+                        echo "主文件更新失败"
+                    }
+                } || {
+                    echo "WebDAV 上传失败，等待重试..."
+                    sleep 10
+                    curl -L -T "./data/webui.db" --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$upload_url" || {
+                        echo "重试失败，放弃本次上传"
+                    }
+                }
+            fi
+            
+            # 清理过期 WebDAV 文件（保留最近 7 天）
+            cleanup_days=7
+            cutoff_date=$(date -d "-${cleanup_days} days" +%Y%m%d)
+            for file in $webdav_files; do
+                file_date=$(echo "$file" | grep -oE '[0-9]{8}')
+                if [ "$file_date" -lt "$cutoff_date" ]; then
+                    delete_url="$WEBDAV_URL/openwebui/$file"
+                    curl -X DELETE --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$delete_url" && echo "删除过期文件: $file"
+                fi
+            done
+        else
+            echo "未找到 webui.db，跳过同步"
+        fi
+        
+        sleep $SYNC_INTERVAL
     done
 }
 
-# 定义 Hugging Face 同步函数
-hf_sync_() {
+# Hugging Face 同步函数
+hf_sync() {
     SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
-    echo "Hugging Face 同步进程启动，等待 ${SYNC_INTERVAL} 秒后开始同步..."
-    sleep $(($SYNC_INTERVAL / 2))  # 错开与 WebDAV 同步的时间
-
-    while true; do
-        echo "开始 Hugging Face 同步 $(date)"
-   
-     if [ -f "./data/webui.db" ]; then
-      # 使用仅���日期的格式（年月日）
-      DATE_FORMAT=$(date +%Y%m%d)
-      backup_file="webui_backup_${DATE_FORMAT}.db"
+    echo "Hugging Face 同步启动，间隔: ${SYNC_INTERVAL} 秒"
     
-        # 复制数据库文件
-        cp ./data/webui.db "/tmp/${backup_file}"
-   echo "正在上传备份到 Hugging Face..."
-   python /tmp/hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
-      rm -f "/tmp/${backup_file}"
-     else
-   echo "数据库文件不存在，跳过 Hugging Face 同步"
-     fi
-   
-        echo "Hugging Face 同步完成，下次同步将在 ${SYNC_INTERVAL} 秒后进行..."
-     sleep $SYNC_INTERVAL
+    while true; do
+        echo "开始 Hugging Face 同步: $(date)"
+        
+        if [ -f "./data/webui.db" ]; then
+            current_date=$(date +'%Y%m%d')
+            backup_file="webui_backup_${current_date}.db"
+            temp_path="/tmp/${backup_file}"
+            cp "./data/webui.db" "$temp_path"
+            
+            echo "正在上传到 Hugging Face..."
+            python /tmp/hf_sync.py upload "$HF_TOKEN" "$DATASET_ID" "$temp_path" "$backup_file"
+            rm -f "$temp_path"
+        else
+            echo "未找到数据库文件，跳过 Hugging Face 同步"
+        fi
+        
+        sleep $SYNC_INTERVAL
     done
 }
 
-# 后台启动同步进程
-webdav_sync_ &
-hf_sync_ & 
+# 启动同步进程
+webdav_sync &
+hf_sync &