nbugs commited on
Commit
7011ee0
·
verified ·
1 Parent(s): feffb8b

Update sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +68 -11
sync_data.sh CHANGED
@@ -6,7 +6,7 @@ if [[ -z "$HF_TOKEN" ]] || [[ -z "$DATASET_ID" ]]; then
6
  exit 0
7
  fi
8
 
9
- # 检查Tunnel
10
  check_tunnel() {
11
  while ! curl -s http://localhost:7860 >/dev/null; do
12
  echo "Waiting for Cloudflare Tunnel to be ready..."
@@ -17,7 +17,7 @@ check_tunnel() {
17
  # 激活虚拟环境
18
  source /opt/venv/bin/activate
19
 
20
- # 上传备份(新增备份数量管理)
21
  upload_backup() {
22
  file_path="$1"
23
  file_name="$2"
@@ -30,21 +30,25 @@ import os
30
 
31
  def manage_backups(api, repo_id, max_files=50):
32
  files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
33
- # 注意前缀改为vaultwarden_backup_
34
  backup_files = [f for f in files if f.startswith('vaultwarden_backup_') and f.endswith('.tar.gz')]
35
  backup_files.sort()
36
  if len(backup_files) >= max_files:
37
- files_to_delete = backup_files[:(len(backup_files) - max_files + 1)]
 
38
  for file_to_delete in files_to_delete:
39
  try:
40
- api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset')
 
 
 
 
41
  print(f'Deleted old backup: {file_to_delete}')
42
  except Exception as e:
43
  print(f'Error deleting {file_to_delete}: {str(e)}')
44
 
45
  api = HfApi(token='$token')
46
  try:
47
- # 先上传新备份
48
  api.upload_file(
49
  path_or_fileobj='$file_path',
50
  path_in_repo='$file_name',
@@ -53,14 +57,64 @@ try:
53
  )
54
  print(f'Successfully uploaded $file_name')
55
 
56
- # 上传完成后执行备份数量管理
57
  manage_backups(api, '$repo_id')
58
  except Exception as e:
59
  print(f'Error uploading file: {str(e)}')
60
  "
61
  }
62
 
63
- # [...] 后续的download_latest_backup和sync_data函数保持不变
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
 
65
  # 同步函数
66
  sync_data() {
@@ -69,7 +123,7 @@ sync_data() {
69
  if [ -d /data ]; then
70
  timestamp=$(date +%Y%m%d_%H%M%S)
71
  backup_file="vaultwarden_backup_${timestamp}.tar.gz"
72
- # 压缩数据目录
73
  tar -czf "/tmp/${backup_file}" -C /data .
74
  echo "Uploading backup to HuggingFace..."
75
  upload_backup "/tmp/${backup_file}" "${backup_file}"
@@ -83,8 +137,11 @@ sync_data() {
83
  done
84
  }
85
 
86
- # 后台启动同步进程
 
 
 
87
  sync_data &
88
 
89
- # 启动 Vaultwarden
90
  exec /start.sh
 
6
  exit 0
7
  fi
8
 
9
+ # 等待Cloudflare Tunnel就绪
10
  check_tunnel() {
11
  while ! curl -s http://localhost:7860 >/dev/null; do
12
  echo "Waiting for Cloudflare Tunnel to be ready..."
 
17
  # 激活虚拟环境
18
  source /opt/venv/bin/activate
19
 
20
+ # 上传备份(含自动清理旧备份)
21
  upload_backup() {
22
  file_path="$1"
23
  file_name="$2"
 
30
 
31
  def manage_backups(api, repo_id, max_files=50):
32
  files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
 
33
  backup_files = [f for f in files if f.startswith('vaultwarden_backup_') and f.endswith('.tar.gz')]
34
  backup_files.sort()
35
  if len(backup_files) >= max_files:
36
+ delete_count = len(backup_files) - max_files + 1
37
+ files_to_delete = backup_files[:delete_count]
38
  for file_to_delete in files_to_delete:
39
  try:
40
+ api.delete_file(
41
+ path_in_repo=file_to_delete,
42
+ repo_id=repo_id,
43
+ repo_type='dataset'
44
+ )
45
  print(f'Deleted old backup: {file_to_delete}')
46
  except Exception as e:
47
  print(f'Error deleting {file_to_delete}: {str(e)}')
48
 
49
  api = HfApi(token='$token')
50
  try:
51
+ # 上传新备份
52
  api.upload_file(
53
  path_or_fileobj='$file_path',
54
  path_in_repo='$file_name',
 
57
  )
58
  print(f'Successfully uploaded $file_name')
59
 
60
+ # 清理旧备份
61
  manage_backups(api, '$repo_id')
62
  except Exception as e:
63
  print(f'Error uploading file: {str(e)}')
64
  "
65
  }
66
 
67
+ # 下载最新备份(完整实现)
68
+ download_latest_backup() {
69
+ token="$HF_TOKEN"
70
+ repo_id="$DATASET_ID"
71
+ python3 -c "
72
+ from huggingface_hub import HfApi
73
+ import sys
74
+ import os
75
+ import tarfile
76
+ import tempfile
77
+ import shutil
78
+
79
+ api = HfApi(token='$token')
80
+ try:
81
+ files = api.list_repo_files(repo_id='$repo_id', repo_type='dataset')
82
+ backup_files = [f for f in files if f.startswith('vaultwarden_backup_') and f.endswith('.tar.gz')]
83
+ if not backup_files:
84
+ print('No backup files found')
85
+ sys.exit()
86
+ latest_backup = sorted(backup_files)[-1]
87
+ with tempfile.TemporaryDirectory() as temp_dir:
88
+ filepath = api.hf_hub_download(
89
+ repo_id='$repo_id',
90
+ filename=latest_backup,
91
+ repo_type='dataset',
92
+ local_dir=temp_dir
93
+ )
94
+ if filepath and os.path.exists(filepath):
95
+ # 确保数据目录存在
96
+ os.makedirs('/data', exist_ok=True)
97
+ # 清空旧数据(防止残留文件干扰)
98
+ if os.listdir('/data'):
99
+ for f in os.listdir('/data'):
100
+ item_path = os.path.join('/data', f)
101
+ if os.path.isfile(item_path):
102
+ os.remove(item_path)
103
+ elif os.path.isdir(item_path):
104
+ shutil.rmtree(item_path)
105
+ # 解压备份文件
106
+ with tarfile.open(filepath, 'r:gz') as tar:
107
+ tar.extractall('/data')
108
+ print(f'Successfully restored backup from {latest_backup}')
109
+ except Exception as e:
110
+ print(f'Error downloading backup: {str(e)}')
111
+ sys.exit(1)
112
+ "
113
+ }
114
+
115
+ # 首次启动时下载最新备份
116
+ echo "Downloading latest backup from HuggingFace..."
117
+ download_latest_backup
118
 
119
  # 同步函数
120
  sync_data() {
 
123
  if [ -d /data ]; then
124
  timestamp=$(date +%Y%m%d_%H%M%S)
125
  backup_file="vaultwarden_backup_${timestamp}.tar.gz"
126
+ # 压缩数据目录(排除临时文件)
127
  tar -czf "/tmp/${backup_file}" -C /data .
128
  echo "Uploading backup to HuggingFace..."
129
  upload_backup "/tmp/${backup_file}" "${backup_file}"
 
137
  done
138
  }
139
 
140
+ # 启动Cloudflare Tunnel检查
141
+ check_tunnel &
142
+
143
+ # 启动备份同步进程
144
  sync_data &
145
 
146
+ # 启动Vaultwarden主程序
147
  exec /start.sh