Update sync_data.sh
Browse files- sync_data.sh +68 -11
sync_data.sh
CHANGED
@@ -6,7 +6,7 @@ if [[ -z "$HF_TOKEN" ]] || [[ -z "$DATASET_ID" ]]; then
|
|
6 |
exit 0
|
7 |
fi
|
8 |
|
9 |
-
#
|
10 |
check_tunnel() {
|
11 |
while ! curl -s http://localhost:7860 >/dev/null; do
|
12 |
echo "Waiting for Cloudflare Tunnel to be ready..."
|
@@ -17,7 +17,7 @@ check_tunnel() {
|
|
17 |
# 激活虚拟环境
|
18 |
source /opt/venv/bin/activate
|
19 |
|
20 |
-
#
|
21 |
upload_backup() {
|
22 |
file_path="$1"
|
23 |
file_name="$2"
|
@@ -30,21 +30,25 @@ import os
|
|
30 |
|
31 |
def manage_backups(api, repo_id, max_files=50):
|
32 |
files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
|
33 |
-
# 注意前缀改为vaultwarden_backup_
|
34 |
backup_files = [f for f in files if f.startswith('vaultwarden_backup_') and f.endswith('.tar.gz')]
|
35 |
backup_files.sort()
|
36 |
if len(backup_files) >= max_files:
|
37 |
-
|
|
|
38 |
for file_to_delete in files_to_delete:
|
39 |
try:
|
40 |
-
api.delete_file(
|
|
|
|
|
|
|
|
|
41 |
print(f'Deleted old backup: {file_to_delete}')
|
42 |
except Exception as e:
|
43 |
print(f'Error deleting {file_to_delete}: {str(e)}')
|
44 |
|
45 |
api = HfApi(token='$token')
|
46 |
try:
|
47 |
-
#
|
48 |
api.upload_file(
|
49 |
path_or_fileobj='$file_path',
|
50 |
path_in_repo='$file_name',
|
@@ -53,14 +57,64 @@ try:
|
|
53 |
)
|
54 |
print(f'Successfully uploaded $file_name')
|
55 |
|
56 |
-
#
|
57 |
manage_backups(api, '$repo_id')
|
58 |
except Exception as e:
|
59 |
print(f'Error uploading file: {str(e)}')
|
60 |
"
|
61 |
}
|
62 |
|
63 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
64 |
|
65 |
# 同步函数
|
66 |
sync_data() {
|
@@ -69,7 +123,7 @@ sync_data() {
|
|
69 |
if [ -d /data ]; then
|
70 |
timestamp=$(date +%Y%m%d_%H%M%S)
|
71 |
backup_file="vaultwarden_backup_${timestamp}.tar.gz"
|
72 |
-
#
|
73 |
tar -czf "/tmp/${backup_file}" -C /data .
|
74 |
echo "Uploading backup to HuggingFace..."
|
75 |
upload_backup "/tmp/${backup_file}" "${backup_file}"
|
@@ -83,8 +137,11 @@ sync_data() {
|
|
83 |
done
|
84 |
}
|
85 |
|
86 |
-
#
|
|
|
|
|
|
|
87 |
sync_data &
|
88 |
|
89 |
-
# 启动
|
90 |
exec /start.sh
|
|
|
6 |
exit 0
|
7 |
fi
|
8 |
|
9 |
+
# 等待Cloudflare Tunnel就绪
|
10 |
check_tunnel() {
|
11 |
while ! curl -s http://localhost:7860 >/dev/null; do
|
12 |
echo "Waiting for Cloudflare Tunnel to be ready..."
|
|
|
17 |
# 激活虚拟环境
|
18 |
source /opt/venv/bin/activate
|
19 |
|
20 |
+
# 上传备份(含自动清理旧备份)
|
21 |
upload_backup() {
|
22 |
file_path="$1"
|
23 |
file_name="$2"
|
|
|
30 |
|
31 |
def manage_backups(api, repo_id, max_files=50):
|
32 |
files = api.list_repo_files(repo_id=repo_id, repo_type='dataset')
|
|
|
33 |
backup_files = [f for f in files if f.startswith('vaultwarden_backup_') and f.endswith('.tar.gz')]
|
34 |
backup_files.sort()
|
35 |
if len(backup_files) >= max_files:
|
36 |
+
delete_count = len(backup_files) - max_files + 1
|
37 |
+
files_to_delete = backup_files[:delete_count]
|
38 |
for file_to_delete in files_to_delete:
|
39 |
try:
|
40 |
+
api.delete_file(
|
41 |
+
path_in_repo=file_to_delete,
|
42 |
+
repo_id=repo_id,
|
43 |
+
repo_type='dataset'
|
44 |
+
)
|
45 |
print(f'Deleted old backup: {file_to_delete}')
|
46 |
except Exception as e:
|
47 |
print(f'Error deleting {file_to_delete}: {str(e)}')
|
48 |
|
49 |
api = HfApi(token='$token')
|
50 |
try:
|
51 |
+
# 上传新备份
|
52 |
api.upload_file(
|
53 |
path_or_fileobj='$file_path',
|
54 |
path_in_repo='$file_name',
|
|
|
57 |
)
|
58 |
print(f'Successfully uploaded $file_name')
|
59 |
|
60 |
+
# 清理旧备份
|
61 |
manage_backups(api, '$repo_id')
|
62 |
except Exception as e:
|
63 |
print(f'Error uploading file: {str(e)}')
|
64 |
"
|
65 |
}
|
66 |
|
67 |
+
# 下载最新备份(完整实现)
|
68 |
+
download_latest_backup() {
|
69 |
+
token="$HF_TOKEN"
|
70 |
+
repo_id="$DATASET_ID"
|
71 |
+
python3 -c "
|
72 |
+
from huggingface_hub import HfApi
|
73 |
+
import sys
|
74 |
+
import os
|
75 |
+
import tarfile
|
76 |
+
import tempfile
|
77 |
+
import shutil
|
78 |
+
|
79 |
+
api = HfApi(token='$token')
|
80 |
+
try:
|
81 |
+
files = api.list_repo_files(repo_id='$repo_id', repo_type='dataset')
|
82 |
+
backup_files = [f for f in files if f.startswith('vaultwarden_backup_') and f.endswith('.tar.gz')]
|
83 |
+
if not backup_files:
|
84 |
+
print('No backup files found')
|
85 |
+
sys.exit()
|
86 |
+
latest_backup = sorted(backup_files)[-1]
|
87 |
+
with tempfile.TemporaryDirectory() as temp_dir:
|
88 |
+
filepath = api.hf_hub_download(
|
89 |
+
repo_id='$repo_id',
|
90 |
+
filename=latest_backup,
|
91 |
+
repo_type='dataset',
|
92 |
+
local_dir=temp_dir
|
93 |
+
)
|
94 |
+
if filepath and os.path.exists(filepath):
|
95 |
+
# 确保数据目录存在
|
96 |
+
os.makedirs('/data', exist_ok=True)
|
97 |
+
# 清空旧数据(防止残留文件干扰)
|
98 |
+
if os.listdir('/data'):
|
99 |
+
for f in os.listdir('/data'):
|
100 |
+
item_path = os.path.join('/data', f)
|
101 |
+
if os.path.isfile(item_path):
|
102 |
+
os.remove(item_path)
|
103 |
+
elif os.path.isdir(item_path):
|
104 |
+
shutil.rmtree(item_path)
|
105 |
+
# 解压备份文件
|
106 |
+
with tarfile.open(filepath, 'r:gz') as tar:
|
107 |
+
tar.extractall('/data')
|
108 |
+
print(f'Successfully restored backup from {latest_backup}')
|
109 |
+
except Exception as e:
|
110 |
+
print(f'Error downloading backup: {str(e)}')
|
111 |
+
sys.exit(1)
|
112 |
+
"
|
113 |
+
}
|
114 |
+
|
115 |
+
# 首次启动时下载最新备份
|
116 |
+
echo "Downloading latest backup from HuggingFace..."
|
117 |
+
download_latest_backup
|
118 |
|
119 |
# 同步函数
|
120 |
sync_data() {
|
|
|
123 |
if [ -d /data ]; then
|
124 |
timestamp=$(date +%Y%m%d_%H%M%S)
|
125 |
backup_file="vaultwarden_backup_${timestamp}.tar.gz"
|
126 |
+
# 压缩数据目录(排除临时文件)
|
127 |
tar -czf "/tmp/${backup_file}" -C /data .
|
128 |
echo "Uploading backup to HuggingFace..."
|
129 |
upload_backup "/tmp/${backup_file}" "${backup_file}"
|
|
|
137 |
done
|
138 |
}
|
139 |
|
140 |
+
# 启动Cloudflare Tunnel检查
|
141 |
+
check_tunnel &
|
142 |
+
|
143 |
+
# 启动备份同步进程
|
144 |
sync_data &
|
145 |
|
146 |
+
# 启动Vaultwarden主程序
|
147 |
exec /start.sh
|