Update sync_data.sh
Browse files- sync_data.sh +72 -2
sync_data.sh
CHANGED
@@ -41,6 +41,74 @@ else:
|
|
41 |
"
|
42 |
}
|
43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
44 |
sync_data() {
|
45 |
echo "启动后首次备份将在${SYNC_INTERVAL:-7200}秒后执行"
|
46 |
sleep ${SYNC_INTERVAL:-7200}
|
@@ -79,7 +147,9 @@ for old_backup in backup_files[:-50]:
|
|
79 |
|
80 |
|
81 |
(
|
82 |
-
restore_latest
|
83 |
-
|
|
|
84 |
exec uvicorn app.main:app --host 0.0.0.0 --port 7860
|
85 |
) 2>&1 | tee -a /app/data/backup.log
|
|
|
|
41 |
"
|
42 |
}
|
43 |
|
44 |
+
backup_upload_download_test() {
|
45 |
+
echo "正在执行备份上传-下载权限完整性测试..."
|
46 |
+
|
47 |
+
TEST_FILE_CONTENT="备份测试内容 $(date)"
|
48 |
+
TEST_FILE_NAME="backup_test_$(date +%Y%m%d_%H%M%S).txt"
|
49 |
+
LOCAL_TEST_PATH="/tmp/${TEST_FILE_NAME}"
|
50 |
+
|
51 |
+
echo "${TEST_FILE_CONTENT}" > "${LOCAL_TEST_PATH}"
|
52 |
+
|
53 |
+
python3 -c "
|
54 |
+
from huggingface_hub import HfApi
|
55 |
+
import os, sys
|
56 |
+
|
57 |
+
api = HfApi()
|
58 |
+
repo_id = '${DATASET_ID}'
|
59 |
+
test_file_name = '${TEST_FILE_NAME}'
|
60 |
+
local_test_path = '${LOCAL_TEST_PATH}'
|
61 |
+
|
62 |
+
# 上传测试文件
|
63 |
+
try:
|
64 |
+
api.upload_file(
|
65 |
+
path_or_fileobj=local_test_path,
|
66 |
+
path_in_repo=test_file_name,
|
67 |
+
repo_id=repo_id,
|
68 |
+
repo_type='dataset'
|
69 |
+
)
|
70 |
+
print('✅ 测试文件上传成功')
|
71 |
+
except Exception as e:
|
72 |
+
print(f'❌ 测试文件上传失败: {e}')
|
73 |
+
sys.exit(1)
|
74 |
+
|
75 |
+
# 下载测试文件
|
76 |
+
try:
|
77 |
+
downloaded_path = api.hf_hub_download(
|
78 |
+
repo_id=repo_id,
|
79 |
+
filename=test_file_name,
|
80 |
+
repo_type='dataset',
|
81 |
+
local_dir='/tmp'
|
82 |
+
)
|
83 |
+
with open(downloaded_path, 'r') as f:
|
84 |
+
content = f.read().strip()
|
85 |
+
if content == '${TEST_FILE_CONTENT}':
|
86 |
+
print('✅ 测试文件下载成功且内容一致')
|
87 |
+
else:
|
88 |
+
print('❌ 测试文件内容不一致')
|
89 |
+
sys.exit(1)
|
90 |
+
except Exception as e:
|
91 |
+
print(f'❌ 测试文件下载失败: {e}')
|
92 |
+
sys.exit(1)
|
93 |
+
|
94 |
+
# 测试完成后删除测试文件
|
95 |
+
try:
|
96 |
+
api.delete_file(
|
97 |
+
path_in_repo=test_file_name,
|
98 |
+
repo_id=repo_id,
|
99 |
+
repo_type='dataset'
|
100 |
+
)
|
101 |
+
print('✅ 测试文件已成功删除')
|
102 |
+
except Exception as e:
|
103 |
+
print(f'⚠️ 测试文件删除失败: {e}')
|
104 |
+
|
105 |
+
# 清理本地临时文件
|
106 |
+
os.remove(local_test_path)
|
107 |
+
os.remove(downloaded_path)
|
108 |
+
"
|
109 |
+
}
|
110 |
+
|
111 |
+
|
112 |
sync_data() {
|
113 |
echo "启动后首次备份将在${SYNC_INTERVAL:-7200}秒后执行"
|
114 |
sleep ${SYNC_INTERVAL:-7200}
|
|
|
147 |
|
148 |
|
149 |
(
|
150 |
+
restore_latest # 恢复备份
|
151 |
+
backup_upload_download_test # 新增:首次启动备份上传下载测试
|
152 |
+
sync_data & # 后台定期备份
|
153 |
exec uvicorn app.main:app --host 0.0.0.0 --port 7860
|
154 |
) 2>&1 | tee -a /app/data/backup.log
|
155 |
+
|