Update sync_data.sh
Browse files- sync_data.sh +8 -2
sync_data.sh
CHANGED
@@ -56,11 +56,16 @@ sync_data() {
|
|
56 |
python3 -c "
|
57 |
from huggingface_hub import HfApi
|
58 |
api = HfApi()
|
59 |
-
api.upload_file(
|
|
|
|
|
|
|
|
|
|
|
60 |
print('备份上传成功')
|
61 |
backup_files = sorted([f for f in api.list_repo_files('${DATASET_ID}', repo_type='dataset') if f.startswith('backup_')])
|
62 |
for old_backup in backup_files[:-50]:
|
63 |
-
api.delete_file(old_backup, '${DATASET_ID}', repo_type='dataset')
|
64 |
print(f'删除旧备份: {old_backup}')
|
65 |
"
|
66 |
rm -f "/tmp/${backup_file}"
|
@@ -72,6 +77,7 @@ for old_backup in backup_files[:-50]:
|
|
72 |
done
|
73 |
}
|
74 |
|
|
|
75 |
(
|
76 |
restore_latest
|
77 |
sync_data &
|
|
|
56 |
python3 -c "
|
57 |
from huggingface_hub import HfApi
|
58 |
api = HfApi()
|
59 |
+
api.upload_file(
|
60 |
+
path_or_fileobj='/tmp/${backup_file}',
|
61 |
+
path_in_repo='${backup_file}',
|
62 |
+
repo_id='${DATASET_ID}',
|
63 |
+
repo_type='dataset'
|
64 |
+
)
|
65 |
print('备份上传成功')
|
66 |
backup_files = sorted([f for f in api.list_repo_files('${DATASET_ID}', repo_type='dataset') if f.startswith('backup_')])
|
67 |
for old_backup in backup_files[:-50]:
|
68 |
+
api.delete_file(path_in_repo=old_backup, repo_id='${DATASET_ID}', repo_type='dataset')
|
69 |
print(f'删除旧备份: {old_backup}')
|
70 |
"
|
71 |
rm -f "/tmp/${backup_file}"
|
|
|
77 |
done
|
78 |
}
|
79 |
|
80 |
+
|
81 |
(
|
82 |
restore_latest
|
83 |
sync_data &
|