Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -1,491 +1,241 @@
|
|
1 |
-
# app.py
|
2 |
import os
|
3 |
import shutil
|
4 |
import zipfile
|
5 |
import threading
|
6 |
import time
|
7 |
-
import
|
8 |
-
from flask import Flask, request, jsonify, render_template_string
|
9 |
import gdown
|
10 |
-
from huggingface_hub import HfApi, login
|
11 |
-
from huggingface_hub.utils import HfHubHTTPError
|
12 |
|
13 |
-
#
|
14 |
-
# Ensure Hugging Face cache and other temp files write to the writable /tmp directory
|
15 |
os.environ["HF_HOME"] = "/tmp/hf_home"
|
16 |
-
os.environ["GDOWN_CACHE_DIR"] = "/tmp/gdown_cache"
|
17 |
|
18 |
-
# Environment variables
|
19 |
-
FOLDER_URL
|
20 |
-
REPO_ID
|
21 |
-
TOKEN
|
|
|
22 |
|
23 |
-
# Directories
|
24 |
-
DOWNLOAD_DIR
|
25 |
-
EXTRACT_DIR
|
26 |
|
27 |
-
#
|
28 |
-
|
29 |
-
|
30 |
-
<html lang="en" data-theme="dark">
|
31 |
-
<head>
|
32 |
-
<meta charset="UTF-8">
|
33 |
-
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
34 |
-
<title>HF Backup & Manager</title>
|
35 |
-
<!-- Pico.css for a clean, modern look -->
|
36 |
-
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@1/css/pico.min.css">
|
37 |
-
<!-- Embedded Custom Styles -->
|
38 |
-
<style>
|
39 |
-
:root {
|
40 |
-
--pico-card-background-color: #1e2025;
|
41 |
-
--pico-card-border-color: #33363d;
|
42 |
-
}
|
43 |
-
body {
|
44 |
-
padding: 1rem;
|
45 |
-
}
|
46 |
-
main.container {
|
47 |
-
max-width: 1000px;
|
48 |
-
padding-top: 1rem;
|
49 |
-
}
|
50 |
-
header {
|
51 |
-
text-align: center;
|
52 |
-
margin-bottom: 2rem;
|
53 |
-
}
|
54 |
-
article {
|
55 |
-
padding: 1.5rem;
|
56 |
-
}
|
57 |
-
.grid {
|
58 |
-
grid-template-columns: 1fr;
|
59 |
-
gap: 1.5rem;
|
60 |
-
}
|
61 |
-
@media (min-width: 992px) {
|
62 |
-
.grid {
|
63 |
-
grid-template-columns: 1fr 1fr;
|
64 |
-
}
|
65 |
-
}
|
66 |
-
.log-box {
|
67 |
-
background-color: #111317;
|
68 |
-
border: 1px solid var(--pico-card-border-color);
|
69 |
-
border-radius: var(--pico-border-radius);
|
70 |
-
padding: 1rem;
|
71 |
-
height: 200px;
|
72 |
-
overflow-y: auto;
|
73 |
-
font-family: monospace;
|
74 |
-
font-size: 0.875em;
|
75 |
-
white-space: pre-wrap;
|
76 |
-
word-break: break-all;
|
77 |
-
}
|
78 |
-
#status-text.idle { color: var(--pico-color-green-400); }
|
79 |
-
#status-text.running { color: var(--pico-color-amber-400); }
|
80 |
-
#status-text.error { color: var(--pico-color-red-400); }
|
81 |
-
button {
|
82 |
-
display: flex;
|
83 |
-
align-items: center;
|
84 |
-
justify-content: center;
|
85 |
-
gap: 0.75rem;
|
86 |
-
}
|
87 |
-
.spinner {
|
88 |
-
border: 3px solid rgba(255, 255, 255, 0.2);
|
89 |
-
border-top: 3px solid var(--pico-primary);
|
90 |
-
border-radius: 50%;
|
91 |
-
width: 16px;
|
92 |
-
height: 16px;
|
93 |
-
animation: spin 1s linear infinite;
|
94 |
-
}
|
95 |
-
@keyframes spin {
|
96 |
-
0% { transform: rotate(0deg); }
|
97 |
-
100% { transform: rotate(360deg); }
|
98 |
-
}
|
99 |
-
.file-manager-container {
|
100 |
-
max-height: 400px;
|
101 |
-
overflow-y: auto;
|
102 |
-
margin-top: 1rem;
|
103 |
-
}
|
104 |
-
.file-manager-container table button {
|
105 |
-
margin: 0;
|
106 |
-
padding: 0.25rem 0.5rem;
|
107 |
-
background-color: var(--pico-color-red-600);
|
108 |
-
border-color: var(--pico-color-red-600);
|
109 |
-
}
|
110 |
-
small {
|
111 |
-
display: block;
|
112 |
-
margin-top: -0.5rem;
|
113 |
-
margin-bottom: 1rem;
|
114 |
-
color: var(--pico-secondary-text);
|
115 |
-
}
|
116 |
-
</style>
|
117 |
-
</head>
|
118 |
-
<body>
|
119 |
-
<main class="container">
|
120 |
-
<header>
|
121 |
-
<hgroup>
|
122 |
-
<h1>Hugging Face Backup & Manager</h1>
|
123 |
-
<p>Automate server backups and manage your dataset on the Hub.</p>
|
124 |
-
</hgroup>
|
125 |
-
</header>
|
126 |
-
|
127 |
-
<div class="grid">
|
128 |
-
<article>
|
129 |
-
<hgroup>
|
130 |
-
<h2>Control Panel</h2>
|
131 |
-
<h3>Manage your backup tasks and schedule.</h3>
|
132 |
-
</hgroup>
|
133 |
-
<button id="run-now-btn" onclick="runNow()">
|
134 |
-
<span id="run-now-spinner" class="spinner" style="display: none;"></span>
|
135 |
-
Run Backup Now
|
136 |
-
</button>
|
137 |
-
<small>Manually trigger a full backup cycle.</small>
|
138 |
-
<form id="schedule-form" onsubmit="setSchedule(event)">
|
139 |
-
<label for="interval">Automatic Backup Interval (minutes)</label>
|
140 |
-
<input type="number" id="interval" name="interval" placeholder="0" min="0">
|
141 |
-
<small>Set to 0 to disable automatic backups.</small>
|
142 |
-
<button type="submit">Set Schedule</button>
|
143 |
-
</form>
|
144 |
-
</article>
|
145 |
-
|
146 |
-
<article>
|
147 |
-
<hgroup>
|
148 |
-
<h2>Live Status</h2>
|
149 |
-
<h3 id="status-text">Status: Fetching...</h3>
|
150 |
-
</hgroup>
|
151 |
-
<p><strong>Last Successful Backup:</strong> <span id="last-backup-time">Never</span></p>
|
152 |
-
<p><strong>Current Schedule:</strong> Every <span id="current-schedule">...</span> minutes</p>
|
153 |
-
<strong>Logs:</strong>
|
154 |
-
<pre id="logs" class="log-box"></pre>
|
155 |
-
</article>
|
156 |
-
</div>
|
157 |
|
158 |
-
<article>
|
159 |
-
<hgroup>
|
160 |
-
<h2>Dataset File Manager</h2>
|
161 |
-
<h3>Manage files in your repository: <a href="https://huggingface.co/datasets/{{ repo_id }}" target="_blank">{{ repo_id }}</a></h3>
|
162 |
-
</hgroup>
|
163 |
-
<button id="refresh-files-btn" onclick="fetchRepoFiles()" aria-busy="false">Refresh File List</button>
|
164 |
-
<div id="file-manager" class="file-manager-container">
|
165 |
-
<p>Loading files...</p>
|
166 |
-
</div>
|
167 |
-
</article>
|
168 |
-
|
169 |
-
</main>
|
170 |
-
|
171 |
-
<!-- Embedded JavaScript -->
|
172 |
-
<script>
|
173 |
-
const runNowBtn = document.getElementById('run-now-btn');
|
174 |
-
const runNowSpinner = document.getElementById('run-now-spinner');
|
175 |
-
const statusText = document.getElementById('status-text');
|
176 |
-
const lastBackupTime = document.getElementById('last-backup-time');
|
177 |
-
const currentSchedule = document.getElementById('current-schedule');
|
178 |
-
const scheduleInput = document.getElementById('interval');
|
179 |
-
const logsBox = document.getElementById('logs');
|
180 |
-
const fileManagerDiv = document.getElementById('file-manager');
|
181 |
-
const refreshFilesBtn = document.getElementById('refresh-files-btn');
|
182 |
-
|
183 |
-
async function fetchAPI(url, options = {}) {
|
184 |
-
try {
|
185 |
-
const response = await fetch(url, options);
|
186 |
-
if (!response.ok) {
|
187 |
-
const errorData = await response.json();
|
188 |
-
throw new Error(errorData.error || `HTTP error! status: ${response.status}`);
|
189 |
-
}
|
190 |
-
return await response.json();
|
191 |
-
} catch (error) {
|
192 |
-
console.error(`API Error on ${url}:`, error);
|
193 |
-
throw error;
|
194 |
-
}
|
195 |
-
}
|
196 |
-
|
197 |
-
async function fetchStatus() {
|
198 |
-
try {
|
199 |
-
const data = await fetchAPI('/api/status');
|
200 |
-
updateStatusUI(data);
|
201 |
-
} catch (error) {
|
202 |
-
statusText.textContent = "Status: Connection Error";
|
203 |
-
statusText.className = "error";
|
204 |
-
}
|
205 |
-
}
|
206 |
-
|
207 |
-
async function runNow() {
|
208 |
-
if (runNowBtn.disabled) return;
|
209 |
-
try {
|
210 |
-
await fetchAPI('/api/start-backup', { method: 'POST' });
|
211 |
-
} catch (error) {
|
212 |
-
alert(`Failed to start backup: ${error.message}`);
|
213 |
-
}
|
214 |
-
}
|
215 |
-
|
216 |
-
async function setSchedule(event) {
|
217 |
-
event.preventDefault();
|
218 |
-
const interval = parseInt(scheduleInput.value, 10);
|
219 |
-
if (isNaN(interval) || interval < 0) {
|
220 |
-
alert("Please enter a valid non-negative number for the interval.");
|
221 |
-
return;
|
222 |
-
}
|
223 |
-
try {
|
224 |
-
await fetchAPI('/api/set-schedule', {
|
225 |
-
method: 'POST',
|
226 |
-
headers: { 'Content-Type': 'application/json' },
|
227 |
-
body: JSON.stringify({ interval })
|
228 |
-
});
|
229 |
-
} catch (error) {
|
230 |
-
alert(`Error setting schedule: ${error.message}`);
|
231 |
-
}
|
232 |
-
}
|
233 |
-
|
234 |
-
async function fetchRepoFiles() {
|
235 |
-
refreshFilesBtn.setAttribute('aria-busy', 'true');
|
236 |
-
try {
|
237 |
-
const data = await fetchAPI('/api/repo-files');
|
238 |
-
renderFileManager(data.files);
|
239 |
-
} catch (error) {
|
240 |
-
fileManagerDiv.innerHTML = `<p style="color: var(--pico-color-red-500);">Error loading files: ${error.message}</p>`;
|
241 |
-
} finally {
|
242 |
-
refreshFilesBtn.setAttribute('aria-busy', 'false');
|
243 |
-
}
|
244 |
-
}
|
245 |
-
|
246 |
-
async function deleteFile(path) {
|
247 |
-
if (!confirm(`Are you sure you want to permanently delete "${path}"? This cannot be undone.`)) return;
|
248 |
-
try {
|
249 |
-
await fetchAPI('/api/delete-file', {
|
250 |
-
method: 'POST',
|
251 |
-
headers: { 'Content-Type': 'application/json' },
|
252 |
-
body: JSON.stringify({ path })
|
253 |
-
});
|
254 |
-
await fetchRepoFiles();
|
255 |
-
} catch (error) {
|
256 |
-
alert(`Error deleting file: ${error.message}`);
|
257 |
-
}
|
258 |
-
}
|
259 |
-
|
260 |
-
function updateStatusUI(data) {
|
261 |
-
statusText.textContent = `Status: ${data.status}`;
|
262 |
-
statusText.className = data.status.toLowerCase();
|
263 |
-
|
264 |
-
const isRunning = data.status === 'Running';
|
265 |
-
runNowBtn.disabled = isRunning;
|
266 |
-
runNowSpinner.style.display = isRunning ? 'inline-block' : 'none';
|
267 |
-
|
268 |
-
lastBackupTime.textContent = data.last_backup_time;
|
269 |
-
const interval = data.schedule_interval;
|
270 |
-
currentSchedule.textContent = interval > 0 ? `${interval}` : '0 (disabled)';
|
271 |
-
if (document.activeElement !== scheduleInput) {
|
272 |
-
scheduleInput.value = interval > 0 ? interval : '';
|
273 |
-
}
|
274 |
-
|
275 |
-
const newLogs = data.logs.join('\\n');
|
276 |
-
if (logsBox.textContent !== newLogs) {
|
277 |
-
logsBox.textContent = newLogs;
|
278 |
-
logsBox.scrollTop = logsBox.scrollHeight;
|
279 |
-
}
|
280 |
-
}
|
281 |
-
|
282 |
-
function renderFileManager(files) {
|
283 |
-
if (!files || files.length === 0) {
|
284 |
-
fileManagerDiv.innerHTML = "<p>No files found in the repository.</p>";
|
285 |
-
return;
|
286 |
-
}
|
287 |
-
let html = '<table><thead><tr><th>File Path</th><th style="text-align: right;">Action</th></tr></thead><tbody>';
|
288 |
-
files.forEach(file => {
|
289 |
-
html += `
|
290 |
-
<tr>
|
291 |
-
<td>${file}</td>
|
292 |
-
<td style="text-align: right;"><button class="outline secondary" onclick="deleteFile('${file}')">Delete</button></td>
|
293 |
-
</tr>
|
294 |
-
`;
|
295 |
-
});
|
296 |
-
html += '</tbody></table>';
|
297 |
-
fileManagerDiv.innerHTML = html;
|
298 |
-
}
|
299 |
-
|
300 |
-
document.addEventListener('DOMContentLoaded', () => {
|
301 |
-
fetchStatus();
|
302 |
-
fetchRepoFiles();
|
303 |
-
setInterval(fetchStatus, 3000);
|
304 |
-
});
|
305 |
-
</script>
|
306 |
-
</body>
|
307 |
-
</html>
|
308 |
-
"""
|
309 |
-
|
310 |
-
# --- FLASK APP & STATE MANAGEMENT ---
|
311 |
app = Flask(__name__)
|
312 |
-
logging.basicConfig(level=logging.INFO)
|
313 |
-
|
314 |
-
app_state = {
|
315 |
-
"status": "Idle", # Idle, Running, Error
|
316 |
-
"logs": [],
|
317 |
-
"last_backup_time": "Never",
|
318 |
-
"schedule_interval": 0, # in minutes
|
319 |
-
"scheduler_thread": None,
|
320 |
-
"lock": threading.Lock(),
|
321 |
-
}
|
322 |
-
|
323 |
-
# --- HUGGING FACE HELPER CLASS ---
|
324 |
-
class HFManager:
|
325 |
-
def __init__(self, token, repo_id, repo_type="dataset"):
|
326 |
-
if not token:
|
327 |
-
raise ValueError("Hugging Face token (HF_TOKEN) is not set.")
|
328 |
-
self.token = token
|
329 |
-
self.repo_id = repo_id
|
330 |
-
self.repo_type = repo_type
|
331 |
-
self.api = HfApi()
|
332 |
-
login(token=self.token)
|
333 |
-
|
334 |
-
def ensure_repo_exists(self):
|
335 |
-
self.api.create_repo(repo_id=self.repo_id, repo_type=self.repo_type, exist_ok=True, token=self.token)
|
336 |
-
|
337 |
-
def list_files(self):
|
338 |
-
try:
|
339 |
-
return sorted(self.api.list_repo_files(repo_id=self.repo_id, repo_type=self.repo_type, token=self.token))
|
340 |
-
except HfHubHTTPError as e:
|
341 |
-
if e.response.status_code == 404: return []
|
342 |
-
raise e
|
343 |
-
|
344 |
-
def delete_file(self, path_in_repo):
|
345 |
-
self.api.delete_file(path_in_repo, repo_id=self.repo_id, repo_type=self.repo_type, token=self.token, commit_message=f"Delete file: {path_in_repo}")
|
346 |
-
|
347 |
-
def upload(self, folder_path, path_in_repo, commit_message):
|
348 |
-
self.api.upload_folder(repo_id=self.repo_id, folder_path=folder_path, repo_type=self.repo_type, token=self.token, path_in_repo=path_in_repo, commit_message=commit_message)
|
349 |
-
|
350 |
-
# --- BACKUP LOGIC ---
|
351 |
-
def run_backup_job():
|
352 |
-
with app_state['lock']:
|
353 |
-
if app_state["status"] == "Running":
|
354 |
-
app_state['logs'].append("Backup is already in progress. Skipping scheduled run.")
|
355 |
-
return
|
356 |
-
app_state["status"] = "Running"
|
357 |
-
app_state["logs"] = ["Starting backup process..."]
|
358 |
-
|
359 |
-
log_entry = lambda msg: app_state['logs'].append(f"[{time.strftime('%H:%M:%S')}] {msg}")
|
360 |
|
|
|
|
|
|
|
361 |
try:
|
362 |
-
|
363 |
-
raise ValueError("FOLDER_URL is not set. Please set it in your Space secrets.")
|
364 |
-
if not TOKEN:
|
365 |
-
raise ValueError("HF_TOKEN is not set. Please set it in your Space secrets.")
|
366 |
-
|
367 |
-
log_entry("Cleaning up temporary directories...")
|
368 |
shutil.rmtree(DOWNLOAD_DIR, ignore_errors=True)
|
369 |
shutil.rmtree(EXTRACT_DIR, ignore_errors=True)
|
|
|
370 |
os.makedirs(EXTRACT_DIR, exist_ok=True)
|
371 |
|
372 |
-
|
373 |
gdown.download_folder(url=FOLDER_URL, output=DOWNLOAD_DIR, use_cookies=False, quiet=True)
|
374 |
-
|
375 |
|
376 |
-
extracted_files = False
|
377 |
for root, _, files in os.walk(DOWNLOAD_DIR):
|
378 |
-
for
|
379 |
-
if
|
380 |
-
|
381 |
-
with zipfile.ZipFile(
|
382 |
z.extractall(EXTRACT_DIR)
|
383 |
-
|
384 |
-
|
385 |
-
|
386 |
-
|
387 |
-
|
388 |
-
|
389 |
-
|
390 |
-
|
391 |
-
|
392 |
-
|
393 |
-
|
394 |
-
|
395 |
-
|
396 |
-
|
397 |
-
|
398 |
-
|
399 |
-
|
400 |
-
|
401 |
-
|
402 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
403 |
else:
|
404 |
-
|
405 |
|
406 |
-
|
407 |
-
|
408 |
-
log_entry(f"Backup complete!")
|
409 |
-
app_state["status"] = "Idle"
|
410 |
|
411 |
except Exception as e:
|
412 |
-
|
413 |
-
|
414 |
-
|
415 |
-
app_state["logs"].append(f"ERROR: {error_message}")
|
416 |
-
app_state["status"] = "Error"
|
417 |
|
418 |
-
|
419 |
-
def scheduler_loop():
|
420 |
while True:
|
421 |
-
|
422 |
-
|
423 |
-
|
424 |
-
next_run_time = time.time() + interval_minutes * 60
|
425 |
-
run_backup_job()
|
426 |
-
sleep_duration = next_run_time - time.time()
|
427 |
-
if sleep_duration > 0:
|
428 |
-
time.sleep(sleep_duration)
|
429 |
else:
|
430 |
-
time.sleep(
|
431 |
|
432 |
-
|
433 |
-
@app.route("/")
|
434 |
-
def index():
|
435 |
-
return render_template_string(HTML_TEMPLATE, repo_id=REPO_ID)
|
436 |
-
|
437 |
-
@app.route("/api/status")
|
438 |
-
def status():
|
439 |
-
# THE ONLY CHANGE IS IN THIS FUNCTION
|
440 |
-
with app_state['lock']:
|
441 |
-
# Create a new dictionary with only the JSON-serializable items.
|
442 |
-
# This prevents the "TypeError: Object of type lock is not JSON serializable"
|
443 |
-
serializable_state = {
|
444 |
-
"status": app_state["status"],
|
445 |
-
"logs": app_state["logs"],
|
446 |
-
"last_backup_time": app_state["last_backup_time"],
|
447 |
-
"schedule_interval": app_state["schedule_interval"],
|
448 |
-
}
|
449 |
-
return jsonify(serializable_state)
|
450 |
-
|
451 |
-
@app.route("/api/start-backup", methods=["POST"])
|
452 |
-
def start_backup():
|
453 |
-
threading.Thread(target=run_backup_job).start()
|
454 |
-
return jsonify({"message": "Backup process initiated."})
|
455 |
-
|
456 |
-
@app.route("/api/set-schedule", methods=["POST"])
|
457 |
-
def set_schedule():
|
458 |
-
try:
|
459 |
-
interval = int(request.json.get("interval", 0))
|
460 |
-
if interval < 0: raise ValueError("Interval cannot be negative.")
|
461 |
-
with app_state['lock']:
|
462 |
-
app_state['schedule_interval'] = interval
|
463 |
-
return jsonify({"message": f"Schedule updated to {interval} minutes."})
|
464 |
-
except (ValueError, TypeError):
|
465 |
-
return jsonify({"error": "Invalid interval. Please provide a non-negative integer."}), 400
|
466 |
|
467 |
-
|
468 |
-
|
469 |
-
|
470 |
-
|
471 |
-
|
472 |
-
|
473 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
474 |
|
475 |
-
@app.route("/
|
476 |
-
def
|
477 |
-
|
478 |
-
|
479 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
480 |
try:
|
481 |
-
|
482 |
-
|
483 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
484 |
except Exception as e:
|
485 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
486 |
|
487 |
-
# --- MAIN EXECUTION ---
|
488 |
if __name__ == "__main__":
|
489 |
-
|
490 |
-
app_state["scheduler_thread"].start()
|
491 |
-
app.run(host="0.0.0.0", port=7860)
|
|
|
|
|
1 |
import os
|
2 |
import shutil
|
3 |
import zipfile
|
4 |
import threading
|
5 |
import time
|
6 |
+
from flask import Flask, request, render_template_string, redirect, url_for
|
|
|
7 |
import gdown
|
8 |
+
from huggingface_hub import HfApi, login, upload_folder, list_repo_files
|
|
|
9 |
|
10 |
+
# Ensure Hugging Face cache writes to tmp
|
|
|
11 |
os.environ["HF_HOME"] = "/tmp/hf_home"
|
|
|
12 |
|
13 |
+
# Environment variables
|
14 |
+
FOLDER_URL = os.getenv("FOLDER_URL")
|
15 |
+
REPO_ID = os.getenv("REPO_ID")
|
16 |
+
TOKEN = os.getenv("HF_TOKEN")
|
17 |
+
GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") # for future Drive API use
|
18 |
|
19 |
+
# Directories
|
20 |
+
DOWNLOAD_DIR = "/tmp/backups"
|
21 |
+
EXTRACT_DIR = "/tmp/extracted_backups"
|
22 |
|
23 |
+
# Application state
|
24 |
+
last_backup_time = "Never"
|
25 |
+
schedule_interval = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
27 |
app = Flask(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
28 |
|
29 |
+
def run_backup():
|
30 |
+
global last_backup_time
|
31 |
+
log = []
|
32 |
try:
|
33 |
+
log.append("Starting backup process")
|
|
|
|
|
|
|
|
|
|
|
34 |
shutil.rmtree(DOWNLOAD_DIR, ignore_errors=True)
|
35 |
shutil.rmtree(EXTRACT_DIR, ignore_errors=True)
|
36 |
+
os.makedirs(DOWNLOAD_DIR, exist_ok=True)
|
37 |
os.makedirs(EXTRACT_DIR, exist_ok=True)
|
38 |
|
39 |
+
log.append(f"Downloading from {FOLDER_URL}")
|
40 |
gdown.download_folder(url=FOLDER_URL, output=DOWNLOAD_DIR, use_cookies=False, quiet=True)
|
41 |
+
log.append("Download complete")
|
42 |
|
|
|
43 |
for root, _, files in os.walk(DOWNLOAD_DIR):
|
44 |
+
for name in files:
|
45 |
+
if name.endswith(".zip"):
|
46 |
+
path = os.path.join(root, name)
|
47 |
+
with zipfile.ZipFile(path) as z:
|
48 |
z.extractall(EXTRACT_DIR)
|
49 |
+
log.append(f"Extracted {name}")
|
50 |
+
|
51 |
+
bad = os.path.join(EXTRACT_DIR, "world_nither")
|
52 |
+
good = os.path.join(EXTRACT_DIR, "world_nether")
|
53 |
+
if os.path.exists(bad) and not os.path.exists(good):
|
54 |
+
os.rename(bad, good)
|
55 |
+
log.append("Fixed world_nether typo")
|
56 |
+
|
57 |
+
login(token=TOKEN)
|
58 |
+
api = HfApi()
|
59 |
+
log.append("Authenticated with Hugging Face")
|
60 |
+
|
61 |
+
api.create_repo(repo_id=REPO_ID, repo_type="dataset", private=False, exist_ok=True, token=TOKEN)
|
62 |
+
log.append(f"Repository ready {REPO_ID}")
|
63 |
+
|
64 |
+
sections = {
|
65 |
+
"world": os.path.join(EXTRACT_DIR, "world"),
|
66 |
+
"world_nether": os.path.join(EXTRACT_DIR, "world_nether"),
|
67 |
+
"world_the_end": os.path.join(EXTRACT_DIR, "world_the_end"),
|
68 |
+
"plugins": os.path.join(EXTRACT_DIR, "plugins")
|
69 |
+
}
|
70 |
+
for key, path in sections.items():
|
71 |
+
if os.path.isdir(path):
|
72 |
+
log.append(f"Uploading section {key}")
|
73 |
+
upload_folder(
|
74 |
+
repo_id=REPO_ID,
|
75 |
+
folder_path=path,
|
76 |
+
repo_type="dataset",
|
77 |
+
token=TOKEN,
|
78 |
+
path_in_repo=key,
|
79 |
+
commit_message=f"Add {key}"
|
80 |
+
)
|
81 |
+
log.append(f"Uploaded {key}")
|
82 |
else:
|
83 |
+
log.append(f"Skipped missing section {key}")
|
84 |
|
85 |
+
last_backup_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
86 |
+
log.append(f"Backup finished at {last_backup_time}")
|
|
|
|
|
87 |
|
88 |
except Exception as e:
|
89 |
+
log.append(f"Error {e}")
|
90 |
+
|
91 |
+
return "\n".join(log)
|
|
|
|
|
92 |
|
93 |
+
def schedule_loop():
|
|
|
94 |
while True:
|
95 |
+
if schedule_interval > 0:
|
96 |
+
run_backup()
|
97 |
+
time.sleep(schedule_interval * 60)
|
|
|
|
|
|
|
|
|
|
|
98 |
else:
|
99 |
+
time.sleep(5)
|
100 |
|
101 |
+
threading.Thread(target=schedule_loop, daemon=True).start()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
102 |
|
103 |
+
HTML = """
|
104 |
+
<!doctype html>
|
105 |
+
<html>
|
106 |
+
<head>
|
107 |
+
<meta charset="utf-8">
|
108 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
109 |
+
<title>Backup & Dataset Manager</title>
|
110 |
+
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
|
111 |
+
<style>
|
112 |
+
body { padding: 2rem; }
|
113 |
+
.log-area { height: 300px; overflow-y: scroll; background: #f9f9f9; padding: 1rem; border-radius: .5rem; font-family: monospace; white-space: pre-wrap; }
|
114 |
+
.nav-link { cursor: pointer; }
|
115 |
+
</style>
|
116 |
+
</head>
|
117 |
+
<body>
|
118 |
+
<h1 class="mb-4">Backup & Dataset Manager</h1>
|
119 |
+
<ul class="nav nav-tabs mb-3" id="mainTabs">
|
120 |
+
<li class="nav-item">
|
121 |
+
<a class="nav-link active" data-bs-target="#backupTab">Backup Control</a>
|
122 |
+
</li>
|
123 |
+
<li class="nav-item">
|
124 |
+
<a class="nav-link" data-bs-target="#datasetTab">Manage Dataset</a>
|
125 |
+
</li>
|
126 |
+
</ul>
|
127 |
+
|
128 |
+
<div class="tab-content">
|
129 |
+
<div id="backupTab" class="tab-pane fade show active">
|
130 |
+
<form method="post" action="/">
|
131 |
+
<div class="mb-3">
|
132 |
+
<label class="form-label">Interval Minutes</label>
|
133 |
+
<input type="number" class="form-control" name="interval" value="{{ interval }}" min="1">
|
134 |
+
</div>
|
135 |
+
<button class="btn btn-primary">Set Schedule</button>
|
136 |
+
<button class="btn btn-secondary ms-2" name="manual_run" value="1">Run Now</button>
|
137 |
+
</form>
|
138 |
+
<div class="mt-4">
|
139 |
+
<strong>Last Backup:</strong> {{ last_run }}<br>
|
140 |
+
<div class="log-area mt-2">{{ status }}</div>
|
141 |
+
</div>
|
142 |
+
</div>
|
143 |
+
|
144 |
+
<div id="datasetTab" class="tab-pane fade">
|
145 |
+
<form method="post" action="/manage">
|
146 |
+
<div class="mb-3">
|
147 |
+
<label class="form-label">Drive File or Folder URL</label>
|
148 |
+
<input type="url" class="form-control" name="drive_url" placeholder="Enter Google Drive link">
|
149 |
+
</div>
|
150 |
+
<div class="mb-3">
|
151 |
+
<label class="form-label">Destination Path in Dataset</label>
|
152 |
+
<input type="text" class="form-control" name="dest_path" placeholder="eg world_extra">
|
153 |
+
</div>
|
154 |
+
<button class="btn btn-success">Upload to Dataset</button>
|
155 |
+
</form>
|
156 |
+
<hr>
|
157 |
+
<h5>Current Dataset Contents</h5>
|
158 |
+
<ul class="list-group">
|
159 |
+
{% for file in files %}
|
160 |
+
<li class="list-group-item">{{ file }}</li>
|
161 |
+
{% endfor %}
|
162 |
+
</ul>
|
163 |
+
{% if manage_status %}
|
164 |
+
<div class="alert alert-info mt-3">{{ manage_status }}</div>
|
165 |
+
{% endif %}
|
166 |
+
</div>
|
167 |
+
</div>
|
168 |
+
|
169 |
+
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
|
170 |
+
<script>
|
171 |
+
document.querySelectorAll('.nav-link').forEach(function(el) {
|
172 |
+
el.addEventListener('click', function() {
|
173 |
+
document.querySelectorAll('.nav-link').forEach(function(x) { x.classList.remove('active') });
|
174 |
+
document.querySelectorAll('.tab-pane').forEach(function(x) { x.classList.remove('show','active') });
|
175 |
+
el.classList.add('active');
|
176 |
+
document.querySelector(el.getAttribute('data-bs-target')).classList.add('show','active');
|
177 |
+
});
|
178 |
+
});
|
179 |
+
</script>
|
180 |
+
</body>
|
181 |
+
</html>
|
182 |
+
"""
|
183 |
|
184 |
+
@app.route("/", methods=["GET","POST"])
|
185 |
+
def index():
|
186 |
+
global schedule_interval
|
187 |
+
status = ""
|
188 |
+
if request.method == "POST":
|
189 |
+
if "manual_run" in request.form:
|
190 |
+
status = run_backup()
|
191 |
+
else:
|
192 |
+
try:
|
193 |
+
schedule_interval = int(request.form.get("interval", "0"))
|
194 |
+
status = f"Scheduled every {schedule_interval} minutes"
|
195 |
+
except:
|
196 |
+
status = "Invalid interval"
|
197 |
+
return render_template_string(
|
198 |
+
HTML,
|
199 |
+
last_run=last_backup_time,
|
200 |
+
interval=schedule_interval,
|
201 |
+
status=status,
|
202 |
+
files=[],
|
203 |
+
manage_status=None
|
204 |
+
)
|
205 |
+
|
206 |
+
@app.route("/manage", methods=["POST"])
|
207 |
+
def manage():
|
208 |
+
drive_url = request.form.get("drive_url")
|
209 |
+
dest_path = request.form.get("dest_path") or ""
|
210 |
+
manage_log = ""
|
211 |
try:
|
212 |
+
# download then upload
|
213 |
+
tmp = os.path.join(DOWNLOAD_DIR, "upload_tmp")
|
214 |
+
shutil.rmtree(tmp, ignore_errors=True)
|
215 |
+
os.makedirs(tmp, exist_ok=True)
|
216 |
+
gdown.download(url=drive_url, output=tmp, quiet=True)
|
217 |
+
login(token=TOKEN)
|
218 |
+
upload_folder(
|
219 |
+
repo_id=REPO_ID,
|
220 |
+
folder_path=tmp,
|
221 |
+
repo_type="dataset",
|
222 |
+
token=TOKEN,
|
223 |
+
path_in_repo=dest_path,
|
224 |
+
commit_message=f"Upload via UI to {dest_path or 'root'}"
|
225 |
+
)
|
226 |
+
manage_log = "Upload successful"
|
227 |
except Exception as e:
|
228 |
+
manage_log = f"Error {e}"
|
229 |
+
api = HfApi()
|
230 |
+
files = api.list_repo_files(repo_id=REPO_ID, repo_type="dataset", token=TOKEN)
|
231 |
+
return render_template_string(
|
232 |
+
HTML,
|
233 |
+
last_run=last_backup_time,
|
234 |
+
interval=schedule_interval,
|
235 |
+
status="",
|
236 |
+
files=files,
|
237 |
+
manage_status=manage_log
|
238 |
+
)
|
239 |
|
|
|
240 |
if __name__ == "__main__":
|
241 |
+
app.run(host="0.0.0.0", port=7860)
|
|
|
|