testdeep123 commited on
Commit
e5343b2
·
verified ·
1 Parent(s): 0916b73

Create main.py

Browse files
Files changed (1) hide show
  1. main.py +64 -0
main.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os, shutil, zipfile, gdown
2
+ from huggingface_hub import HfApi, login, upload_folder
3
+
4
+ FOLDER_URL = os.environ.get("FOLDER_URL")
5
+ DOWNLOAD_DIR = "backups"
6
+ EXTRACT_DIR = "extracted_backups"
7
+ REPO_ID = os.environ.get("REPO_ID")
8
+ TOKEN = os.environ.get("HF_TOKEN")
9
+
10
+
11
+ def run_backup():
12
+ shutil.rmtree(DOWNLOAD_DIR, ignore_errors=True)
13
+ shutil.rmtree(EXTRACT_DIR, ignore_errors=True)
14
+ os.makedirs(DOWNLOAD_DIR, exist_ok=True)
15
+ os.makedirs(EXTRACT_DIR, exist_ok=True)
16
+
17
+ gdown.download_folder(
18
+ url=FOLDER_URL,
19
+ output=DOWNLOAD_DIR,
20
+ use_cookies=False,
21
+ quiet=False
22
+ )
23
+
24
+ for root, _, files in os.walk(DOWNLOAD_DIR):
25
+ for f in files:
26
+ if f.endswith(".zip"):
27
+ zp = os.path.join(root, f)
28
+ with zipfile.ZipFile(zp) as z:
29
+ z.extractall(EXTRACT_DIR)
30
+
31
+ # Fix folder name typo
32
+ bad = os.path.join(EXTRACT_DIR, "world_nither")
33
+ good = os.path.join(EXTRACT_DIR, "world_nether")
34
+ if os.path.exists(bad) and not os.path.exists(good):
35
+ os.rename(bad, good)
36
+
37
+ login(token=TOKEN)
38
+ api = HfApi()
39
+ try:
40
+ api.delete_repo(repo_id=REPO_ID, repo_type="dataset")
41
+ except Exception as err:
42
+ print("delete skipped:", err)
43
+
44
+ api.create_repo(repo_id=REPO_ID, repo_type="dataset", private=False, exist_ok=True)
45
+
46
+ subfolders = {
47
+ "world": os.path.join(EXTRACT_DIR, "world"),
48
+ "world_nether": os.path.join(EXTRACT_DIR, "world_nether"),
49
+ "world_the_end": os.path.join(EXTRACT_DIR, "world_the_end"),
50
+ "plugins": os.path.join(EXTRACT_DIR, "plugins")
51
+ }
52
+
53
+ for name, path in subfolders.items():
54
+ if os.path.exists(path):
55
+ upload_folder(
56
+ repo_id=REPO_ID,
57
+ folder_path=path,
58
+ repo_type="dataset",
59
+ token=TOKEN,
60
+ path_in_repo=name,
61
+ commit_message="add " + name
62
+ )
63
+
64
+ return "Backup complete"