Spaces:
Running
on
Zero
Running
on
Zero
Permalink Update - storage version 0.1.0
Browse files- app.py +11 -7
- utils/constants.py +5 -1
- utils/storage.md +154 -0
- utils/storage.py +209 -15
- utils/version_info.py +11 -0
app.py
CHANGED
|
@@ -807,7 +807,7 @@ def load_trellis_model():
|
|
| 807 |
gr.Info("TRELLIS_PIPELINE load start", 60)
|
| 808 |
global TRELLIS_PIPELINE
|
| 809 |
loaded = False
|
| 810 |
-
if TRELLIS_PIPELINE == None:
|
| 811 |
try:
|
| 812 |
TRELLIS_PIPELINE = TrellisImageTo3DPipeline.from_pretrained("Surn/TRELLIS-image-large")
|
| 813 |
TRELLIS_PIPELINE.cuda()
|
|
@@ -1177,8 +1177,9 @@ def update_permalink(glb, gaussian, depth_out, depth_src_file):
|
|
| 1177 |
return gr.update(visible=False), ""
|
| 1178 |
|
| 1179 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
| 1180 |
-
|
| 1181 |
-
|
|
|
|
| 1182 |
|
| 1183 |
def create_permalink(glb, gaussian, depth_out, depth_src_file, folder_name_permalink):
|
| 1184 |
"""
|
|
@@ -1202,14 +1203,17 @@ def create_permalink(glb, gaussian, depth_out, depth_src_file, folder_name_perma
|
|
| 1202 |
# return gr.update(visible=False), ""
|
| 1203 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
| 1204 |
|
| 1205 |
-
|
| 1206 |
files=[smallest_file, depth_out, depth_src_file],
|
| 1207 |
-
repo_id=
|
| 1208 |
folder_name=folder_name_permalink,
|
| 1209 |
create_permalink=True,
|
| 1210 |
repo_type="dataset"
|
| 1211 |
-
)
|
| 1212 |
-
|
|
|
|
|
|
|
|
|
|
| 1213 |
|
| 1214 |
def open_permalink(url: str) -> str:
|
| 1215 |
if url and url.strip():
|
|
|
|
| 807 |
gr.Info("TRELLIS_PIPELINE load start", 60)
|
| 808 |
global TRELLIS_PIPELINE
|
| 809 |
loaded = False
|
| 810 |
+
if (TRELLIS_PIPELINE == None):
|
| 811 |
try:
|
| 812 |
TRELLIS_PIPELINE = TrellisImageTo3DPipeline.from_pretrained("Surn/TRELLIS-image-large")
|
| 813 |
TRELLIS_PIPELINE.cuda()
|
|
|
|
| 1177 |
return gr.update(visible=False), ""
|
| 1178 |
|
| 1179 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
| 1180 |
+
# generate_permalink_from_urls now returns a string directly
|
| 1181 |
+
permalink_url = generate_permalink_from_urls(smallest_file, depth_out, depth_src_file)
|
| 1182 |
+
return gr.update(visible=True), permalink_url
|
| 1183 |
|
| 1184 |
def create_permalink(glb, gaussian, depth_out, depth_src_file, folder_name_permalink):
|
| 1185 |
"""
|
|
|
|
| 1203 |
# return gr.update(visible=False), ""
|
| 1204 |
smallest_file = min(file_candidates, key=file_candidates.get)
|
| 1205 |
|
| 1206 |
+
upload_result = upload_files_to_repo(
|
| 1207 |
files=[smallest_file, depth_out, depth_src_file],
|
| 1208 |
+
repo_id=constants.HF_REPO_ID, # Use constant for repo_id
|
| 1209 |
folder_name=folder_name_permalink,
|
| 1210 |
create_permalink=True,
|
| 1211 |
repo_type="dataset"
|
| 1212 |
+
)
|
| 1213 |
+
# upload_files_to_repo returns a dict with "permalink" and "short_permalink"
|
| 1214 |
+
# We are interested in the "short_permalink" if available, otherwise the "permalink"
|
| 1215 |
+
permalink_url = upload_result.get("short_permalink") or upload_result.get("permalink")
|
| 1216 |
+
return permalink_url
|
| 1217 |
|
| 1218 |
def open_permalink(url: str) -> str:
|
| 1219 |
if url and url.strip():
|
utils/constants.py
CHANGED
|
@@ -675,4 +675,8 @@ card_colors_alternating = [
|
|
| 675 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", # Clubs
|
| 676 |
"#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", # Diamonds
|
| 677 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000" # Spades
|
| 678 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 675 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", # Clubs
|
| 676 |
"#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", "#FF0000", # Diamonds
|
| 677 |
"#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000", "#000000" # Spades
|
| 678 |
+
]
|
| 679 |
+
|
| 680 |
+
# Constants for URL shortener
|
| 681 |
+
HF_REPO_ID = "Surn/Storage" # Or your desired repository
|
| 682 |
+
SHORTENER_JSON_FILE = "shortener.json"
|
utils/storage.md
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Storage Module (`modules/storage.py`) Usage Guide
|
| 2 |
+
|
| 3 |
+
The `storage.py` module provides helper functions for:
|
| 4 |
+
- Generating permalinks for 3D viewer projects.
|
| 5 |
+
- Uploading files in batches to a Hugging Face repository.
|
| 6 |
+
- Managing URL shortening by storing (short URL, full URL) pairs in a JSON file on the repository.
|
| 7 |
+
|
| 8 |
+
## Key Functions
|
| 9 |
+
|
| 10 |
+
### 1. `generate_permalink(valid_files, base_url_external, permalink_viewer_url="surn-3d-viewer.hf.space")`
|
| 11 |
+
- **Purpose:**
|
| 12 |
+
Given a list of file paths, it looks for exactly one model file (with an extension defined in `model_extensions`) and exactly two image files (extensions defined in `image_extensions`). If the criteria are met, it returns a permalink URL built from the base URL and query parameters.
|
| 13 |
+
- **Usage Example:**from modules.storage import generate_permalink
|
| 14 |
+
|
| 15 |
+
valid_files = [
|
| 16 |
+
"models/3d_model.glb",
|
| 17 |
+
"images/model_texture.png",
|
| 18 |
+
"images/model_depth.png"
|
| 19 |
+
]
|
| 20 |
+
base_url_external = "https://huggingface.co/datasets/Surn/Storage/resolve/main/saved_models/my_model"
|
| 21 |
+
permalink = generate_permalink(valid_files, base_url_external)
|
| 22 |
+
if permalink:
|
| 23 |
+
print("Permalink:", permalink)
|
| 24 |
+
### 2. `generate_permalink_from_urls(model_url, hm_url, img_url, permalink_viewer_url="surn-3d-viewer.hf.space")`
|
| 25 |
+
- **Purpose:**
|
| 26 |
+
Constructs a permalink URL by combining individual URLs for a 3D model (`model_url`), height map (`hm_url`), and image (`img_url`) into a single URL with corresponding query parameters.
|
| 27 |
+
- **Usage Example:**from modules.storage import generate_permalink_from_urls
|
| 28 |
+
|
| 29 |
+
model_url = "https://example.com/model.glb"
|
| 30 |
+
hm_url = "https://example.com/heightmap.png"
|
| 31 |
+
img_url = "https://example.com/source.png"
|
| 32 |
+
|
| 33 |
+
permalink = generate_permalink_from_urls(model_url, hm_url, img_url)
|
| 34 |
+
print("Generated Permalink:", permalink)
|
| 35 |
+
### 3. `upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, repo_type="dataset", permalink_viewer_url="surn-3d-viewer.hf.space")`
|
| 36 |
+
- **Purpose:**
|
| 37 |
+
Uploads a batch of files (each file represented as a path string) to a specified Hugging Face repository (e.g. `"Surn/Storage"`) under a given folder.
|
| 38 |
+
The function's return type is `Union[Dict[str, Any], List[Tuple[Any, str]]]`.
|
| 39 |
+
- When `create_permalink` is `True` and exactly three valid files (one model and two images) are provided, the function returns a dictionary:```
|
| 40 |
+
{
|
| 41 |
+
"response": <upload_folder_response>,
|
| 42 |
+
"permalink": "<full_permalink_url>",
|
| 43 |
+
"short_permalink": "<shortened_permalink_url_with_sid>"
|
| 44 |
+
}
|
| 45 |
+
``` - Otherwise (or if `create_permalink` is `False` or conditions for permalink creation are not met), it returns a list of tuples, where each tuple is `(upload_folder_response, individual_file_link)`.
|
| 46 |
+
- If no valid files are provided, it returns an empty list `[]` (this case should ideally also return the dictionary with empty/None values for consistency, but currently returns `[]` as per the code).
|
| 47 |
+
- **Usage Example:**
|
| 48 |
+
|
| 49 |
+
**a. Uploading with permalink creation:**from modules.storage import upload_files_to_repo
|
| 50 |
+
|
| 51 |
+
files_for_permalink = [
|
| 52 |
+
"local/path/to/model.glb",
|
| 53 |
+
"local/path/to/heightmap.png",
|
| 54 |
+
"local/path/to/image.png"
|
| 55 |
+
]
|
| 56 |
+
repo_id = "Surn/Storage" # Make sure this is defined, e.g., from constants
|
| 57 |
+
folder_name = "my_new_model_with_permalink"
|
| 58 |
+
|
| 59 |
+
upload_result = upload_files_to_repo(
|
| 60 |
+
files_for_permalink,
|
| 61 |
+
repo_id,
|
| 62 |
+
folder_name,
|
| 63 |
+
create_permalink=True
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
if isinstance(upload_result, dict):
|
| 67 |
+
print("Upload Response:", upload_result.get("response"))
|
| 68 |
+
print("Full Permalink:", upload_result.get("permalink"))
|
| 69 |
+
print("Short Permalink:", upload_result.get("short_permalink"))
|
| 70 |
+
elif upload_result: # Check if list is not empty
|
| 71 |
+
print("Upload Response for individual files:")
|
| 72 |
+
for res, link in upload_result:
|
| 73 |
+
print(f" Response: {res}, Link: {link}")
|
| 74 |
+
else:
|
| 75 |
+
print("No files uploaded or error occurred.")
|
| 76 |
+
**b. Uploading without permalink creation (or if conditions for permalink are not met):**from modules.storage import upload_files_to_repo
|
| 77 |
+
|
| 78 |
+
files_individual = [
|
| 79 |
+
"local/path/to/another_model.obj",
|
| 80 |
+
"local/path/to/texture.jpg"
|
| 81 |
+
]
|
| 82 |
+
repo_id = "Surn/Storage"
|
| 83 |
+
folder_name = "my_other_uploads"
|
| 84 |
+
|
| 85 |
+
upload_results_list = upload_files_to_repo(
|
| 86 |
+
files_individual,
|
| 87 |
+
repo_id,
|
| 88 |
+
folder_name,
|
| 89 |
+
create_permalink=False # Or if create_permalink=True but not 1 model & 2 images
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
if upload_results_list: # Will be a list of tuples
|
| 93 |
+
print("Upload results for individual files:")
|
| 94 |
+
for res, link in upload_results_list:
|
| 95 |
+
print(f" Upload Response: {res}, File Link: {link}")
|
| 96 |
+
else:
|
| 97 |
+
print("No files uploaded or error occurred.")
|
| 98 |
+
### 4. URL Shortening Functions: `gen_full_url(...)` and Helpers
|
| 99 |
+
The module also enables URL shortening by managing a JSON file (e.g. `shortener.json`) in a Hugging Face repository. It supports CRUD-like operations:
|
| 100 |
+
- **Read:** Look up the full URL using a provided short URL ID.
|
| 101 |
+
- **Create:** Generate a new short URL ID for a full URL if no existing mapping exists.
|
| 102 |
+
- **Update/Conflict Handling:**
|
| 103 |
+
If both short URL ID and full URL are provided, it checks consistency and either confirms or reports a conflict.
|
| 104 |
+
|
| 105 |
+
#### `gen_full_url(short_url=None, full_url=None, repo_id=None, repo_type="dataset", permalink_viewer_url="surn-3d-viewer.hf.space", json_file="shortener.json")`
|
| 106 |
+
- **Purpose:**
|
| 107 |
+
Based on which parameter is provided, it retrieves or creates a mapping between a short URL ID and a full URL.
|
| 108 |
+
- If only `short_url` (the ID) is given, it returns the corresponding `full_url`.
|
| 109 |
+
- If only `full_url` is given, it looks up an existing `short_url` ID or generates and stores a new one.
|
| 110 |
+
- If both are given, it validates and returns the mapping or an error status.
|
| 111 |
+
- **Returns:** A tuple `(status_message, result_url)`, where `status_message` indicates the outcome (e.g., `"success_retrieved_full"`, `"created_short"`) and `result_url` is the relevant URL (full or short ID).
|
| 112 |
+
- **Usage Examples:**
|
| 113 |
+
|
| 114 |
+
**a. Convert a full URL into a short URL ID:**from modules.storage import gen_full_url
|
| 115 |
+
from modules.constants import HF_REPO_ID, SHORTENER_JSON_FILE # Assuming these are defined
|
| 116 |
+
|
| 117 |
+
full_permalink = "https://surn-3d-viewer.hf.space/?3d=https%3A%2F%2Fexample.com%2Fmodel.glb&hm=https%3A%2F%2Fexample.com%2Fheightmap.png&image=https%3A%2F%2Fexample.com%2Fsource.png"
|
| 118 |
+
|
| 119 |
+
status, short_id = gen_full_url(
|
| 120 |
+
full_url=full_permalink,
|
| 121 |
+
repo_id=HF_REPO_ID,
|
| 122 |
+
json_file=SHORTENER_JSON_FILE
|
| 123 |
+
)
|
| 124 |
+
print("Status:", status)
|
| 125 |
+
if status == "created_short" or status == "success_retrieved_short":
|
| 126 |
+
print("Shortened URL ID:", short_id)
|
| 127 |
+
# Construct the full short URL for sharing:
|
| 128 |
+
# permalink_viewer_url = "surn-3d-viewer.hf.space" # Or from constants
|
| 129 |
+
# shareable_short_url = f"https://{permalink_viewer_url}/?sid={short_id}"
|
| 130 |
+
# print("Shareable Short URL:", shareable_short_url)
|
| 131 |
+
**b. Retrieve the full URL from a short URL ID:**from modules.storage import gen_full_url
|
| 132 |
+
from modules.constants import HF_REPO_ID, SHORTENER_JSON_FILE # Assuming these are defined
|
| 133 |
+
|
| 134 |
+
short_id_to_lookup = "aBcDeFg1" # Example short URL ID
|
| 135 |
+
|
| 136 |
+
status, retrieved_full_url = gen_full_url(
|
| 137 |
+
short_url=short_id_to_lookup,
|
| 138 |
+
repo_id=HF_REPO_ID,
|
| 139 |
+
json_file=SHORTENER_JSON_FILE
|
| 140 |
+
)
|
| 141 |
+
print("Status:", status)
|
| 142 |
+
if status == "success_retrieved_full":
|
| 143 |
+
print("Retrieved Full URL:", retrieved_full_url)
|
| 144 |
+
## Notes
|
| 145 |
+
- **Authentication:** All functions that interact with Hugging Face Hub use the HF API token defined as `HF_API_TOKEN` in `modules/constants.py`. Ensure this environment variable is correctly set.
|
| 146 |
+
- **Constants:** Functions like `gen_full_url` and `upload_files_to_repo` (when creating short links) rely on `HF_REPO_ID` and `SHORTENER_JSON_FILE` from `modules/constants.py` for the URL shortening feature.
|
| 147 |
+
- **File Types:** Only files with extensions included in `upload_file_types` (a combination of `model_extensions` and `image_extensions` from `modules/constants.py`) are processed by `upload_files_to_repo`.
|
| 148 |
+
- **Repository Configuration:** When using URL shortening and file uploads, ensure that the specified Hugging Face repository (e.g., defined by `HF_REPO_ID`) exists and that you have write permissions.
|
| 149 |
+
- **Temporary Directory:** `upload_files_to_repo` temporarily copies files to a local directory (configured by `TMPDIR` in `modules/constants.py`) before uploading.
|
| 150 |
+
- **Error Handling:** Functions include basic error handling (e.g., catching `RepositoryNotFoundError`, `EntryNotFoundError`, JSON decoding errors, or upload issues) and print messages to the console for debugging. Review function return values to handle these cases appropriately in your application.
|
| 151 |
+
|
| 152 |
+
---
|
| 153 |
+
|
| 154 |
+
This guide provides the essential usage examples for interacting with the storage and URL-shortening functionality. You can integrate these examples into your application or use them as a reference when extending functionality.
|
utils/storage.py
CHANGED
|
@@ -1,10 +1,17 @@
|
|
| 1 |
# utils/storage.py
|
|
|
|
| 2 |
import os
|
| 3 |
import urllib.parse
|
| 4 |
import tempfile
|
| 5 |
import shutil
|
| 6 |
-
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
def generate_permalink(valid_files, base_url_external, permalink_viewer_url="surn-3d-viewer.hf.space"):
|
| 10 |
"""
|
|
@@ -49,7 +56,14 @@ def generate_permalink_from_urls(model_url, hm_url, img_url, permalink_viewer_ur
|
|
| 49 |
query_str = urllib.parse.urlencode(params)
|
| 50 |
return f"https://{permalink_viewer_url}/?{query_str}"
|
| 51 |
|
| 52 |
-
def upload_files_to_repo(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
"""
|
| 54 |
Uploads multiple files to a Hugging Face repository using a batch upload approach via upload_folder.
|
| 55 |
|
|
@@ -61,18 +75,24 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
| 61 |
returns a single permalink to the project with query parameters.
|
| 62 |
Otherwise, returns individual permalinks for each file.
|
| 63 |
repo_type (str): Repository type ("space", "dataset", etc.). Default is "dataset".
|
|
|
|
| 64 |
|
| 65 |
Returns:
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
"""
|
| 72 |
# Log in using the HF API token.
|
| 73 |
login(token=HF_API_TOKEN)
|
| 74 |
|
| 75 |
valid_files = []
|
|
|
|
| 76 |
|
| 77 |
# Ensure folder_name does not have a trailing slash.
|
| 78 |
folder_name = folder_name.rstrip("/")
|
|
@@ -87,10 +107,17 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
| 87 |
valid_files.append(f)
|
| 88 |
|
| 89 |
if not valid_files:
|
| 90 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
|
| 92 |
# Create a temporary directory; copy valid files directly into it.
|
| 93 |
-
with tempfile.TemporaryDirectory() as temp_dir:
|
| 94 |
for file_path in valid_files:
|
| 95 |
filename = os.path.basename(file_path)
|
| 96 |
dest_path = os.path.join(temp_dir, filename)
|
|
@@ -107,8 +134,6 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
| 107 |
)
|
| 108 |
|
| 109 |
# Construct external URLs for each uploaded file.
|
| 110 |
-
# For datasets, files are served at:
|
| 111 |
-
# https://huggingface.co/datasets/<repo_id>/resolve/main/<folder_name>/<filename>
|
| 112 |
base_url_external = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{folder_name}"
|
| 113 |
individual_links = []
|
| 114 |
for file_path in valid_files:
|
|
@@ -118,10 +143,179 @@ def upload_files_to_repo(files, repo_id, folder_name, create_permalink=False, re
|
|
| 118 |
|
| 119 |
# If permalink creation is requested and exactly 3 valid files are provided,
|
| 120 |
# try to generate a permalink using generate_permalink().
|
| 121 |
-
if create_permalink
|
| 122 |
-
permalink =
|
| 123 |
if permalink:
|
| 124 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
|
| 126 |
# Otherwise, return individual tuples for each file.
|
| 127 |
return [(response, link) for link in individual_links]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
# utils/storage.py
|
| 2 |
+
__version__ = "0.1.0" # Added version
|
| 3 |
import os
|
| 4 |
import urllib.parse
|
| 5 |
import tempfile
|
| 6 |
import shutil
|
| 7 |
+
import json
|
| 8 |
+
import base64
|
| 9 |
+
from huggingface_hub import login, upload_folder, hf_hub_download, HfApi
|
| 10 |
+
from huggingface_hub.utils import RepositoryNotFoundError, EntryNotFoundError
|
| 11 |
+
from utils.constants import HF_API_TOKEN, upload_file_types, model_extensions, image_extensions, HF_REPO_ID, SHORTENER_JSON_FILE
|
| 12 |
+
from typing import Any, Dict, List, Tuple, Union
|
| 13 |
+
|
| 14 |
+
# see storage.md for detailed information about the storage module and its functions.
|
| 15 |
|
| 16 |
def generate_permalink(valid_files, base_url_external, permalink_viewer_url="surn-3d-viewer.hf.space"):
|
| 17 |
"""
|
|
|
|
| 56 |
query_str = urllib.parse.urlencode(params)
|
| 57 |
return f"https://{permalink_viewer_url}/?{query_str}"
|
| 58 |
|
| 59 |
+
def upload_files_to_repo(
|
| 60 |
+
files: List[Any],
|
| 61 |
+
repo_id: str,
|
| 62 |
+
folder_name: str,
|
| 63 |
+
create_permalink: bool = False,
|
| 64 |
+
repo_type: str = "dataset",
|
| 65 |
+
permalink_viewer_url: str = "surn-3d-viewer.hf.space"
|
| 66 |
+
) -> Union[Dict[str, Any], List[Tuple[Any, str]]]:
|
| 67 |
"""
|
| 68 |
Uploads multiple files to a Hugging Face repository using a batch upload approach via upload_folder.
|
| 69 |
|
|
|
|
| 75 |
returns a single permalink to the project with query parameters.
|
| 76 |
Otherwise, returns individual permalinks for each file.
|
| 77 |
repo_type (str): Repository type ("space", "dataset", etc.). Default is "dataset".
|
| 78 |
+
permalink_viewer_url (str): The base viewer URL.
|
| 79 |
|
| 80 |
Returns:
|
| 81 |
+
Union[Dict[str, Any], List[Tuple[Any, str]]]:
|
| 82 |
+
If create_permalink is True and files match the criteria:
|
| 83 |
+
dict: {
|
| 84 |
+
"response": <upload response>,
|
| 85 |
+
"permalink": <full_permalink URL>,
|
| 86 |
+
"short_permalink": <shortened permalink URL>
|
| 87 |
+
}
|
| 88 |
+
Otherwise:
|
| 89 |
+
list: A list of tuples (response, permalink) for each file.
|
| 90 |
"""
|
| 91 |
# Log in using the HF API token.
|
| 92 |
login(token=HF_API_TOKEN)
|
| 93 |
|
| 94 |
valid_files = []
|
| 95 |
+
permalink_short = None
|
| 96 |
|
| 97 |
# Ensure folder_name does not have a trailing slash.
|
| 98 |
folder_name = folder_name.rstrip("/")
|
|
|
|
| 107 |
valid_files.append(f)
|
| 108 |
|
| 109 |
if not valid_files:
|
| 110 |
+
# Return a dictionary with None values for permalinks if create_permalink was True
|
| 111 |
+
if create_permalink:
|
| 112 |
+
return {
|
| 113 |
+
"response": "No valid files to upload.",
|
| 114 |
+
"permalink": None,
|
| 115 |
+
"short_permalink": None
|
| 116 |
+
}
|
| 117 |
+
return []
|
| 118 |
|
| 119 |
# Create a temporary directory; copy valid files directly into it.
|
| 120 |
+
with tempfile.TemporaryDirectory(dir=os.getenv("TMPDIR", "/tmp")) as temp_dir:
|
| 121 |
for file_path in valid_files:
|
| 122 |
filename = os.path.basename(file_path)
|
| 123 |
dest_path = os.path.join(temp_dir, filename)
|
|
|
|
| 134 |
)
|
| 135 |
|
| 136 |
# Construct external URLs for each uploaded file.
|
|
|
|
|
|
|
| 137 |
base_url_external = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{folder_name}"
|
| 138 |
individual_links = []
|
| 139 |
for file_path in valid_files:
|
|
|
|
| 143 |
|
| 144 |
# If permalink creation is requested and exactly 3 valid files are provided,
|
| 145 |
# try to generate a permalink using generate_permalink().
|
| 146 |
+
if create_permalink: # No need to check len(valid_files) == 3 here, generate_permalink will handle it
|
| 147 |
+
permalink = generate_permalink(valid_files, base_url_external, permalink_viewer_url)
|
| 148 |
if permalink:
|
| 149 |
+
status, short_id = gen_full_url(
|
| 150 |
+
full_url=permalink,
|
| 151 |
+
repo_id=HF_REPO_ID, # This comes from constants
|
| 152 |
+
json_file=SHORTENER_JSON_FILE # This comes from constants
|
| 153 |
+
)
|
| 154 |
+
if status in ["created_short", "success_retrieved_short", "exists_match"]:
|
| 155 |
+
permalink_short = f"https://{permalink_viewer_url}/?sid={short_id}"
|
| 156 |
+
else: # Shortening failed or conflict not resolved to a usable short_id
|
| 157 |
+
permalink_short = None
|
| 158 |
+
print(f"URL shortening status: {status} for {permalink}")
|
| 159 |
+
|
| 160 |
+
return {
|
| 161 |
+
"response": response,
|
| 162 |
+
"permalink": permalink,
|
| 163 |
+
"short_permalink": permalink_short
|
| 164 |
+
}
|
| 165 |
+
else: # generate_permalink returned None (criteria not met)
|
| 166 |
+
return {
|
| 167 |
+
"response": response, # Still return upload response
|
| 168 |
+
"permalink": None,
|
| 169 |
+
"short_permalink": None
|
| 170 |
+
}
|
| 171 |
|
| 172 |
# Otherwise, return individual tuples for each file.
|
| 173 |
return [(response, link) for link in individual_links]
|
| 174 |
+
|
| 175 |
+
def _generate_short_id(length=8):
|
| 176 |
+
"""Generates a random base64 URL-safe string."""
|
| 177 |
+
return base64.urlsafe_b64encode(os.urandom(length * 2))[:length].decode('utf-8')
|
| 178 |
+
|
| 179 |
+
def _get_json_from_repo(repo_id, json_file_name, repo_type="dataset"):
|
| 180 |
+
"""Downloads and loads the JSON file from the repo. Returns empty list if not found or error."""
|
| 181 |
+
try:
|
| 182 |
+
login(token=HF_API_TOKEN)
|
| 183 |
+
json_path = hf_hub_download(
|
| 184 |
+
repo_id=repo_id,
|
| 185 |
+
filename=json_file_name,
|
| 186 |
+
repo_type=repo_type,
|
| 187 |
+
token=HF_API_TOKEN # Added token for consistency, though login might suffice
|
| 188 |
+
)
|
| 189 |
+
with open(json_path, 'r') as f:
|
| 190 |
+
data = json.load(f)
|
| 191 |
+
os.remove(json_path) # Clean up downloaded file
|
| 192 |
+
return data
|
| 193 |
+
except RepositoryNotFoundError:
|
| 194 |
+
print(f"Repository {repo_id} not found.")
|
| 195 |
+
return []
|
| 196 |
+
except EntryNotFoundError:
|
| 197 |
+
print(f"JSON file {json_file_name} not found in {repo_id}. Initializing with empty list.")
|
| 198 |
+
return []
|
| 199 |
+
except json.JSONDecodeError:
|
| 200 |
+
print(f"Error decoding JSON from {json_file_name}. Returning empty list.")
|
| 201 |
+
return []
|
| 202 |
+
except Exception as e:
|
| 203 |
+
print(f"An unexpected error occurred while fetching {json_file_name}: {e}")
|
| 204 |
+
return []
|
| 205 |
+
|
| 206 |
+
def _upload_json_to_repo(data, repo_id, json_file_name, repo_type="dataset"):
|
| 207 |
+
"""Uploads the JSON data to the specified file in the repo."""
|
| 208 |
+
try:
|
| 209 |
+
login(token=HF_API_TOKEN)
|
| 210 |
+
api = HfApi()
|
| 211 |
+
# Use a temporary directory specified by TMPDIR or default to system temp
|
| 212 |
+
temp_dir_for_json = os.getenv("TMPDIR", tempfile.gettempdir())
|
| 213 |
+
os.makedirs(temp_dir_for_json, exist_ok=True)
|
| 214 |
+
|
| 215 |
+
with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json", dir=temp_dir_for_json) as tmp_file:
|
| 216 |
+
json.dump(data, tmp_file, indent=2)
|
| 217 |
+
tmp_file_path = tmp_file.name
|
| 218 |
+
|
| 219 |
+
api.upload_file(
|
| 220 |
+
path_or_fileobj=tmp_file_path,
|
| 221 |
+
path_in_repo=json_file_name,
|
| 222 |
+
repo_id=repo_id,
|
| 223 |
+
repo_type=repo_type,
|
| 224 |
+
commit_message=f"Update {json_file_name}"
|
| 225 |
+
)
|
| 226 |
+
os.remove(tmp_file_path) # Clean up temporary file
|
| 227 |
+
return True
|
| 228 |
+
except Exception as e:
|
| 229 |
+
print(f"Failed to upload {json_file_name} to {repo_id}: {e}")
|
| 230 |
+
if 'tmp_file_path' in locals() and os.path.exists(tmp_file_path):
|
| 231 |
+
os.remove(tmp_file_path) # Ensure cleanup on error too
|
| 232 |
+
return False
|
| 233 |
+
|
| 234 |
+
def _find_url_in_json(data, short_url=None, full_url=None):
|
| 235 |
+
"""
|
| 236 |
+
Searches the JSON data.
|
| 237 |
+
If short_url is provided, returns the corresponding full_url or None.
|
| 238 |
+
If full_url is provided, returns the corresponding short_url or None.
|
| 239 |
+
"""
|
| 240 |
+
if not data: # Handles cases where data might be None or empty
|
| 241 |
+
return None
|
| 242 |
+
if short_url:
|
| 243 |
+
for item in data:
|
| 244 |
+
if item.get("short_url") == short_url:
|
| 245 |
+
return item.get("full_url")
|
| 246 |
+
if full_url:
|
| 247 |
+
for item in data:
|
| 248 |
+
if item.get("full_url") == full_url:
|
| 249 |
+
return item.get("short_url")
|
| 250 |
+
return None
|
| 251 |
+
|
| 252 |
+
def _add_url_to_json(data, short_url, full_url):
|
| 253 |
+
"""Adds a new short_url/full_url pair to the data. Returns updated data."""
|
| 254 |
+
if data is None:
|
| 255 |
+
data = []
|
| 256 |
+
data.append({"short_url": short_url, "full_url": full_url})
|
| 257 |
+
return data
|
| 258 |
+
|
| 259 |
+
def gen_full_url(short_url=None, full_url=None, repo_id=None, repo_type="dataset", permalink_viewer_url="surn-3d-viewer.hf.space", json_file="shortener.json"):
|
| 260 |
+
"""
|
| 261 |
+
Manages short URLs and their corresponding full URLs in a JSON file stored in a Hugging Face repository.
|
| 262 |
+
|
| 263 |
+
- If short_url is provided, attempts to retrieve and return the full_url.
|
| 264 |
+
- If full_url is provided, attempts to retrieve an existing short_url or creates a new one, stores it, and returns the short_url.
|
| 265 |
+
- If both are provided, checks for consistency or creates a new entry.
|
| 266 |
+
- If neither is provided, or repo_id is missing, returns an error status.
|
| 267 |
+
|
| 268 |
+
Returns:
|
| 269 |
+
tuple: (status_message, result_url)
|
| 270 |
+
status_message can be "success", "created", "exists", "error", "not_found".
|
| 271 |
+
result_url is the relevant URL (short or full) or None if an error occurs or not found.
|
| 272 |
+
"""
|
| 273 |
+
if not repo_id:
|
| 274 |
+
return "error_repo_id_missing", None
|
| 275 |
+
if not short_url and not full_url:
|
| 276 |
+
return "error_no_input", None
|
| 277 |
+
|
| 278 |
+
login(token=HF_API_TOKEN) # Ensure login at the beginning
|
| 279 |
+
url_data = _get_json_from_repo(repo_id, json_file, repo_type)
|
| 280 |
+
|
| 281 |
+
# Case 1: Only short_url provided (lookup full_url)
|
| 282 |
+
if short_url and not full_url:
|
| 283 |
+
found_full_url = _find_url_in_json(url_data, short_url=short_url)
|
| 284 |
+
return ("success_retrieved_full", found_full_url) if found_full_url else ("not_found_short", None)
|
| 285 |
+
|
| 286 |
+
# Case 2: Only full_url provided (lookup or create short_url)
|
| 287 |
+
if full_url and not short_url:
|
| 288 |
+
existing_short_url = _find_url_in_json(url_data, full_url=full_url)
|
| 289 |
+
if existing_short_url:
|
| 290 |
+
return "success_retrieved_short", existing_short_url
|
| 291 |
+
else:
|
| 292 |
+
# Create new short_url
|
| 293 |
+
new_short_id = _generate_short_id()
|
| 294 |
+
url_data = _add_url_to_json(url_data, new_short_id, full_url)
|
| 295 |
+
if _upload_json_to_repo(url_data, repo_id, json_file, repo_type):
|
| 296 |
+
return "created_short", new_short_id
|
| 297 |
+
else:
|
| 298 |
+
return "error_upload", None
|
| 299 |
+
|
| 300 |
+
# Case 3: Both short_url and full_url provided
|
| 301 |
+
if short_url and full_url:
|
| 302 |
+
found_full_for_short = _find_url_in_json(url_data, short_url=short_url)
|
| 303 |
+
found_short_for_full = _find_url_in_json(url_data, full_url=full_url)
|
| 304 |
+
|
| 305 |
+
if found_full_for_short == full_url:
|
| 306 |
+
return "exists_match", short_url
|
| 307 |
+
if found_full_for_short is not None and found_full_for_short != full_url:
|
| 308 |
+
return "error_conflict_short_exists_different_full", short_url
|
| 309 |
+
if found_short_for_full is not None and found_short_for_full != short_url:
|
| 310 |
+
return "error_conflict_full_exists_different_short", found_short_for_full
|
| 311 |
+
|
| 312 |
+
# If short_url is provided and not found, or full_url is provided and not found,
|
| 313 |
+
# or neither is found, then create a new entry with the provided short_url and full_url.
|
| 314 |
+
# This effectively allows specifying a custom short_url if it's not already taken.
|
| 315 |
+
url_data = _add_url_to_json(url_data, short_url, full_url)
|
| 316 |
+
if _upload_json_to_repo(url_data, repo_id, json_file, repo_type):
|
| 317 |
+
return "created_specific_pair", short_url
|
| 318 |
+
else:
|
| 319 |
+
return "error_upload", None
|
| 320 |
+
|
| 321 |
+
return "error_unhandled_case", None # Should not be reached
|
utils/version_info.py
CHANGED
|
@@ -103,6 +103,15 @@ def versions_html():
|
|
| 103 |
</a>
|
| 104 |
'''
|
| 105 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
v_html = f"""
|
| 107 |
version: <a href="https://huggingface.co/spaces/Surn/HexaGrid/commit/{"huggingface" if commit == "<none>" else commit}" target="_blank">{"huggingface" if commit == "<none>" else commit}</a>
|
| 108 |
 • 
|
|
@@ -123,6 +132,8 @@ def versions_html():
|
|
| 123 |
gradio: {gr.__version__}
|
| 124 |
 • 
|
| 125 |
{toggle_dark_link}
|
|
|
|
|
|
|
| 126 |
<br>
|
| 127 |
Full GPU Info:{get_torch_info()}
|
| 128 |
"""
|
|
|
|
| 103 |
</a>
|
| 104 |
'''
|
| 105 |
|
| 106 |
+
# Add a link to the shortener JSON file in the Hugging Face repo
|
| 107 |
+
from utils.constants import HF_REPO_ID, SHORTENER_JSON_FILE # Import constants
|
| 108 |
+
shortener_url = f"https://huggingface.co/datasets/{HF_REPO_ID}/resolve/main/{SHORTENER_JSON_FILE}"
|
| 109 |
+
shortener_link = f'''
|
| 110 |
+
<a href="{shortener_url}" target="_blank" style="cursor: pointer; text-decoration: underline;">
|
| 111 |
+
View Shortener JSON
|
| 112 |
+
</a>
|
| 113 |
+
'''
|
| 114 |
+
|
| 115 |
v_html = f"""
|
| 116 |
version: <a href="https://huggingface.co/spaces/Surn/HexaGrid/commit/{"huggingface" if commit == "<none>" else commit}" target="_blank">{"huggingface" if commit == "<none>" else commit}</a>
|
| 117 |
 • 
|
|
|
|
| 132 |
gradio: {gr.__version__}
|
| 133 |
 • 
|
| 134 |
{toggle_dark_link}
|
| 135 |
+
 • 
|
| 136 |
+
{shortener_link}
|
| 137 |
<br>
|
| 138 |
Full GPU Info:{get_torch_info()}
|
| 139 |
"""
|