Spaces:
Running
Running
change permission
Browse files- Dockerfile +10 -2
- app.py +76 -30
- download_images.py +38 -9
Dockerfile
CHANGED
@@ -10,7 +10,7 @@ RUN apt-get update && \
|
|
10 |
|
11 |
# Create a non-root user to run the application
|
12 |
RUN useradd -m appuser && \
|
13 |
-
mkdir -p /home/appuser/app /home/appuser/.cache && \
|
14 |
chown -R appuser:appuser /home/appuser
|
15 |
|
16 |
# Copy requirements first to leverage Docker cache
|
@@ -22,7 +22,9 @@ COPY . .
|
|
22 |
|
23 |
# Create upload directory with proper permissions
|
24 |
RUN mkdir -p static/uploads static/metadata && \
|
25 |
-
chmod -R 777 static
|
|
|
|
|
26 |
|
27 |
# Set environment variables for Hugging Face
|
28 |
ENV PYTHONUNBUFFERED=1
|
@@ -47,6 +49,12 @@ RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true \
|
|
47 |
echo "Hugging Face credentials not set, skipping image download."; \
|
48 |
fi
|
49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
50 |
# Expose port for Hugging Face Spaces (uses port 7860)
|
51 |
EXPOSE 7860
|
52 |
|
|
|
10 |
|
11 |
# Create a non-root user to run the application
|
12 |
RUN useradd -m appuser && \
|
13 |
+
mkdir -p /home/appuser/app /home/appuser/.cache /home/appuser/.image_uploader && \
|
14 |
chown -R appuser:appuser /home/appuser
|
15 |
|
16 |
# Copy requirements first to leverage Docker cache
|
|
|
22 |
|
23 |
# Create upload directory with proper permissions
|
24 |
RUN mkdir -p static/uploads static/metadata && \
|
25 |
+
chmod -R 777 static && \
|
26 |
+
mkdir -p /tmp/.image_uploader && \
|
27 |
+
chmod -R 777 /tmp/.image_uploader
|
28 |
|
29 |
# Set environment variables for Hugging Face
|
30 |
ENV PYTHONUNBUFFERED=1
|
|
|
49 |
echo "Hugging Face credentials not set, skipping image download."; \
|
50 |
fi
|
51 |
|
52 |
+
# Set proper ownership for all files
|
53 |
+
RUN chown -R appuser:appuser /app
|
54 |
+
|
55 |
+
# Switch to non-root user
|
56 |
+
USER appuser
|
57 |
+
|
58 |
# Expose port for Hugging Face Spaces (uses port 7860)
|
59 |
EXPOSE 7860
|
60 |
|
app.py
CHANGED
@@ -39,10 +39,22 @@ METADATA_DIR = Path("static/metadata")
|
|
39 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
40 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
42 |
# Initialize metadata file if it doesn't exist
|
43 |
-
if not METADATA_FILE.exists():
|
44 |
-
|
45 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
46 |
|
47 |
# Mount static directory
|
48 |
app.mount("/static", StaticFiles(directory="static"), name="static")
|
@@ -138,44 +150,78 @@ def verify_auth(request: Request):
|
|
138 |
)
|
139 |
return True
|
140 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
141 |
def get_image_metadata():
|
142 |
"""Get all image metadata including hashtags from local storage and sync with HF if needed."""
|
143 |
-
|
144 |
-
with open(METADATA_FILE, "r") as f:
|
145 |
-
metadata = json.load(f)
|
146 |
-
|
147 |
-
# In production, sync metadata to Hugging Face if it exists locally but not on HF
|
148 |
-
if (
|
149 |
-
os.environ.get("ENV", "development") == "production"
|
150 |
-
and HF_USERNAME
|
151 |
-
and HF_TOKEN
|
152 |
-
):
|
153 |
-
try:
|
154 |
-
# Only upload if there are changes (we'd need to implement a proper change tracking mechanism)
|
155 |
-
# For now, we'll upload every time to ensure consistency
|
156 |
-
metadata_str = json.dumps(metadata)
|
157 |
-
hf_api.upload_file(
|
158 |
-
path_or_fileobj=io.BytesIO(metadata_str.encode()),
|
159 |
-
path_in_repo=f"{METADATA_PATH}/image_metadata.json",
|
160 |
-
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
161 |
-
repo_type="dataset",
|
162 |
-
token=HF_TOKEN,
|
163 |
-
)
|
164 |
-
except Exception as e:
|
165 |
-
print(f"Error syncing metadata to Hugging Face: {e}")
|
166 |
|
167 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
168 |
|
169 |
# If metadata file doesn't exist locally, create it
|
170 |
-
with open(
|
171 |
json.dump({}, f)
|
172 |
return {}
|
173 |
|
174 |
def save_image_metadata(metadata):
|
175 |
"""Save image metadata to the local JSON file and sync with HF."""
|
|
|
|
|
176 |
# Always save locally first
|
177 |
-
|
178 |
-
|
|
|
|
|
|
|
|
|
|
|
179 |
|
180 |
# In production, also save to Hugging Face
|
181 |
if os.environ.get("ENV", "development") == "production" and HF_USERNAME and HF_TOKEN:
|
|
|
39 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
40 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
41 |
|
42 |
+
# Alternative metadata location with guaranteed write permissions
|
43 |
+
HOME_DIR = Path(os.environ.get("HOME", "/tmp"))
|
44 |
+
ALT_METADATA_DIR = HOME_DIR / ".image_uploader"
|
45 |
+
ALT_METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
46 |
+
ALT_METADATA_FILE = ALT_METADATA_DIR / "image_metadata.json"
|
47 |
+
|
48 |
# Initialize metadata file if it doesn't exist
|
49 |
+
if not METADATA_FILE.exists() and not ALT_METADATA_FILE.exists():
|
50 |
+
try:
|
51 |
+
with open(METADATA_FILE, "w") as f:
|
52 |
+
json.dump({}, f)
|
53 |
+
print(f"Initialized metadata file at {METADATA_FILE}")
|
54 |
+
except PermissionError:
|
55 |
+
with open(ALT_METADATA_FILE, "w") as f:
|
56 |
+
json.dump({}, f)
|
57 |
+
print(f"Initialized metadata file at alternative location: {ALT_METADATA_FILE}")
|
58 |
|
59 |
# Mount static directory
|
60 |
app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
|
150 |
)
|
151 |
return True
|
152 |
|
153 |
+
|
154 |
+
def get_metadata_file():
|
155 |
+
"""Get the appropriate metadata file based on write permissions."""
|
156 |
+
# Try to write to the primary location
|
157 |
+
try:
|
158 |
+
if not METADATA_FILE.exists():
|
159 |
+
with open(METADATA_FILE, "w") as f:
|
160 |
+
json.dump({}, f)
|
161 |
+
# Test write permission
|
162 |
+
os.access(METADATA_FILE, os.W_OK)
|
163 |
+
return METADATA_FILE
|
164 |
+
except (PermissionError, OSError):
|
165 |
+
print(
|
166 |
+
f"Warning: Cannot write to {METADATA_FILE}, using alternative location: {ALT_METADATA_FILE}"
|
167 |
+
)
|
168 |
+
if not ALT_METADATA_FILE.exists():
|
169 |
+
with open(ALT_METADATA_FILE, "w") as f:
|
170 |
+
json.dump({}, f)
|
171 |
+
return ALT_METADATA_FILE
|
172 |
+
|
173 |
+
|
174 |
def get_image_metadata():
|
175 |
"""Get all image metadata including hashtags from local storage and sync with HF if needed."""
|
176 |
+
metadata_file = get_metadata_file()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
177 |
|
178 |
+
if metadata_file.exists():
|
179 |
+
try:
|
180 |
+
with open(metadata_file, "r") as f:
|
181 |
+
metadata = json.load(f)
|
182 |
+
|
183 |
+
# In production, sync metadata to Hugging Face if it exists locally but not on HF
|
184 |
+
if (
|
185 |
+
os.environ.get("ENV", "development") == "production"
|
186 |
+
and HF_USERNAME
|
187 |
+
and HF_TOKEN
|
188 |
+
):
|
189 |
+
try:
|
190 |
+
# Only upload if there are changes (we'd need to implement a proper change tracking mechanism)
|
191 |
+
# For now, we'll upload every time to ensure consistency
|
192 |
+
metadata_str = json.dumps(metadata)
|
193 |
+
hf_api.upload_file(
|
194 |
+
path_or_fileobj=io.BytesIO(metadata_str.encode()),
|
195 |
+
path_in_repo=f"{METADATA_PATH}/image_metadata.json",
|
196 |
+
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
197 |
+
repo_type="dataset",
|
198 |
+
token=HF_TOKEN,
|
199 |
+
)
|
200 |
+
except Exception as e:
|
201 |
+
print(f"Error syncing metadata to Hugging Face: {e}")
|
202 |
+
|
203 |
+
return metadata
|
204 |
+
except Exception as e:
|
205 |
+
print(f"Error reading metadata file: {e}")
|
206 |
+
return {}
|
207 |
|
208 |
# If metadata file doesn't exist locally, create it
|
209 |
+
with open(metadata_file, "w") as f:
|
210 |
json.dump({}, f)
|
211 |
return {}
|
212 |
|
213 |
def save_image_metadata(metadata):
|
214 |
"""Save image metadata to the local JSON file and sync with HF."""
|
215 |
+
metadata_file = get_metadata_file()
|
216 |
+
|
217 |
# Always save locally first
|
218 |
+
try:
|
219 |
+
with open(metadata_file, "w") as f:
|
220 |
+
json.dump(metadata, f)
|
221 |
+
print(f"Metadata saved to {metadata_file}")
|
222 |
+
except Exception as e:
|
223 |
+
print(f"Error saving metadata locally: {e}")
|
224 |
+
# In case of a file error, we'll still try to save to HF
|
225 |
|
226 |
# In production, also save to Hugging Face
|
227 |
if os.environ.get("ENV", "development") == "production" and HF_USERNAME and HF_TOKEN:
|
download_images.py
CHANGED
@@ -40,10 +40,34 @@ UPLOAD_DIR = Path("static/uploads")
|
|
40 |
METADATA_DIR = Path("static/metadata")
|
41 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
# Create directories if they don't exist
|
44 |
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
45 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
46 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
47 |
# Initialize HfApi
|
48 |
hf_api = HfApi(token=HF_TOKEN)
|
49 |
|
@@ -56,7 +80,7 @@ try:
|
|
56 |
# Download metadata first
|
57 |
print(f"Downloading metadata from {HF_USERNAME}/{DATASET_REPO}")
|
58 |
try:
|
59 |
-
|
60 |
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
61 |
filename=f"{METADATA_PATH}/image_metadata.json",
|
62 |
repo_type="dataset",
|
@@ -64,21 +88,24 @@ try:
|
|
64 |
local_dir=os.path.join(tempfile.gettempdir(), "hf_downloads"),
|
65 |
)
|
66 |
|
67 |
-
print(f"Metadata downloaded to {
|
68 |
-
with open(
|
69 |
metadata = json.load(f)
|
70 |
|
71 |
-
# Save metadata locally
|
72 |
-
|
|
|
73 |
json.dump(metadata, f)
|
74 |
-
print(f"Metadata saved to {
|
75 |
except Exception as e:
|
76 |
print(f"Error downloading metadata: {e}")
|
77 |
print("Creating empty metadata file")
|
78 |
metadata = {}
|
79 |
-
# Initialize metadata file
|
80 |
-
|
|
|
81 |
json.dump({}, f)
|
|
|
82 |
|
83 |
# List all files in the dataset
|
84 |
print("Listing files in the dataset")
|
@@ -123,5 +150,7 @@ try:
|
|
123 |
except Exception as e:
|
124 |
print(f"Error: {e}")
|
125 |
print("Creating empty metadata file")
|
126 |
-
|
|
|
127 |
json.dump({}, f)
|
|
|
|
40 |
METADATA_DIR = Path("static/metadata")
|
41 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
42 |
|
43 |
+
# Alternative metadata location with guaranteed write permissions
|
44 |
+
HOME_DIR = Path(os.environ.get("HOME", "/tmp"))
|
45 |
+
ALT_METADATA_DIR = HOME_DIR / ".image_uploader"
|
46 |
+
ALT_METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
47 |
+
ALT_METADATA_FILE = ALT_METADATA_DIR / "image_metadata.json"
|
48 |
+
|
49 |
# Create directories if they don't exist
|
50 |
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
51 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
52 |
|
53 |
+
|
54 |
+
# Function to get the appropriate metadata file
|
55 |
+
def get_metadata_file():
|
56 |
+
# Try to write to the primary location
|
57 |
+
try:
|
58 |
+
if not METADATA_FILE.exists():
|
59 |
+
with open(METADATA_FILE, "w") as f:
|
60 |
+
json.dump({}, f)
|
61 |
+
# Test write permission
|
62 |
+
if os.access(METADATA_FILE, os.W_OK):
|
63 |
+
return METADATA_FILE
|
64 |
+
raise PermissionError(f"No write permission for {METADATA_FILE}")
|
65 |
+
except (PermissionError, OSError) as e:
|
66 |
+
print(f"Warning: Cannot use {METADATA_FILE}: {e}")
|
67 |
+
print(f"Using alternative location: {ALT_METADATA_FILE}")
|
68 |
+
return ALT_METADATA_FILE
|
69 |
+
|
70 |
+
|
71 |
# Initialize HfApi
|
72 |
hf_api = HfApi(token=HF_TOKEN)
|
73 |
|
|
|
80 |
# Download metadata first
|
81 |
print(f"Downloading metadata from {HF_USERNAME}/{DATASET_REPO}")
|
82 |
try:
|
83 |
+
metadata_file_path = hf_api.hf_hub_download(
|
84 |
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
85 |
filename=f"{METADATA_PATH}/image_metadata.json",
|
86 |
repo_type="dataset",
|
|
|
88 |
local_dir=os.path.join(tempfile.gettempdir(), "hf_downloads"),
|
89 |
)
|
90 |
|
91 |
+
print(f"Metadata downloaded to {metadata_file_path}")
|
92 |
+
with open(metadata_file_path, "r") as f:
|
93 |
metadata = json.load(f)
|
94 |
|
95 |
+
# Save metadata locally to the appropriate file
|
96 |
+
save_path = get_metadata_file()
|
97 |
+
with open(save_path, "w") as f:
|
98 |
json.dump(metadata, f)
|
99 |
+
print(f"Metadata saved to {save_path}")
|
100 |
except Exception as e:
|
101 |
print(f"Error downloading metadata: {e}")
|
102 |
print("Creating empty metadata file")
|
103 |
metadata = {}
|
104 |
+
# Initialize metadata file
|
105 |
+
save_path = get_metadata_file()
|
106 |
+
with open(save_path, "w") as f:
|
107 |
json.dump({}, f)
|
108 |
+
print(f"Created empty metadata file at {save_path}")
|
109 |
|
110 |
# List all files in the dataset
|
111 |
print("Listing files in the dataset")
|
|
|
150 |
except Exception as e:
|
151 |
print(f"Error: {e}")
|
152 |
print("Creating empty metadata file")
|
153 |
+
save_path = get_metadata_file()
|
154 |
+
with open(save_path, "w") as f:
|
155 |
json.dump({}, f)
|
156 |
+
print(f"Created empty metadata file at {save_path}")
|