vumichien commited on
Commit
387d45c
·
1 Parent(s): 16b8492

change permission

Browse files
Files changed (3) hide show
  1. Dockerfile +10 -2
  2. app.py +76 -30
  3. download_images.py +38 -9
Dockerfile CHANGED
@@ -10,7 +10,7 @@ RUN apt-get update && \
10
 
11
  # Create a non-root user to run the application
12
  RUN useradd -m appuser && \
13
- mkdir -p /home/appuser/app /home/appuser/.cache && \
14
  chown -R appuser:appuser /home/appuser
15
 
16
  # Copy requirements first to leverage Docker cache
@@ -22,7 +22,9 @@ COPY . .
22
 
23
  # Create upload directory with proper permissions
24
  RUN mkdir -p static/uploads static/metadata && \
25
- chmod -R 777 static
 
 
26
 
27
  # Set environment variables for Hugging Face
28
  ENV PYTHONUNBUFFERED=1
@@ -47,6 +49,12 @@ RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true \
47
  echo "Hugging Face credentials not set, skipping image download."; \
48
  fi
49
 
 
 
 
 
 
 
50
  # Expose port for Hugging Face Spaces (uses port 7860)
51
  EXPOSE 7860
52
 
 
10
 
11
  # Create a non-root user to run the application
12
  RUN useradd -m appuser && \
13
+ mkdir -p /home/appuser/app /home/appuser/.cache /home/appuser/.image_uploader && \
14
  chown -R appuser:appuser /home/appuser
15
 
16
  # Copy requirements first to leverage Docker cache
 
22
 
23
  # Create upload directory with proper permissions
24
  RUN mkdir -p static/uploads static/metadata && \
25
+ chmod -R 777 static && \
26
+ mkdir -p /tmp/.image_uploader && \
27
+ chmod -R 777 /tmp/.image_uploader
28
 
29
  # Set environment variables for Hugging Face
30
  ENV PYTHONUNBUFFERED=1
 
49
  echo "Hugging Face credentials not set, skipping image download."; \
50
  fi
51
 
52
+ # Set proper ownership for all files
53
+ RUN chown -R appuser:appuser /app
54
+
55
+ # Switch to non-root user
56
+ USER appuser
57
+
58
  # Expose port for Hugging Face Spaces (uses port 7860)
59
  EXPOSE 7860
60
 
app.py CHANGED
@@ -39,10 +39,22 @@ METADATA_DIR = Path("static/metadata")
39
  METADATA_DIR.mkdir(parents=True, exist_ok=True)
40
  METADATA_FILE = METADATA_DIR / "image_metadata.json"
41
 
 
 
 
 
 
 
42
  # Initialize metadata file if it doesn't exist
43
- if not METADATA_FILE.exists():
44
- with open(METADATA_FILE, "w") as f:
45
- json.dump({}, f)
 
 
 
 
 
 
46
 
47
  # Mount static directory
48
  app.mount("/static", StaticFiles(directory="static"), name="static")
@@ -138,44 +150,78 @@ def verify_auth(request: Request):
138
  )
139
  return True
140
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
  def get_image_metadata():
142
  """Get all image metadata including hashtags from local storage and sync with HF if needed."""
143
- if METADATA_FILE.exists():
144
- with open(METADATA_FILE, "r") as f:
145
- metadata = json.load(f)
146
-
147
- # In production, sync metadata to Hugging Face if it exists locally but not on HF
148
- if (
149
- os.environ.get("ENV", "development") == "production"
150
- and HF_USERNAME
151
- and HF_TOKEN
152
- ):
153
- try:
154
- # Only upload if there are changes (we'd need to implement a proper change tracking mechanism)
155
- # For now, we'll upload every time to ensure consistency
156
- metadata_str = json.dumps(metadata)
157
- hf_api.upload_file(
158
- path_or_fileobj=io.BytesIO(metadata_str.encode()),
159
- path_in_repo=f"{METADATA_PATH}/image_metadata.json",
160
- repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
161
- repo_type="dataset",
162
- token=HF_TOKEN,
163
- )
164
- except Exception as e:
165
- print(f"Error syncing metadata to Hugging Face: {e}")
166
 
167
- return metadata
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
 
169
  # If metadata file doesn't exist locally, create it
170
- with open(METADATA_FILE, "w") as f:
171
  json.dump({}, f)
172
  return {}
173
 
174
  def save_image_metadata(metadata):
175
  """Save image metadata to the local JSON file and sync with HF."""
 
 
176
  # Always save locally first
177
- with open(METADATA_FILE, "w") as f:
178
- json.dump(metadata, f)
 
 
 
 
 
179
 
180
  # In production, also save to Hugging Face
181
  if os.environ.get("ENV", "development") == "production" and HF_USERNAME and HF_TOKEN:
 
39
  METADATA_DIR.mkdir(parents=True, exist_ok=True)
40
  METADATA_FILE = METADATA_DIR / "image_metadata.json"
41
 
42
+ # Alternative metadata location with guaranteed write permissions
43
+ HOME_DIR = Path(os.environ.get("HOME", "/tmp"))
44
+ ALT_METADATA_DIR = HOME_DIR / ".image_uploader"
45
+ ALT_METADATA_DIR.mkdir(parents=True, exist_ok=True)
46
+ ALT_METADATA_FILE = ALT_METADATA_DIR / "image_metadata.json"
47
+
48
  # Initialize metadata file if it doesn't exist
49
+ if not METADATA_FILE.exists() and not ALT_METADATA_FILE.exists():
50
+ try:
51
+ with open(METADATA_FILE, "w") as f:
52
+ json.dump({}, f)
53
+ print(f"Initialized metadata file at {METADATA_FILE}")
54
+ except PermissionError:
55
+ with open(ALT_METADATA_FILE, "w") as f:
56
+ json.dump({}, f)
57
+ print(f"Initialized metadata file at alternative location: {ALT_METADATA_FILE}")
58
 
59
  # Mount static directory
60
  app.mount("/static", StaticFiles(directory="static"), name="static")
 
150
  )
151
  return True
152
 
153
+
154
+ def get_metadata_file():
155
+ """Get the appropriate metadata file based on write permissions."""
156
+ # Try to write to the primary location
157
+ try:
158
+ if not METADATA_FILE.exists():
159
+ with open(METADATA_FILE, "w") as f:
160
+ json.dump({}, f)
161
+ # Test write permission
162
+ os.access(METADATA_FILE, os.W_OK)
163
+ return METADATA_FILE
164
+ except (PermissionError, OSError):
165
+ print(
166
+ f"Warning: Cannot write to {METADATA_FILE}, using alternative location: {ALT_METADATA_FILE}"
167
+ )
168
+ if not ALT_METADATA_FILE.exists():
169
+ with open(ALT_METADATA_FILE, "w") as f:
170
+ json.dump({}, f)
171
+ return ALT_METADATA_FILE
172
+
173
+
174
  def get_image_metadata():
175
  """Get all image metadata including hashtags from local storage and sync with HF if needed."""
176
+ metadata_file = get_metadata_file()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
 
178
+ if metadata_file.exists():
179
+ try:
180
+ with open(metadata_file, "r") as f:
181
+ metadata = json.load(f)
182
+
183
+ # In production, sync metadata to Hugging Face if it exists locally but not on HF
184
+ if (
185
+ os.environ.get("ENV", "development") == "production"
186
+ and HF_USERNAME
187
+ and HF_TOKEN
188
+ ):
189
+ try:
190
+ # Only upload if there are changes (we'd need to implement a proper change tracking mechanism)
191
+ # For now, we'll upload every time to ensure consistency
192
+ metadata_str = json.dumps(metadata)
193
+ hf_api.upload_file(
194
+ path_or_fileobj=io.BytesIO(metadata_str.encode()),
195
+ path_in_repo=f"{METADATA_PATH}/image_metadata.json",
196
+ repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
197
+ repo_type="dataset",
198
+ token=HF_TOKEN,
199
+ )
200
+ except Exception as e:
201
+ print(f"Error syncing metadata to Hugging Face: {e}")
202
+
203
+ return metadata
204
+ except Exception as e:
205
+ print(f"Error reading metadata file: {e}")
206
+ return {}
207
 
208
  # If metadata file doesn't exist locally, create it
209
+ with open(metadata_file, "w") as f:
210
  json.dump({}, f)
211
  return {}
212
 
213
  def save_image_metadata(metadata):
214
  """Save image metadata to the local JSON file and sync with HF."""
215
+ metadata_file = get_metadata_file()
216
+
217
  # Always save locally first
218
+ try:
219
+ with open(metadata_file, "w") as f:
220
+ json.dump(metadata, f)
221
+ print(f"Metadata saved to {metadata_file}")
222
+ except Exception as e:
223
+ print(f"Error saving metadata locally: {e}")
224
+ # In case of a file error, we'll still try to save to HF
225
 
226
  # In production, also save to Hugging Face
227
  if os.environ.get("ENV", "development") == "production" and HF_USERNAME and HF_TOKEN:
download_images.py CHANGED
@@ -40,10 +40,34 @@ UPLOAD_DIR = Path("static/uploads")
40
  METADATA_DIR = Path("static/metadata")
41
  METADATA_FILE = METADATA_DIR / "image_metadata.json"
42
 
 
 
 
 
 
 
43
  # Create directories if they don't exist
44
  UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
45
  METADATA_DIR.mkdir(parents=True, exist_ok=True)
46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  # Initialize HfApi
48
  hf_api = HfApi(token=HF_TOKEN)
49
 
@@ -56,7 +80,7 @@ try:
56
  # Download metadata first
57
  print(f"Downloading metadata from {HF_USERNAME}/{DATASET_REPO}")
58
  try:
59
- metadata_file = hf_api.hf_hub_download(
60
  repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
61
  filename=f"{METADATA_PATH}/image_metadata.json",
62
  repo_type="dataset",
@@ -64,21 +88,24 @@ try:
64
  local_dir=os.path.join(tempfile.gettempdir(), "hf_downloads"),
65
  )
66
 
67
- print(f"Metadata downloaded to {metadata_file}")
68
- with open(metadata_file, "r") as f:
69
  metadata = json.load(f)
70
 
71
- # Save metadata locally
72
- with open(METADATA_FILE, "w") as f:
 
73
  json.dump(metadata, f)
74
- print(f"Metadata saved to {METADATA_FILE}")
75
  except Exception as e:
76
  print(f"Error downloading metadata: {e}")
77
  print("Creating empty metadata file")
78
  metadata = {}
79
- # Initialize metadata file if it doesn't exist
80
- with open(METADATA_FILE, "w") as f:
 
81
  json.dump({}, f)
 
82
 
83
  # List all files in the dataset
84
  print("Listing files in the dataset")
@@ -123,5 +150,7 @@ try:
123
  except Exception as e:
124
  print(f"Error: {e}")
125
  print("Creating empty metadata file")
126
- with open(METADATA_FILE, "w") as f:
 
127
  json.dump({}, f)
 
 
40
  METADATA_DIR = Path("static/metadata")
41
  METADATA_FILE = METADATA_DIR / "image_metadata.json"
42
 
43
+ # Alternative metadata location with guaranteed write permissions
44
+ HOME_DIR = Path(os.environ.get("HOME", "/tmp"))
45
+ ALT_METADATA_DIR = HOME_DIR / ".image_uploader"
46
+ ALT_METADATA_DIR.mkdir(parents=True, exist_ok=True)
47
+ ALT_METADATA_FILE = ALT_METADATA_DIR / "image_metadata.json"
48
+
49
  # Create directories if they don't exist
50
  UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
51
  METADATA_DIR.mkdir(parents=True, exist_ok=True)
52
 
53
+
54
+ # Function to get the appropriate metadata file
55
+ def get_metadata_file():
56
+ # Try to write to the primary location
57
+ try:
58
+ if not METADATA_FILE.exists():
59
+ with open(METADATA_FILE, "w") as f:
60
+ json.dump({}, f)
61
+ # Test write permission
62
+ if os.access(METADATA_FILE, os.W_OK):
63
+ return METADATA_FILE
64
+ raise PermissionError(f"No write permission for {METADATA_FILE}")
65
+ except (PermissionError, OSError) as e:
66
+ print(f"Warning: Cannot use {METADATA_FILE}: {e}")
67
+ print(f"Using alternative location: {ALT_METADATA_FILE}")
68
+ return ALT_METADATA_FILE
69
+
70
+
71
  # Initialize HfApi
72
  hf_api = HfApi(token=HF_TOKEN)
73
 
 
80
  # Download metadata first
81
  print(f"Downloading metadata from {HF_USERNAME}/{DATASET_REPO}")
82
  try:
83
+ metadata_file_path = hf_api.hf_hub_download(
84
  repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
85
  filename=f"{METADATA_PATH}/image_metadata.json",
86
  repo_type="dataset",
 
88
  local_dir=os.path.join(tempfile.gettempdir(), "hf_downloads"),
89
  )
90
 
91
+ print(f"Metadata downloaded to {metadata_file_path}")
92
+ with open(metadata_file_path, "r") as f:
93
  metadata = json.load(f)
94
 
95
+ # Save metadata locally to the appropriate file
96
+ save_path = get_metadata_file()
97
+ with open(save_path, "w") as f:
98
  json.dump(metadata, f)
99
+ print(f"Metadata saved to {save_path}")
100
  except Exception as e:
101
  print(f"Error downloading metadata: {e}")
102
  print("Creating empty metadata file")
103
  metadata = {}
104
+ # Initialize metadata file
105
+ save_path = get_metadata_file()
106
+ with open(save_path, "w") as f:
107
  json.dump({}, f)
108
+ print(f"Created empty metadata file at {save_path}")
109
 
110
  # List all files in the dataset
111
  print("Listing files in the dataset")
 
150
  except Exception as e:
151
  print(f"Error: {e}")
152
  print("Creating empty metadata file")
153
+ save_path = get_metadata_file()
154
+ with open(save_path, "w") as f:
155
  json.dump({}, f)
156
+ print(f"Created empty metadata file at {save_path}")