Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -1,213 +1,363 @@
|
|
1 |
import requests
|
2 |
import os
|
3 |
import gradio as gr
|
4 |
-
from huggingface_hub import update_repo_visibility,
|
5 |
from slugify import slugify
|
6 |
-
import gradio as gr
|
7 |
import re
|
8 |
import uuid
|
9 |
-
from typing import Optional
|
10 |
import json
|
11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
|
13 |
-
|
14 |
|
15 |
-
def get_json_data(url):
|
16 |
url_split = url.split('/')
|
17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
18 |
try:
|
19 |
-
response = requests.get(api_url)
|
20 |
response.raise_for_status()
|
21 |
return response.json()
|
22 |
except requests.exceptions.RequestException as e:
|
23 |
-
print(f"Error fetching JSON data: {e}")
|
24 |
return None
|
25 |
|
26 |
-
def check_nsfw(json_data, profile):
|
27 |
-
if json_data
|
|
|
28 |
return False
|
29 |
-
|
30 |
-
if
|
|
|
31 |
return True
|
32 |
-
|
33 |
-
|
34 |
-
|
|
|
|
|
35 |
return False
|
36 |
return True
|
37 |
|
38 |
-
def get_prompts_from_image(image_id):
|
39 |
-
print("image_id: ", image_id)
|
40 |
url = f'https://civitai.com/api/trpc/image.getGenerationData?input={{"json":{{"id":{image_id}}}}}'
|
41 |
-
print(url)
|
42 |
-
response = requests.get(url)
|
43 |
-
print(response)
|
44 |
prompt = ""
|
45 |
negative_prompt = ""
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
|
|
|
|
|
|
|
|
54 |
return prompt, negative_prompt
|
55 |
|
56 |
-
def extract_info(json_data):
|
57 |
-
if json_data
|
58 |
-
|
59 |
-
if model_version["baseModel"] in ["SDXL 1.0", "SDXL 0.9", "SD 1.5", "SD 1.4", "SD 2.1", "SD 2.0", "SD 2.0 768", "SD 2.1 768", "SD 3", "Flux.1 D", "Flux.1 S"]:
|
60 |
-
for file in model_version["files"]:
|
61 |
-
print(file)
|
62 |
-
if "primary" in file:
|
63 |
-
# Start by adding the primary file to the list
|
64 |
-
urls_to_download = [{"url": file["downloadUrl"], "filename": file["name"], "type": "weightName"}]
|
65 |
-
|
66 |
-
# Then append all image URLs to the list
|
67 |
-
for image in model_version["images"]:
|
68 |
-
image_id = image["url"].split("/")[-1].split(".")[0]
|
69 |
-
prompt, negative_prompt = get_prompts_from_image(image_id)
|
70 |
-
if image["nsfwLevel"] > 5:
|
71 |
-
pass #ugly before checking the actual logic
|
72 |
-
else:
|
73 |
-
urls_to_download.append({
|
74 |
-
"url": image["url"],
|
75 |
-
"filename": os.path.basename(image["url"]),
|
76 |
-
"type": "imageName",
|
77 |
-
"prompt": prompt, #if "meta" in image and "prompt" in image["meta"] else ""
|
78 |
-
"negative_prompt": negative_prompt
|
79 |
-
})
|
80 |
-
model_mapping = {
|
81 |
-
"SDXL 1.0": "stabilityai/stable-diffusion-xl-base-1.0",
|
82 |
-
"SDXL 0.9": "stabilityai/stable-diffusion-xl-base-1.0",
|
83 |
-
"SD 1.5": "runwayml/stable-diffusion-v1-5",
|
84 |
-
"SD 1.4": "CompVis/stable-diffusion-v1-4",
|
85 |
-
"SD 2.1": "stabilityai/stable-diffusion-2-1-base",
|
86 |
-
"SD 2.0": "stabilityai/stable-diffusion-2-base",
|
87 |
-
"SD 2.1 768": "stabilityai/stable-diffusion-2-1",
|
88 |
-
"SD 2.0 768": "stabilityai/stable-diffusion-2",
|
89 |
-
"SD 3": "stabilityai/stable-diffusion-3-medium-diffusers",
|
90 |
-
"Flux.1 D": "black-forest-labs/FLUX.1-dev",
|
91 |
-
"Flux.1 S": "black-forest-labs/FLUX.1-schnell"
|
92 |
-
}
|
93 |
-
base_model = model_mapping[model_version["baseModel"]]
|
94 |
-
info = {
|
95 |
-
"urls_to_download": urls_to_download,
|
96 |
-
"id": model_version["id"],
|
97 |
-
"baseModel": base_model,
|
98 |
-
"modelId": model_version.get("modelId", ""),
|
99 |
-
"name": json_data["name"],
|
100 |
-
"description": json_data["description"],
|
101 |
-
"trainedWords": model_version["trainedWords"] if "trainedWords" in model_version else [],
|
102 |
-
"creator": json_data["creator"]["username"],
|
103 |
-
"tags": json_data["tags"],
|
104 |
-
"allowNoCredit": json_data["allowNoCredit"],
|
105 |
-
"allowCommercialUse": json_data["allowCommercialUse"],
|
106 |
-
"allowDerivatives": json_data["allowDerivatives"],
|
107 |
-
"allowDifferentLicense": json_data["allowDifferentLicense"]
|
108 |
-
}
|
109 |
-
return info
|
110 |
-
return None
|
111 |
|
112 |
-
|
113 |
-
|
114 |
-
"
|
115 |
-
"
|
116 |
-
"
|
117 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
118 |
}
|
119 |
-
|
120 |
-
|
121 |
-
|
122 |
-
if
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
-
|
127 |
-
|
128 |
-
|
129 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
130 |
headers = {}
|
|
|
131 |
try:
|
132 |
-
|
|
|
|
|
|
|
|
|
|
|
133 |
response.raise_for_status()
|
134 |
-
except requests.exceptions.HTTPError as e:
|
135 |
-
print(e)
|
136 |
-
if response.status_code == 401:
|
137 |
-
headers['Authorization'] = f'Bearer {os.environ["CIVITAI_API"]}'
|
138 |
-
try:
|
139 |
-
response = requests.get(url, headers=headers)
|
140 |
-
response.raise_for_status()
|
141 |
-
except requests.exceptions.RequestException as e:
|
142 |
-
raise gr.Error(f"Error downloading file: {e}")
|
143 |
-
else:
|
144 |
-
raise gr.Error(f"Error downloading file: {e}")
|
145 |
-
except requests.exceptions.RequestException as e:
|
146 |
-
raise gr.Error(f"Error downloading file: {e}")
|
147 |
|
148 |
-
|
149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
150 |
|
151 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
152 |
json_data = get_json_data(url)
|
153 |
if json_data:
|
154 |
if check_nsfw(json_data, profile):
|
155 |
info = extract_info(json_data)
|
156 |
if info:
|
157 |
-
|
158 |
-
|
159 |
-
|
160 |
-
|
161 |
-
return info, downloaded_files
|
162 |
else:
|
163 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
164 |
else:
|
165 |
-
raise gr.Error("This model
|
166 |
else:
|
167 |
-
raise gr.Error("
|
168 |
|
169 |
-
|
170 |
-
|
171 |
original_url = f"https://civitai.com/models/{info['modelId']}"
|
172 |
link_civit_disclaimer = f'([CivitAI]({original_url}))'
|
173 |
non_author_disclaimer = f'This model was originally uploaded on [CivitAI]({original_url}), by [{info["creator"]}](https://civitai.com/user/{info["creator"]}/models). The information below was provided by the author on CivitAI:'
|
174 |
-
|
175 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
176 |
tags = default_tags + civit_tags
|
177 |
-
unpacked_tags = "\n- ".join(tags)
|
178 |
|
179 |
-
trained_words =
|
180 |
formatted_words = ', '.join(f'`{word}`' for word in trained_words)
|
181 |
-
if formatted_words
|
182 |
-
trigger_words_section = f"""## Trigger words
|
183 |
-
You should use {formatted_words} to trigger the image generation.
|
184 |
-
"""
|
185 |
-
else:
|
186 |
-
trigger_words_section = ""
|
187 |
|
188 |
widget_content = ""
|
189 |
-
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
195 |
{negative_prompt_content}
|
196 |
output:
|
197 |
url: >-
|
198 |
-
{
|
199 |
"""
|
200 |
-
|
|
|
201 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
202 |
content = f"""---
|
203 |
license: other
|
204 |
license_name: bespoke-lora-trained-license
|
205 |
-
license_link: https://multimodal.art/civitai-licenses?allowNoCredit={info["allowNoCredit"]}&allowCommercialUse={
|
206 |
tags:
|
207 |
- {unpacked_tags}
|
208 |
-
|
209 |
base_model: {info["baseModel"]}
|
210 |
-
instance_prompt: {
|
211 |
widget:
|
212 |
{widget_content}
|
213 |
---
|
@@ -217,224 +367,388 @@ widget:
|
|
217 |
<Gallery />
|
218 |
|
219 |
{non_author_disclaimer if not is_author else ''}
|
220 |
-
|
221 |
{link_civit_disclaimer if link_civit else ''}
|
222 |
|
223 |
## Model description
|
224 |
-
|
225 |
{info["description"]}
|
226 |
|
227 |
{trigger_words_section}
|
228 |
|
229 |
## Download model
|
230 |
-
|
231 |
Weights for this model are available in Safetensors format.
|
232 |
-
|
233 |
-
[Download](/{user_repo_id}/tree/main) them in the Files & versions tab.
|
234 |
|
235 |
## Use it with the [🧨 diffusers library](https://github.com/huggingface/diffusers)
|
|
|
236 |
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
|
|
242 |
|
243 |
-
pipeline = AutoPipelineForText2Image.from_pretrained('{info["baseModel"]}', torch_dtype={dtype}).to(device)
|
244 |
-
pipeline.load_lora_weights('{user_repo_id}', weight_name='{downloaded_files["weightName"][0]}')
|
245 |
-
image = pipeline('{prompt if prompt else (formatted_words if formatted_words else 'Your custom prompt')}').images[0]
|
246 |
-
```
|
247 |
|
248 |
-
|
249 |
-
|
250 |
-
|
251 |
-
|
252 |
-
|
253 |
-
|
254 |
-
# content += f"\n\n> {prompt}\n"
|
255 |
-
readme_content += content + "\n"
|
256 |
-
with open(f"{folder}/README.md", "w") as file:
|
257 |
-
file.write(readme_content)
|
258 |
-
|
259 |
-
def get_creator(username):
|
260 |
url = f"https://civitai.com/api/trpc/user.getCreator?input=%7B%22json%22%3A%7B%22username%22%3A%22{username}%22%2C%22authed%22%3Atrue%7D%7D"
|
261 |
headers = {
|
262 |
-
"authority": "civitai.com",
|
263 |
-
"
|
264 |
-
"accept-language": "en-BR,en;q=0.9,pt-BR;q=0.8,pt;q=0.7,es-ES;q=0.6,es;q=0.5,de-LI;q=0.4,de;q=0.3,en-GB;q=0.2,en-US;q=0.1,sk;q=0.1",
|
265 |
-
"content-type": "application/json",
|
266 |
-
"cookie": f'{os.environ["COOKIE_INFO"]}',
|
267 |
-
"if-modified-since": "Tue, 22 Aug 2023 07:18:52 GMT",
|
268 |
"referer": f"https://civitai.com/user/{username}/models",
|
269 |
-
"
|
270 |
-
"sec-ch-ua-mobile": "?0",
|
271 |
-
"sec-ch-ua-platform": "macOS",
|
272 |
-
"sec-fetch-dest": "empty",
|
273 |
-
"sec-fetch-mode": "cors",
|
274 |
-
"sec-fetch-site": "same-origin",
|
275 |
-
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
|
276 |
}
|
277 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
278 |
|
279 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
280 |
|
281 |
-
|
282 |
-
data = get_creator(username)
|
283 |
-
links = data.get('result', {}).get('data', {}).get('json', {}).get('links', [])
|
284 |
-
for link in links:
|
285 |
-
url = link.get('url', '')
|
286 |
-
if url.startswith('https://huggingface.co/') or url.startswith('https://www.huggingface.co/'):
|
287 |
-
username = url.split('/')[-1]
|
288 |
-
return username
|
289 |
|
290 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
291 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
292 |
|
293 |
-
|
294 |
-
|
295 |
-
hf_username = extract_huggingface_username(info['creator'])
|
296 |
-
attributes_methods = dir(profile)
|
297 |
|
298 |
-
if
|
299 |
-
return '', gr.update(interactive=True), gr.update(visible=False), gr.update(visible=True)
|
300 |
-
|
301 |
-
if(not hf_username):
|
302 |
-
no_username_text = f'If you are {info["creator"]} on CivitAI, hi! Your CivitAI profile seems to not have information about your Hugging Face account. Please visit <a href="https://civitai.com/user/account" target="_blank">https://civitai.com/user/account</a> and include your 🤗 username there, here\'s mine:<br><img width="60%" src="https://i.imgur.com/hCbo9uL.png" /><br>(if you are not {info["creator"]}, you cannot submit their model at this time)'
|
303 |
-
return no_username_text, gr.update(interactive=False), gr.update(visible=True), gr.update(visible=False)
|
304 |
-
if(profile.username != hf_username):
|
305 |
-
unmatched_username_text = '<h4>Oops, the Hugging Face account in your CivitAI profile seems to be different than the one your are using here. Please visit <a href="https://civitai.com/user/account">https://civitai.com/user/account</a> and update it there to match your Hugging Face account<br><img src="https://i.imgur.com/hCbo9uL.png" /></h4>'
|
306 |
-
return unmatched_username_text, gr.update(interactive=False), gr.update(visible=True), gr.update(visible=False)
|
307 |
-
else:
|
308 |
-
return '', gr.update(interactive=True), gr.update(visible=False), gr.update(visible=True)
|
309 |
|
310 |
-
|
311 |
-
|
312 |
-
|
313 |
-
|
314 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
315 |
|
316 |
-
def
|
317 |
return gr.update(visible=True)
|
318 |
|
319 |
-
def list_civit_models(username):
|
320 |
-
|
|
|
|
|
321 |
json_models_list = []
|
322 |
-
|
323 |
-
|
324 |
-
|
325 |
-
|
326 |
-
|
327 |
-
|
328 |
-
|
329 |
-
|
330 |
-
|
331 |
-
|
332 |
-
|
333 |
-
|
334 |
-
|
335 |
-
|
|
|
|
|
|
|
|
|
|
|
336 |
|
337 |
-
|
338 |
-
|
339 |
-
|
340 |
-
|
341 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
342 |
|
343 |
-
folder = str(uuid.uuid4())
|
344 |
-
os.makedirs(folder, exist_ok=False)
|
345 |
-
gr.Info(f"Starting download of model {url}")
|
346 |
-
info, downloaded_files = process_url(url, profile, folder=folder)
|
347 |
-
username = {profile.username}
|
348 |
-
slug_name = slugify(info["name"])
|
349 |
-
user_repo_id = f"{profile.username}/{slug_name}"
|
350 |
-
create_readme(info, downloaded_files, user_repo_id, link_civit, folder=folder)
|
351 |
try:
|
352 |
-
|
353 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
354 |
upload_folder(
|
355 |
-
folder_path=
|
356 |
-
|
357 |
-
repo_type="model",
|
358 |
-
token=oauth_token.token
|
359 |
)
|
360 |
-
update_repo_visibility(repo_id=user_repo_id, private=False, token=
|
361 |
-
gr.Info(f"Model uploaded!")
|
|
|
|
|
|
|
|
|
362 |
except Exception as e:
|
363 |
-
print(e)
|
364 |
-
raise gr.Error("
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
365 |
|
366 |
-
|
367 |
-
|
368 |
-
|
369 |
-
|
370 |
-
|
371 |
-
|
372 |
-
|
373 |
-
|
374 |
-
|
375 |
-
|
376 |
-
|
377 |
-
|
378 |
-
|
379 |
-
|
380 |
-
|
|
|
|
|
|
|
|
|
|
|
381 |
|
|
|
382 |
css = '''
|
383 |
-
#
|
384 |
-
|
385 |
-
|
386 |
-
}
|
387 |
-
#
|
388 |
-
|
389 |
-
pointer-events:none;
|
390 |
-
}
|
391 |
'''
|
392 |
|
393 |
-
with gr.Blocks(css=css) as demo:
|
|
|
|
|
|
|
|
|
394 |
gr.Markdown('''# Upload your CivitAI LoRA to Hugging Face 🤗
|
395 |
By uploading your LoRAs to Hugging Face you get diffusers compatibility, a free GPU-based Inference Widget, you'll be listed in [LoRA Studio](https://lorastudio.co/models) after a short review, and get the possibility to submit your model to the [LoRA the Explorer](https://huggingface.co/spaces/multimodalart/LoraTheExplorer) ✨
|
396 |
''')
|
397 |
-
|
398 |
-
with gr.
|
399 |
-
|
400 |
-
|
401 |
-
|
402 |
-
|
403 |
-
|
404 |
-
|
405 |
-
|
|
|
|
|
|
|
|
|
|
|
406 |
with gr.Column(visible=False) as enabled_area:
|
407 |
-
|
408 |
-
|
409 |
-
|
410 |
-
|
411 |
-
|
|
|
|
|
|
|
|
|
|
|
412 |
|
413 |
-
|
414 |
-
|
415 |
-
|
416 |
-
|
417 |
-
|
418 |
-
|
419 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
420 |
)
|
421 |
-
|
422 |
-
|
423 |
|
424 |
-
|
425 |
-
try_again_button = gr.Button("I have added my HF profile to my account (it may take 1 minute to refresh)", visible=False)
|
426 |
-
submit_button_civit = gr.Button("Upload model to Hugging Face", interactive=False)
|
427 |
-
output = gr.Markdown(label="Output progress", visible=False)
|
428 |
|
429 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
430 |
|
431 |
-
|
432 |
-
civit_username_to_bulk.
|
433 |
-
|
|
|
|
|
|
|
|
|
434 |
|
435 |
-
|
436 |
-
|
437 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
438 |
|
439 |
-
|
440 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
import requests
|
2 |
import os
|
3 |
import gradio as gr
|
4 |
+
from huggingface_hub import update_repo_visibility, upload_folder, create_repo, upload_file
|
5 |
from slugify import slugify
|
|
|
6 |
import re
|
7 |
import uuid
|
8 |
+
from typing import Optional, Dict, Any, List
|
9 |
import json
|
10 |
+
import shutil # For cleaning up local folders
|
11 |
+
import traceback # For debugging
|
12 |
+
|
13 |
+
TRUSTED_UPLOADERS = [
|
14 |
+
"KappaNeuro", "CiroN2022", "multimodalart", "Norod78", "joachimsallstrom",
|
15 |
+
"blink7630", "e-n-v-y", "DoctorDiffusion", "RalFinger", "artificialguybr"
|
16 |
+
]
|
17 |
|
18 |
+
# --- Helper Functions (CivitAI API, Data Extraction, File Handling) ---
|
19 |
|
20 |
+
def get_json_data(url: str) -> Optional[Dict[str, Any]]:
|
21 |
url_split = url.split('/')
|
22 |
+
if len(url_split) < 5 or not url_split[4].isdigit(): # Check if model ID is present and numeric
|
23 |
+
print(f"Error: Invalid CivitAI URL format or missing model ID: {url}")
|
24 |
+
# Try to extract model ID if it's just the ID
|
25 |
+
if url.isdigit():
|
26 |
+
model_id = url
|
27 |
+
else:
|
28 |
+
return None
|
29 |
+
else:
|
30 |
+
model_id = url_split[4]
|
31 |
+
|
32 |
+
api_url = f"https://civitai.com/api/v1/models/{model_id}"
|
33 |
try:
|
34 |
+
response = requests.get(api_url, timeout=15)
|
35 |
response.raise_for_status()
|
36 |
return response.json()
|
37 |
except requests.exceptions.RequestException as e:
|
38 |
+
print(f"Error fetching JSON data from {api_url}: {e}")
|
39 |
return None
|
40 |
|
41 |
+
def check_nsfw(json_data: Dict[str, Any], profile: Optional[gr.OAuthProfile]) -> bool:
|
42 |
+
if json_data.get("nsfw", False):
|
43 |
+
print(f"Model {json_data.get('id', 'Unknown')} flagged as NSFW at model level.")
|
44 |
return False
|
45 |
+
|
46 |
+
if profile and profile.username in TRUSTED_UPLOADERS:
|
47 |
+
print(f"Trusted uploader {profile.username}, bypassing strict image NSFW check for model {json_data.get('id', 'Unknown')}.")
|
48 |
return True
|
49 |
+
|
50 |
+
for model_version in json_data.get("modelVersions", []):
|
51 |
+
for image_media in model_version.get("images", []): # 'images' can contain videos
|
52 |
+
if image_media.get("nsfwLevel", 0) > 5: # Allow 0-5 (None, Soft, Moderate, Mature, X)
|
53 |
+
print(f"Model {json_data.get('id', 'Unknown')} version {model_version.get('id')} has media with nsfwLevel > 5.")
|
54 |
return False
|
55 |
return True
|
56 |
|
57 |
+
def get_prompts_from_image(image_id: int) -> (str, str):
|
|
|
58 |
url = f'https://civitai.com/api/trpc/image.getGenerationData?input={{"json":{{"id":{image_id}}}}}'
|
|
|
|
|
|
|
59 |
prompt = ""
|
60 |
negative_prompt = ""
|
61 |
+
try:
|
62 |
+
response = requests.get(url, timeout=10)
|
63 |
+
if response.status_code == 200:
|
64 |
+
data = response.json()
|
65 |
+
result = data.get('result', {}).get('data', {}).get('json', {})
|
66 |
+
if result and result.get('meta') is not None:
|
67 |
+
prompt = result['meta'].get('prompt', "")
|
68 |
+
negative_prompt = result['meta'].get('negativePrompt', "")
|
69 |
+
# else:
|
70 |
+
# print(f"Prompt fetch for {image_id}: Status {response.status_code}")
|
71 |
+
except requests.exceptions.RequestException as e:
|
72 |
+
print(f"Error fetching prompt data for image_id {image_id}: {e}")
|
73 |
return prompt, negative_prompt
|
74 |
|
75 |
+
def extract_info(json_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
76 |
+
if json_data.get("type") != "LORA":
|
77 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
78 |
|
79 |
+
model_mapping = {
|
80 |
+
"SDXL 1.0": "stabilityai/stable-diffusion-xl-base-1.0", "SDXL 0.9": "stabilityai/stable-diffusion-xl-base-1.0",
|
81 |
+
"SD 1.5": "runwayml/stable-diffusion-v1-5", "SD 1.4": "CompVis/stable-diffusion-v1-4",
|
82 |
+
"SD 2.1": "stabilityai/stable-diffusion-2-1-base", "SD 2.0": "stabilityai/stable-diffusion-2-base",
|
83 |
+
"SD 2.1 768": "stabilityai/stable-diffusion-2-1", "SD 2.0 768": "stabilityai/stable-diffusion-2",
|
84 |
+
"SD 3": "stabilityai/stable-diffusion-3-medium-diffusers",
|
85 |
+
"SD 3.5": "stabilityai/stable-diffusion-3-medium",
|
86 |
+
"SD 3.5 Large": "stabilityai/stable-diffusion-3-medium", # Adjusted to medium as large might not be public LoRA base
|
87 |
+
"SD 3.5 Medium": "stabilityai/stable-diffusion-3-medium",
|
88 |
+
"SD 3.5 Large Turbo": "stabilityai/stable-diffusion-3-medium-turbo", # Placeholder
|
89 |
+
"Flux.1 D": "black-forest-labs/FLUX.1-dev", "Flux.1 S": "black-forest-labs/FLUX.1-schnell",
|
90 |
+
"LTXV": "Lightricks/LTX-Video-0.9.7-dev",
|
91 |
+
"Hunyuan Video": "hunyuanvideo-community/HunyuanVideo", # Default T2V
|
92 |
+
"Wan Video 1.3B t2v": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
93 |
+
"Wan Video 14B t2v": "Wan-AI/Wan2.1-T2V-14B-Diffusers",
|
94 |
+
"Wan Video 14B i2v 480p": "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
95 |
+
"Wan Video 14B i2v 720p": "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
|
96 |
}
|
97 |
+
|
98 |
+
for model_version in json_data.get("modelVersions", []):
|
99 |
+
civic_base_model_name = model_version.get("baseModel")
|
100 |
+
if civic_base_model_name in model_mapping:
|
101 |
+
base_model_hf_name = model_mapping[civic_base_model_name]
|
102 |
+
|
103 |
+
urls_to_download: List[Dict[str, Any]] = []
|
104 |
+
primary_file_found = False
|
105 |
+
for file_data in model_version.get("files", []):
|
106 |
+
if file_data.get("primary") and file_data.get("type") == "Model":
|
107 |
+
urls_to_download.append({
|
108 |
+
"url": file_data["downloadUrl"],
|
109 |
+
"filename": os.path.basename(file_data["name"]),
|
110 |
+
"type": "weightName", "is_video": False
|
111 |
+
})
|
112 |
+
primary_file_found = True
|
113 |
+
break
|
114 |
+
|
115 |
+
if not primary_file_found: continue
|
116 |
+
|
117 |
+
for media_data in model_version.get("images", []): # CivitAI uses 'images' for both images and videos
|
118 |
+
if media_data.get("nsfwLevel", 0) > 5: continue
|
119 |
+
|
120 |
+
media_url_parts = media_data["url"].split("/")
|
121 |
+
if not media_url_parts: continue
|
122 |
+
|
123 |
+
filename_part = media_url_parts[-1]
|
124 |
+
# Robustly extract ID: try to get it before the first dot or before query params
|
125 |
+
id_candidate = filename_part.split(".")[0].split("?")[0]
|
126 |
+
|
127 |
+
prompt, negative_prompt = "", ""
|
128 |
+
if media_data.get("hasMeta", False) and media_data.get("type") == "image": # Prompts mainly for images
|
129 |
+
if id_candidate.isdigit():
|
130 |
+
try:
|
131 |
+
prompt, negative_prompt = get_prompts_from_image(int(id_candidate))
|
132 |
+
except ValueError:
|
133 |
+
print(f"Warning: Non-integer ID '{id_candidate}' for prompt fetching.")
|
134 |
+
except Exception as e:
|
135 |
+
print(f"Warning: Prompt fetch failed for ID {id_candidate}: {e}")
|
136 |
+
|
137 |
+
is_video_file = media_data.get("type") == "video"
|
138 |
+
media_type_key = "videoName" if is_video_file else "imageName"
|
139 |
+
|
140 |
+
urls_to_download.append({
|
141 |
+
"url": media_data["url"], "filename": os.path.basename(filename_part),
|
142 |
+
"type": media_type_key, "prompt": prompt, "negative_prompt": negative_prompt,
|
143 |
+
"is_video": is_video_file
|
144 |
+
})
|
145 |
+
|
146 |
+
# Ensure 'allowCommercialUse' is processed correctly
|
147 |
+
allow_commercial_use = json_data.get("allowCommercialUse", "Sell") # Default
|
148 |
+
if isinstance(allow_commercial_use, list):
|
149 |
+
allow_commercial_use = allow_commercial_use[0] if allow_commercial_use else "Sell"
|
150 |
+
elif not isinstance(allow_commercial_use, str): # If boolean or other, convert to expected string
|
151 |
+
allow_commercial_use = "Sell" if allow_commercial_use else "None"
|
152 |
+
|
153 |
+
|
154 |
+
info_dict = {
|
155 |
+
"urls_to_download": urls_to_download, "id": model_version.get("id"),
|
156 |
+
"baseModel": base_model_hf_name, "modelId": model_version.get("modelId", json_data.get("id")),
|
157 |
+
"name": json_data.get("name", "Untitled LoRA"),
|
158 |
+
"description": json_data.get("description", "No description provided."),
|
159 |
+
"trainedWords": model_version.get("trainedWords", []),
|
160 |
+
"creator": json_data.get("creator", {}).get("username", "Unknown Creator"),
|
161 |
+
"tags": json_data.get("tags", []),
|
162 |
+
"allowNoCredit": json_data.get("allowNoCredit", True),
|
163 |
+
"allowCommercialUse": allow_commercial_use,
|
164 |
+
"allowDerivatives": json_data.get("allowDerivatives", True),
|
165 |
+
"allowDifferentLicense": json_data.get("allowDifferentLicense", True)
|
166 |
+
}
|
167 |
+
return info_dict
|
168 |
+
return None
|
169 |
+
|
170 |
+
def download_file_from_url(url: str, filename: str, folder: str = "."):
|
171 |
headers = {}
|
172 |
+
local_filepath = os.path.join(folder, filename)
|
173 |
try:
|
174 |
+
# Add a User-Agent to mimic a browser, as some CDNs might block default requests User-Agent
|
175 |
+
headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
176 |
+
if "CIVITAI_API_TOKEN" in os.environ and os.environ["CIVITAI_API_TOKEN"]: # Check for token existence and value
|
177 |
+
headers['Authorization'] = f'Bearer {os.environ["CIVITAI_API_TOKEN"]}'
|
178 |
+
|
179 |
+
response = requests.get(url, headers=headers, stream=True, timeout=120) # Increased timeout
|
180 |
response.raise_for_status()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
181 |
|
182 |
+
with open(local_filepath, 'wb') as f:
|
183 |
+
for chunk in response.iter_content(chunk_size=8192):
|
184 |
+
f.write(chunk)
|
185 |
+
# print(f"Successfully downloaded {filename} to {folder}")
|
186 |
+
|
187 |
+
except requests.exceptions.HTTPError as e_http:
|
188 |
+
# If 401/403 and no token was used, it's a clear auth issue.
|
189 |
+
# If token was used and still 401/403, token might be invalid or insufficient.
|
190 |
+
if e_http.response.status_code in [401, 403] and not headers.get('Authorization'):
|
191 |
+
print(f"Authorization error downloading {url}. Consider setting CIVITAI_API_TOKEN for restricted files.")
|
192 |
+
raise gr.Error(f"HTTP Error downloading {filename}: {e_http.response.status_code} {e_http.response.reason}. URL: {url}")
|
193 |
+
except requests.exceptions.RequestException as e_req:
|
194 |
+
raise gr.Error(f"Request Error downloading {filename}: {e_req}. URL: {url}")
|
195 |
+
|
196 |
+
|
197 |
+
def download_files(info: Dict[str, Any], folder: str = ".") -> Dict[str, List[Any]]:
|
198 |
+
downloaded_media_items: List[Dict[str, Any]] = []
|
199 |
+
downloaded_weights: List[str] = []
|
200 |
|
201 |
+
for item in info["urls_to_download"]:
|
202 |
+
filename_to_save = item["filename"]
|
203 |
+
|
204 |
+
# Sanitize filename (though os.path.basename usually handles paths well)
|
205 |
+
filename_to_save = re.sub(r'[<>:"/\\|?*]', '_', filename_to_save) # Basic sanitization
|
206 |
+
if not filename_to_save: # Handle case where filename becomes empty
|
207 |
+
filename_to_save = f"downloaded_file_{uuid.uuid4().hex[:8]}" + os.path.splitext(item["url"])[1]
|
208 |
+
|
209 |
+
|
210 |
+
gr.Info(f"Downloading {filename_to_save}...")
|
211 |
+
download_file_from_url(item["url"], filename_to_save, folder)
|
212 |
+
|
213 |
+
if item["type"] == "weightName":
|
214 |
+
downloaded_weights.append(filename_to_save)
|
215 |
+
elif item["type"] in ["imageName", "videoName"]:
|
216 |
+
prompt_clean = re.sub(r'<.*?>', '', item.get("prompt", ""))
|
217 |
+
negative_prompt_clean = re.sub(r'<.*?>', '', item.get("negative_prompt", ""))
|
218 |
+
downloaded_media_items.append({
|
219 |
+
"filename": filename_to_save, "prompt": prompt_clean,
|
220 |
+
"negative_prompt": negative_prompt_clean, "is_video": item.get("is_video", False)
|
221 |
+
})
|
222 |
+
|
223 |
+
return {"media_items": downloaded_media_items, "weightName": downloaded_weights}
|
224 |
+
|
225 |
+
def process_url(url: str, profile: Optional[gr.OAuthProfile], do_download: bool = True, folder: str = ".") -> (Optional[Dict[str, Any]], Optional[Dict[str, List[Any]]]):
|
226 |
json_data = get_json_data(url)
|
227 |
if json_data:
|
228 |
if check_nsfw(json_data, profile):
|
229 |
info = extract_info(json_data)
|
230 |
if info:
|
231 |
+
downloaded_files_dict = None
|
232 |
+
if do_download:
|
233 |
+
downloaded_files_dict = download_files(info, folder)
|
234 |
+
return info, downloaded_files_dict
|
|
|
235 |
else:
|
236 |
+
model_type = json_data.get("type", "Unknown type")
|
237 |
+
base_models_in_json = [mv.get("baseModel", "Unknown base") for mv in json_data.get("modelVersions", [])]
|
238 |
+
error_message = f"This LoRA is not supported. Details:\n"
|
239 |
+
error_message += f"- Model Type: {model_type} (expected LORA)\n"
|
240 |
+
if base_models_in_json:
|
241 |
+
error_message += f"- Detected Base Models in CivitAI: {', '.join(list(set(base_models_in_json)))}\n"
|
242 |
+
error_message += "Ensure it's a LORA for a supported base (SD, SDXL, Pony, Flux, LTXV, Hunyuan, Wan) and has primary files."
|
243 |
+
raise gr.Error(error_message)
|
244 |
else:
|
245 |
+
raise gr.Error("This model is flagged as NSFW by CivitAI or its media exceeds the allowed NSFW level (max level 5).")
|
246 |
else:
|
247 |
+
raise gr.Error("Could not fetch CivitAI API data. Check URL or model ID. Example: https://civitai.com/models/12345 or just 12345")
|
248 |
|
249 |
+
# --- README Creation ---
|
250 |
+
def create_readme(info: Dict[str, Any], downloaded_files: Dict[str, List[Any]], user_repo_id: str, link_civit: bool = False, is_author: bool = True, folder: str = "."):
|
251 |
original_url = f"https://civitai.com/models/{info['modelId']}"
|
252 |
link_civit_disclaimer = f'([CivitAI]({original_url}))'
|
253 |
non_author_disclaimer = f'This model was originally uploaded on [CivitAI]({original_url}), by [{info["creator"]}](https://civitai.com/user/{info["creator"]}/models). The information below was provided by the author on CivitAI:'
|
254 |
+
|
255 |
+
is_video_model = False
|
256 |
+
video_base_models_hf = [
|
257 |
+
"Lightricks/LTX-Video-0.9.7-dev", "hunyuanvideo-community/HunyuanVideo",
|
258 |
+
"hunyuanvideo-community/HunyuanVideo-I2V", "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
259 |
+
"Wan-AI/Wan2.1-T2V-14B-Diffusers", "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
260 |
+
"Wan-AI/Wan2.1-I2V-14B-720P-Diffusers"
|
261 |
+
]
|
262 |
+
if info["baseModel"] in video_base_models_hf: is_video_model = True
|
263 |
+
is_i2v_model = "i2v" in info["baseModel"].lower()
|
264 |
+
|
265 |
+
default_tags = ["lora", "diffusers", "migrated"]
|
266 |
+
if is_video_model:
|
267 |
+
default_tags.append("video")
|
268 |
+
default_tags.append("image-to-video" if is_i2v_model else "text-to-video")
|
269 |
+
default_tags.append("template:video-lora") # Added a template tag for video
|
270 |
+
else:
|
271 |
+
default_tags.extend(["text-to-image", "stable-diffusion", "template:sd-lora"])
|
272 |
+
|
273 |
+
civit_tags = [t.replace(":", "").strip() for t in info.get("tags", []) if t.replace(":", "").strip() and t.replace(":", "").strip() not in default_tags]
|
274 |
tags = default_tags + civit_tags
|
275 |
+
unpacked_tags = "\n- ".join(sorted(list(set(tags))))
|
276 |
|
277 |
+
trained_words = [word for word in info.get('trainedWords', []) if word]
|
278 |
formatted_words = ', '.join(f'`{word}`' for word in trained_words)
|
279 |
+
trigger_words_section = f"## Trigger words\nYou should use {formatted_words} to trigger the generation." if formatted_words else ""
|
|
|
|
|
|
|
|
|
|
|
280 |
|
281 |
widget_content = ""
|
282 |
+
media_items_for_widget = downloaded_files.get("media_items", [])
|
283 |
+
if not media_items_for_widget:
|
284 |
+
widget_content = "# No example media available for widget.\n"
|
285 |
+
else:
|
286 |
+
for media_item in media_items_for_widget[:5]: # Limit to 5 examples for widget
|
287 |
+
prompt = media_item["prompt"]
|
288 |
+
negative_prompt = media_item["negative_prompt"]
|
289 |
+
filename = media_item["filename"]
|
290 |
+
|
291 |
+
escaped_prompt = prompt.replace("'", "''").replace("\n", " ") # Escape and remove newlines
|
292 |
+
negative_prompt_content = f"""parameters:
|
293 |
+
negative_prompt: '{negative_prompt.replace("'", "''").replace("\n", " ")}'""" if negative_prompt else ""
|
294 |
+
widget_content += f"""- text: '{escaped_prompt if escaped_prompt else ' ' }'
|
295 |
{negative_prompt_content}
|
296 |
output:
|
297 |
url: >-
|
298 |
+
{filename}
|
299 |
"""
|
300 |
+
flux_models_bf16 = ["black-forest-labs/FLUX.1-dev", "black-forest-labs/FLUX.1-schnell"]
|
301 |
+
dtype = "torch.bfloat16" if info["baseModel"] in flux_models_bf16 else "torch.float16"
|
302 |
|
303 |
+
pipeline_import = "AutoPipelineForText2Image"
|
304 |
+
pipeline_call_example = f"image = pipeline('{formatted_words if formatted_words else 'Your custom prompt'}').images[0]"
|
305 |
+
example_prompt_for_pipeline = formatted_words if formatted_words else 'Your custom prompt'
|
306 |
+
if media_items_for_widget and media_items_for_widget[0]["prompt"]:
|
307 |
+
example_prompt_for_pipeline = media_items_for_widget[0]["prompt"]
|
308 |
+
pipeline_call_example = f"image = pipeline('{example_prompt_for_pipeline.replace ciclo '','' ')').images[0]"
|
309 |
+
|
310 |
+
|
311 |
+
if is_video_model:
|
312 |
+
pipeline_import = "DiffusionPipeline"
|
313 |
+
video_prompt_example = example_prompt_for_pipeline
|
314 |
+
|
315 |
+
pipeline_call_example = f"# Example prompt for video generation\nprompt = \"{video_prompt_example.replace ciclico '','' ')}\"\n"
|
316 |
+
pipeline_call_example += "# Adjust parameters like num_frames, num_inference_steps, height, width as needed for the specific pipeline.\n"
|
317 |
+
pipeline_call_example += "# video_frames = pipeline(prompt, num_frames=16, guidance_scale=7.5, num_inference_steps=25).frames # Example parameters"
|
318 |
+
if "LTX-Video" in info["baseModel"]:
|
319 |
+
pipeline_call_example += "\n# LTX-Video uses a specific setup. Check its model card on Hugging Face."
|
320 |
+
elif "HunyuanVideo" in info["baseModel"]:
|
321 |
+
pipeline_call_example += "\n# HunyuanVideo often uses custom pipeline scripts or specific classes (e.g., HunyuanDiTPipeline). Check its HF model card."
|
322 |
+
elif "Wan-AI" in info["baseModel"]:
|
323 |
+
pipeline_call_example += "\n# Wan-AI models (e.g., WanVideoTextToVideoPipeline) require specific pipeline classes. Check model card for usage."
|
324 |
+
|
325 |
+
weight_name = (downloaded_files["weightName"][0] if downloaded_files.get("weightName")
|
326 |
+
else "your_lora_weights.safetensors")
|
327 |
+
|
328 |
+
diffusers_code_block = f"""```py
|
329 |
+
from diffusers import {pipeline_import}
|
330 |
+
import torch
|
331 |
+
|
332 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
333 |
+
|
334 |
+
# Note: The pipeline class '{pipeline_import}' is a general suggestion.
|
335 |
+
# For specific video models (LTX, Hunyuan, Wan), you will likely need a dedicated pipeline class
|
336 |
+
# (e.g., TextToVideoSDPipeline, HunyuanDiTPipeline, WanVideoTextToVideoPipeline, etc.).
|
337 |
+
# Please refer to the documentation of the base model '{info["baseModel"]}' on Hugging Face for precise usage.
|
338 |
+
pipeline = {pipeline_import}.from_pretrained('{info["baseModel"]}', torch_dtype={dtype})
|
339 |
+
pipeline.to(device)
|
340 |
+
|
341 |
+
# Load LoRA weights
|
342 |
+
pipeline.load_lora_weights('{user_repo_id}', weight_name='{weight_name}')
|
343 |
+
|
344 |
+
# For some pipelines, you might need to fuse LoRA layers:
|
345 |
+
# pipeline.fuse_lora() # or pipeline.unfuse_lora()
|
346 |
+
|
347 |
+
# Example generation call (adjust parameters as needed for the specific pipeline)
|
348 |
+
{pipeline_call_example}
|
349 |
+
```"""
|
350 |
+
|
351 |
+
commercial_use_val = info["allowCommercialUse"] # Already processed in extract_info
|
352 |
+
|
353 |
content = f"""---
|
354 |
license: other
|
355 |
license_name: bespoke-lora-trained-license
|
356 |
+
license_link: https://multimodal.art/civitai-licenses?allowNoCredit={info["allowNoCredit"]}&allowCommercialUse={commercial_use_val}&allowDerivatives={info["allowDerivatives"]}&allowDifferentLicense={info["allowDifferentLicense"]}
|
357 |
tags:
|
358 |
- {unpacked_tags}
|
|
|
359 |
base_model: {info["baseModel"]}
|
360 |
+
instance_prompt: {trained_words[0] if trained_words else ''}
|
361 |
widget:
|
362 |
{widget_content}
|
363 |
---
|
|
|
367 |
<Gallery />
|
368 |
|
369 |
{non_author_disclaimer if not is_author else ''}
|
|
|
370 |
{link_civit_disclaimer if link_civit else ''}
|
371 |
|
372 |
## Model description
|
|
|
373 |
{info["description"]}
|
374 |
|
375 |
{trigger_words_section}
|
376 |
|
377 |
## Download model
|
|
|
378 |
Weights for this model are available in Safetensors format.
|
379 |
+
[Download](/{user_repo_id}/tree/main/{weight_name}) the LoRA in the Files & versions tab.
|
|
|
380 |
|
381 |
## Use it with the [🧨 diffusers library](https://github.com/huggingface/diffusers)
|
382 |
+
{diffusers_code_block}
|
383 |
|
384 |
+
For more details, including weighting, merging and fusing LoRAs, check the [documentation on loading LoRAs in diffusers](https://huggingface.co/docs/diffusers/main/en/using-diffusers/loading_adapters).
|
385 |
+
"""
|
386 |
+
readme_path = os.path.join(folder, "README.md")
|
387 |
+
with open(readme_path, "w", encoding="utf-8") as file:
|
388 |
+
file.write(content)
|
389 |
+
# print(f"README.md created at {readme_path}")
|
390 |
|
|
|
|
|
|
|
|
|
391 |
|
392 |
+
# --- Hugging Face Profile / Authorship ---
|
393 |
+
def get_creator(username: str) -> Dict:
|
394 |
+
if "COOKIE_INFO" not in os.environ or not os.environ["COOKIE_INFO"]:
|
395 |
+
print("Warning: COOKIE_INFO env var not set. Cannot fetch CivitAI creator's HF username.")
|
396 |
+
return {"result": {"data": {"json": {"links": []}}}}
|
397 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
398 |
url = f"https://civitai.com/api/trpc/user.getCreator?input=%7B%22json%22%3A%7B%22username%22%3A%22{username}%22%2C%22authed%22%3Atrue%7D%7D"
|
399 |
headers = {
|
400 |
+
"authority": "civitai.com", "accept": "*/*", "accept-language": "en-US,en;q=0.9",
|
401 |
+
"content-type": "application/json", "cookie": os.environ["COOKIE_INFO"],
|
|
|
|
|
|
|
|
|
402 |
"referer": f"https://civitai.com/user/{username}/models",
|
403 |
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36"
|
|
|
|
|
|
|
|
|
|
|
|
|
404 |
}
|
405 |
+
try:
|
406 |
+
response = requests.get(url, headers=headers, timeout=10)
|
407 |
+
response.raise_for_status()
|
408 |
+
return response.json()
|
409 |
+
except requests.RequestException as e:
|
410 |
+
print(f"Error fetching CivitAI creator data for {username}: {e}")
|
411 |
+
return {"result": {"data": {"json": {"links": []}}}}
|
412 |
|
413 |
+
def extract_huggingface_username(civitai_username: str) -> Optional[str]:
|
414 |
+
data = get_creator(civitai_username)
|
415 |
+
try:
|
416 |
+
links = data.get('result', {}).get('data', {}).get('json', {}).get('links', [])
|
417 |
+
if not isinstance(links, list): return None
|
418 |
+
for link in links:
|
419 |
+
if not isinstance(link, dict): continue
|
420 |
+
url = link.get('url', '')
|
421 |
+
if isinstance(url, str) and \
|
422 |
+
(url.startswith('https://huggingface.co/') or url.startswith('https://www.huggingface.co/')):
|
423 |
+
hf_username = url.split('/')[-1].split('?')[0].split('#')[0]
|
424 |
+
if hf_username: return hf_username
|
425 |
+
except Exception as e:
|
426 |
+
print(f"Error parsing CivitAI creator data for HF username: {e}")
|
427 |
+
return None
|
428 |
|
429 |
+
# --- Gradio UI Logic Functions ---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
430 |
|
431 |
+
def check_civit_link(profile_state: Optional[gr.OAuthProfile], url_input: str):
|
432 |
+
url_input = url_input.strip()
|
433 |
+
if not url_input:
|
434 |
+
return "", gr.update(interactive=False, visible=False), gr.update(visible=False), gr.update(visible=False)
|
435 |
+
|
436 |
+
if not profile_state:
|
437 |
+
return "Please log in with Hugging Face first.", gr.update(interactive=False, visible=False), gr.update(visible=False), gr.update(visible=False)
|
438 |
|
439 |
+
try:
|
440 |
+
info, _ = process_url(url_input, profile_state, do_download=False)
|
441 |
+
if not info:
|
442 |
+
return "Could not process this CivitAI URL. Model might be unsupported.", gr.update(interactive=False), gr.update(visible=True), gr.update(visible=False)
|
443 |
+
except gr.Error as e:
|
444 |
+
return str(e), gr.update(interactive=False), gr.update(visible=True), gr.update(visible=False)
|
445 |
+
except Exception as e:
|
446 |
+
print(f"Unexpected error in check_civit_link: {e}\n{traceback.format_exc()}")
|
447 |
+
return f"An unexpected error occurred: {str(e)}", gr.update(interactive=False), gr.update(visible=True), gr.update(visible=False)
|
448 |
|
449 |
+
civitai_creator_username = info['creator']
|
450 |
+
hf_username_on_civitai = extract_huggingface_username(civitai_creator_username)
|
|
|
|
|
451 |
|
452 |
+
if profile_state.username in TRUSTED_UPLOADERS:
|
453 |
+
return f'Welcome, trusted uploader {profile_state.username}! You can upload this model by "{civitai_creator_username}".', gr.update(interactive=True, visible=True), gr.update(visible=False), gr.update(visible=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
454 |
|
455 |
+
if not hf_username_on_civitai:
|
456 |
+
no_username_text = (
|
457 |
+
f'If you are "{civitai_creator_username}" on CivitAI, hi! Your CivitAI profile does not seem to have a Hugging Face username linked. '
|
458 |
+
f'Please visit <a href="https://civitai.com/user/account" target="_blank">your CivitAI account settings</a> and add your 🤗 username ({profile_state.username}). '
|
459 |
+
f'Example: <br/><img width="60%" src="https://i.imgur.com/hCbo9uL.png" alt="CivitAI profile settings example"/><br/>'
|
460 |
+
f'(If you are not "{civitai_creator_username}", you cannot submit their model at this time.)'
|
461 |
+
)
|
462 |
+
return no_username_text, gr.update(interactive=False, visible=False), gr.update(visible=True), gr.update(visible=False) # Hide upload, show try_again
|
463 |
+
|
464 |
+
if profile_state.username.lower() != hf_username_on_civitai.lower():
|
465 |
+
unmatched_username_text = (
|
466 |
+
f'The Hugging Face username on "{civitai_creator_username}"\'s CivitAI profile ("{hf_username_on_civitai}") '
|
467 |
+
f'does not match your logged-in Hugging Face account ("{profile_state.username}"). '
|
468 |
+
f'Please update it on <a href="https://civitai.com/user/account" target="_blank">CivitAI</a> or log in to Hugging Face as "{hf_username_on_civitai}".<br/>'
|
469 |
+
f'<img src="https://i.imgur.com/hCbo9uL.png" alt="CivitAI profile settings example"/>'
|
470 |
+
)
|
471 |
+
return unmatched_username_text, gr.update(interactive=False, visible=False), gr.update(visible=True), gr.update(visible=False) # Hide upload, show try_again
|
472 |
+
|
473 |
+
return f'Authorship verified for "{civitai_creator_username}" (🤗 {profile_state.username}). Ready to upload!', gr.update(interactive=True, visible=True), gr.update(visible=False), gr.update(visible=True) # Show upload, hide try_again
|
474 |
+
|
475 |
+
def handle_auth_change(profile: Optional[gr.OAuthProfile]):
|
476 |
+
# This function is called by demo.load when auth state changes
|
477 |
+
# It updates the visibility of UI areas and clears inputs.
|
478 |
+
if profile: # Logged in
|
479 |
+
return gr.update(visible=False), gr.update(visible=True), "", gr.update(value=""), gr.update(interactive=False, visible=False), gr.update(visible=False)
|
480 |
+
else: # Logged out
|
481 |
+
return gr.update(visible=True), gr.update(visible=False), "", gr.update(value=""), gr.update(interactive=False, visible=False), gr.update(visible=False)
|
482 |
|
483 |
+
def show_output_area():
|
484 |
return gr.update(visible=True)
|
485 |
|
486 |
+
def list_civit_models(username: str) -> str:
|
487 |
+
if not username.strip(): return ""
|
488 |
+
|
489 |
+
url = f"https://civitai.com/api/v1/models?username={username}&limit=100&sort=Newest"
|
490 |
json_models_list = []
|
491 |
+
page_count, max_pages = 0, 5 # Limit pages
|
492 |
+
|
493 |
+
gr.Info(f"Fetching LoRAs for CivitAI user: {username}...")
|
494 |
+
while url and page_count < max_pages:
|
495 |
+
try:
|
496 |
+
response = requests.get(url, timeout=10)
|
497 |
+
response.raise_for_status()
|
498 |
+
data = response.json()
|
499 |
+
|
500 |
+
current_items = data.get('items', [])
|
501 |
+
# Filter for LORAs and ensure they have a name for slugify
|
502 |
+
json_models_list.extend(item for item in current_items if item.get("type") == "LORA" and item.get("name"))
|
503 |
+
|
504 |
+
metadata = data.get('metadata', {})
|
505 |
+
url = metadata.get('nextPage', None)
|
506 |
+
page_count += 1
|
507 |
+
except requests.RequestException as e:
|
508 |
+
gr.Warning(f"Failed to fetch page {page_count + 1} for {username}: {e}")
|
509 |
+
break
|
510 |
|
511 |
+
if not json_models_list:
|
512 |
+
gr.Info(f"No suitable LoRA models found for {username} or failed to fetch.")
|
513 |
+
return ""
|
514 |
+
|
515 |
+
urls_text = "\n".join(
|
516 |
+
f'https://civitai.com/models/{model["id"]}/{slugify(model["name"])}'
|
517 |
+
for model in json_models_list
|
518 |
+
)
|
519 |
+
gr.Info(f"Found {len(json_models_list)} LoRA models for {username}.")
|
520 |
+
return urls_text.strip()
|
521 |
+
|
522 |
+
# --- Main Upload Functions ---
|
523 |
+
def upload_civit_to_hf(profile: Optional[gr.OAuthProfile], oauth_token_obj: gr.OAuthToken, url: str, link_civit_checkbox_val: bool):
|
524 |
+
if not profile or not profile.username:
|
525 |
+
raise gr.Error("User profile not available. Please log in.")
|
526 |
+
if not oauth_token_obj or not oauth_token_obj.token:
|
527 |
+
raise gr.Error("Hugging Face token not available. Please log in again.")
|
528 |
+
|
529 |
+
hf_auth_token = oauth_token_obj.token
|
530 |
+
|
531 |
+
folder_uuid = str(uuid.uuid4())
|
532 |
+
# Create a unique subfolder in a general 'temp_uploads' directory
|
533 |
+
base_temp_dir = "temp_uploads"
|
534 |
+
os.makedirs(base_temp_dir, exist_ok=True)
|
535 |
+
folder_path = os.path.join(base_temp_dir, folder_uuid)
|
536 |
+
os.makedirs(folder_path, exist_ok=True)
|
537 |
+
|
538 |
+
gr.Info(f"Starting processing of model {url}")
|
539 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
540 |
try:
|
541 |
+
info, downloaded_data = process_url(url, profile, do_download=True, folder=folder_path)
|
542 |
+
if not info or not downloaded_data:
|
543 |
+
raise gr.Error("Failed to process URL or download files after initial checks.")
|
544 |
+
|
545 |
+
slug_name = slugify(info["name"])
|
546 |
+
user_repo_id = f"{profile.username}/{slug_name}"
|
547 |
+
|
548 |
+
is_author = False # Default
|
549 |
+
hf_username_on_civitai = extract_huggingface_username(info["creator"])
|
550 |
+
if profile.username in TRUSTED_UPLOADERS or \
|
551 |
+
(hf_username_on_civitai and profile.username.lower() == hf_username_on_civitai.lower()):
|
552 |
+
is_author = True # Or at least authorized to upload as/for them
|
553 |
+
|
554 |
+
create_readme(info, downloaded_data, user_repo_id, link_civit_checkbox_val, is_author=is_author, folder=folder_path)
|
555 |
+
|
556 |
+
repo_url_huggingface = f"https://huggingface.co/{user_repo_id}"
|
557 |
+
|
558 |
+
gr.Info(f"Creating/updating repository {user_repo_id} on Hugging Face...")
|
559 |
+
create_repo(repo_id=user_repo_id, private=True, exist_ok=True, token=hf_auth_token)
|
560 |
+
|
561 |
+
gr.Info(f"Starting upload to {repo_url_huggingface}...")
|
562 |
upload_folder(
|
563 |
+
folder_path=folder_path, repo_id=user_repo_id, repo_type="model",
|
564 |
+
token=hf_auth_token, commit_message=f"Upload LoRA: {info['name']} from CivitAI ID {info['modelId']}"
|
|
|
|
|
565 |
)
|
566 |
+
update_repo_visibility(repo_id=user_repo_id, private=False, token=hf_auth_token)
|
567 |
+
gr.Info(f"Model uploaded successfully!")
|
568 |
+
|
569 |
+
return f'''# Model uploaded to 🤗!
|
570 |
+
## Access it here [{user_repo_id}]({repo_url_huggingface}) '''
|
571 |
+
|
572 |
except Exception as e:
|
573 |
+
print(f"Error during Hugging Face repo operations for {url}: {e}\n{traceback.format_exc()}")
|
574 |
+
raise gr.Error(f"Upload failed for {url}: {str(e)}. Token might be expired. Try re-logging or check server logs.")
|
575 |
+
finally:
|
576 |
+
# Cleanup local folder
|
577 |
+
try:
|
578 |
+
if os.path.exists(folder_path):
|
579 |
+
shutil.rmtree(folder_path)
|
580 |
+
# print(f"Cleaned up temporary folder: {folder_path}")
|
581 |
+
except Exception as e_clean:
|
582 |
+
print(f"Error cleaning up folder {folder_path}: {e_clean}")
|
583 |
+
|
584 |
+
|
585 |
+
def bulk_upload(profile: Optional[gr.OAuthProfile], oauth_token_obj: gr.OAuthToken, urls_text: str, link_civit_checkbox_val: bool):
|
586 |
+
if not profile or not oauth_token_obj or not oauth_token_obj.token:
|
587 |
+
raise gr.Error("Authentication missing for bulk upload. Please log in.")
|
588 |
+
|
589 |
+
urls = [url.strip() for url in urls_text.splitlines() if url.strip()]
|
590 |
+
if not urls:
|
591 |
+
return "No URLs provided for bulk upload."
|
592 |
|
593 |
+
upload_results = []
|
594 |
+
total_urls = len(urls)
|
595 |
+
gr.Info(f"Starting bulk upload for {total_urls} models.")
|
596 |
+
|
597 |
+
for i, url in enumerate(urls):
|
598 |
+
gr.Info(f"Processing model {i+1}/{total_urls}: {url}")
|
599 |
+
try:
|
600 |
+
# Each call to upload_civit_to_hf will handle its own folder creation/cleanup
|
601 |
+
result_message = upload_civit_to_hf(profile, oauth_token_obj, url, link_civit_checkbox_val)
|
602 |
+
upload_results.append(result_message)
|
603 |
+
gr.Info(f"Successfully processed {url}")
|
604 |
+
except gr.Error as ge:
|
605 |
+
gr.Warning(f"Skipping model {url} due to error: {str(ge)}")
|
606 |
+
upload_results.append(f"Failed to upload {url}: {str(ge)}")
|
607 |
+
except Exception as e:
|
608 |
+
gr.Warning(f"Unhandled error uploading model {url}: {str(e)}")
|
609 |
+
upload_results.append(f"Failed to upload {url}: Unhandled exception - {str(e)}")
|
610 |
+
print(f"Unhandled exception during bulk upload for {url}: {e}\n{traceback.format_exc()}")
|
611 |
+
|
612 |
+
return "\n\n---\n\n".join(upload_results) if upload_results else "No URLs were processed or all failed."
|
613 |
|
614 |
+
# --- Gradio UI Definition ---
|
615 |
css = '''
|
616 |
+
#login_button_area { margin-bottom: 10px; }
|
617 |
+
#disabled_upload_area { opacity: 0.6; pointer-events: none; }
|
618 |
+
.gr-html ul { list-style-type: disc; margin-left: 20px; }
|
619 |
+
.gr-html ol { list-style-type: decimal; margin-left: 20px; }
|
620 |
+
.gr-html a { color: #007bff; text-decoration: underline; }
|
621 |
+
.gr-html img { max-width: 100%; height: auto; margin-top: 5px; margin-bottom: 5px; border: 1px solid #ddd; }
|
|
|
|
|
622 |
'''
|
623 |
|
624 |
+
with gr.Blocks(css=css, title="CivitAI to Hugging Face LoRA Uploader") as demo:
|
625 |
+
# States to hold authentication info globally within the Blocks context
|
626 |
+
auth_profile_state = gr.State()
|
627 |
+
# oauth_token_state = gr.State() # Token string will be passed directly from gr.OAuthToken
|
628 |
+
|
629 |
gr.Markdown('''# Upload your CivitAI LoRA to Hugging Face 🤗
|
630 |
By uploading your LoRAs to Hugging Face you get diffusers compatibility, a free GPU-based Inference Widget, you'll be listed in [LoRA Studio](https://lorastudio.co/models) after a short review, and get the possibility to submit your model to the [LoRA the Explorer](https://huggingface.co/spaces/multimodalart/LoraTheExplorer) ✨
|
631 |
''')
|
632 |
+
|
633 |
+
with gr.Row(elem_id="login_button_area"):
|
634 |
+
login_button = gr.LoginButton() # Default uses HF OAuth
|
635 |
+
|
636 |
+
# This column is visible when the user is NOT logged in
|
637 |
+
with gr.Column(visible=True, elem_id="disabled_upload_area") as disabled_area:
|
638 |
+
gr.HTML("<h3>Please log in with Hugging Face to enable uploads.</h3>")
|
639 |
+
gr.Textbox(
|
640 |
+
placeholder="e.g., https://civitai.com/models/12345/my-lora or just 12345",
|
641 |
+
label="CivitAI Model URL or ID (Log in to enable)",
|
642 |
+
interactive=False
|
643 |
+
)
|
644 |
+
|
645 |
+
# This column is visible when the user IS logged in
|
646 |
with gr.Column(visible=False) as enabled_area:
|
647 |
+
gr.HTML("<h3 style='color:green;'>Logged in! You can now upload models.</h3>")
|
648 |
+
|
649 |
+
with gr.Tabs():
|
650 |
+
with gr.TabItem("Single Model Upload"):
|
651 |
+
submit_source_civit_enabled = gr.Textbox(
|
652 |
+
placeholder="e.g., https://civitai.com/models/12345/my-lora or just 12345",
|
653 |
+
label="CivitAI Model URL or ID",
|
654 |
+
info="Enter the full URL or just the numeric ID of the CivitAI LoRA model page.",
|
655 |
+
)
|
656 |
+
instructions_html = gr.HTML(elem_id="instructions_area")
|
657 |
|
658 |
+
try_again_button = gr.Button("I've updated my CivitAI profile (Re-check Authorship)", visible=False)
|
659 |
+
|
660 |
+
link_civit_checkbox_single = gr.Checkbox(label="Add a link back to CivitAI in the README?", value=True, visible=True)
|
661 |
+
submit_button_single_model = gr.Button("Upload This Model to Hugging Face", interactive=False, visible=False, variant="primary")
|
662 |
+
|
663 |
+
with gr.TabItem("Bulk Upload"):
|
664 |
+
civit_username_to_bulk = gr.Textbox(
|
665 |
+
label="Your CivitAI Username (Optional)",
|
666 |
+
info="Enter your CivitAI username to auto-populate the list below with your LoRAs (up to 50 newest)."
|
667 |
+
)
|
668 |
+
submit_bulk_civit_urls = gr.Textbox(
|
669 |
+
label="CivitAI Model URLs or IDs (One per line)",
|
670 |
+
info="Paste multiple CivitAI model page URLs or just IDs here, one on each line.",
|
671 |
+
lines=8,
|
672 |
)
|
673 |
+
link_civit_checkbox_bulk = gr.Checkbox(label="Add a link back to CivitAI in READMEs?", value=True)
|
674 |
+
bulk_upload_button = gr.Button("Start Bulk Upload", variant="primary")
|
675 |
|
676 |
+
output_markdown_area = gr.Markdown(label="Upload Progress & Results", visible=False)
|
|
|
|
|
|
|
677 |
|
678 |
+
# --- Event Handlers Wiring ---
|
679 |
+
|
680 |
+
# Handle login/logout and initial load
|
681 |
+
# login_button.login() or logout() implicitly triggers demo.load()
|
682 |
+
# The .load event is triggered when the Gradio app starts or when login/logout happens.
|
683 |
+
# It receives profile and token from the gr.LoginButton's state.
|
684 |
+
# Inputs to handle_auth_change must match how gr.LoginButton provides them.
|
685 |
+
# LoginButton provides profile (OAuthProfile) and token (OAuthToken)
|
686 |
+
# These are implicitly passed to the function called by demo.load if it's the only .load.
|
687 |
+
# Using gr.State() for auth_profile_state.
|
688 |
+
|
689 |
+
# This demo.load will be triggered by login/logout from gr.LoginButton
|
690 |
+
# and also on initial page load.
|
691 |
+
demo.load(
|
692 |
+
fn=handle_auth_change,
|
693 |
+
inputs=[auth_profile_state], # Pass the state which will be updated by login
|
694 |
+
outputs=[disabled_area, enabled_area, instructions_html, submit_source_civit_enabled, submit_button_single_model, try_again_button],
|
695 |
+
api_name=False, queue=False
|
696 |
+
).then(
|
697 |
+
# After login/logout, update the auth_profile_state
|
698 |
+
# This is a bit of a workaround to get profile into a state for other functions
|
699 |
+
lambda profile: profile, # Identity function
|
700 |
+
inputs=[gr.Variable()], # This will receive the profile from LoginButton
|
701 |
+
outputs=[auth_profile_state],
|
702 |
+
api_name=False, queue=False
|
703 |
+
)
|
704 |
+
|
705 |
+
# When CivitAI URL changes (in the enabled area)
|
706 |
+
submit_source_civit_enabled.change(
|
707 |
+
fn=check_civit_link,
|
708 |
+
inputs=[auth_profile_state, submit_source_civit_enabled],
|
709 |
+
outputs=[instructions_html, submit_button_single_model, try_again_button, submit_button_single_model],
|
710 |
+
api_name=False
|
711 |
+
)
|
712 |
+
|
713 |
+
# When "Try Again" button is clicked
|
714 |
+
try_again_button.click(
|
715 |
+
fn=check_civit_link,
|
716 |
+
inputs=[auth_profile_state, submit_source_civit_enabled],
|
717 |
+
outputs=[instructions_html, submit_button_single_model, try_again_button, submit_button_single_model],
|
718 |
+
api_name=False
|
719 |
+
)
|
720 |
|
721 |
+
# When CivitAI username for bulk input changes
|
722 |
+
civit_username_to_bulk.submit( # Use .submit for when user presses Enter or blurs
|
723 |
+
fn=list_civit_models,
|
724 |
+
inputs=[civit_username_to_bulk],
|
725 |
+
outputs=[submit_bulk_civit_urls],
|
726 |
+
api_name=False
|
727 |
+
)
|
728 |
|
729 |
+
# Single model upload button
|
730 |
+
submit_button_single_model.click(
|
731 |
+
fn=show_output_area, inputs=[], outputs=[output_markdown_area], api_name=False
|
732 |
+
).then(
|
733 |
+
fn=upload_civit_to_hf,
|
734 |
+
inputs=[auth_profile_state, gr.OAuthToken(scopes=["write_repository","read_repository"]), submit_source_civit_enabled, link_civit_checkbox_single],
|
735 |
+
outputs=[output_markdown_area],
|
736 |
+
api_name="upload_single_model"
|
737 |
+
)
|
738 |
|
739 |
+
# Bulk model upload button
|
740 |
+
bulk_upload_button.click(
|
741 |
+
fn=show_output_area, inputs=[], outputs=[output_markdown_area], api_name=False
|
742 |
+
).then(
|
743 |
+
fn=bulk_upload,
|
744 |
+
inputs=[auth_profile_state, gr.OAuthToken(scopes=["write_repository","read_repository"]), submit_bulk_civit_urls, link_civit_checkbox_bulk],
|
745 |
+
outputs=[output_markdown_area],
|
746 |
+
api_name="upload_bulk_models"
|
747 |
+
)
|
748 |
+
|
749 |
+
demo.queue(default_concurrency_limit=3, max_size=10) # Adjusted concurrency
|
750 |
+
if __name__ == "__main__":
|
751 |
+
# For local testing, you might need to set COOKIE_INFO and CIVITAI_API_TOKEN
|
752 |
+
# os.environ["COOKIE_INFO"] = "your_civitai_cookie_string_here"
|
753 |
+
# os.environ["CIVITAI_API_TOKEN"] = "your_civitai_api_token_here_if_needed"
|
754 |
+
demo.launch(debug=True, share=os.environ.get("GRADIO_SHARE") == "true")
|