Spaces:
Sleeping
Sleeping
refactor logic
Browse files- src/gradio_space_ci/webhook.py +34 -77
src/gradio_space_ci/webhook.py
CHANGED
@@ -213,7 +213,7 @@ def recover_after_restart(space_id: str) -> None:
|
|
213 |
if repo_exists(repo_id=ci_space_id, repo_type="space"):
|
214 |
# Found a PR for which the CI space has not been deleted
|
215 |
print(f"Recovery. Found a closed PR with an active CI space: {discussion.url}. Deleting it.")
|
216 |
-
background_pool.submit(delete_ci_space, space_id=space_id, pr_num=discussion.num)
|
217 |
|
218 |
|
219 |
###
|
@@ -258,7 +258,7 @@ async def trigger_ci_on_pr(payload: WebhookPayload, task_queue: BackgroundTasks)
|
|
258 |
and payload.discussion.isPullRequest
|
259 |
and payload.discussion.status in ACTIVE_PR_STATUS
|
260 |
):
|
261 |
-
# A comment, is it
|
262 |
if payload.event.scope == "discussion.comment":
|
263 |
handle_command(space_id=space_id, payload=payload)
|
264 |
# Always sync (in case the space was sleeping or building)
|
@@ -274,11 +274,7 @@ async def trigger_ci_on_pr(payload: WebhookPayload, task_queue: BackgroundTasks)
|
|
274 |
and payload.discussion.isPullRequest
|
275 |
and (payload.discussion.status == "merged" or payload.discussion.status == "closed")
|
276 |
):
|
277 |
-
task_queue.add_task(
|
278 |
-
delete_ci_space,
|
279 |
-
space_id=space_id,
|
280 |
-
pr_num=payload.discussion.num,
|
281 |
-
)
|
282 |
has_task = True
|
283 |
elif (
|
284 |
# Means "some content has been pushed to the Space" (any branch)
|
@@ -390,7 +386,7 @@ def create_ephemeral_space(space_id: str, pr_num: int) -> bool:
|
|
390 |
raise
|
391 |
|
392 |
|
393 |
-
def configure_ephemeral_space(space_id: str, pr_num: int) -> bool:
|
394 |
# Config values
|
395 |
ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
|
396 |
trusted_authors: List[str] = EPHEMERAL_SPACES_CONFIG["trusted_authors"]
|
@@ -401,25 +397,25 @@ def configure_ephemeral_space(space_id: str, pr_num: int) -> bool:
|
|
401 |
|
402 |
# Check if trusted author
|
403 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=pr_num)
|
404 |
-
if details.author
|
405 |
-
|
406 |
-
|
407 |
-
|
408 |
-
|
409 |
-
|
410 |
-
|
411 |
-
|
412 |
-
|
413 |
-
|
414 |
-
|
415 |
-
|
416 |
-
if storage is not None:
|
417 |
-
request_space_storage(ci_space_id, storage)
|
418 |
|
419 |
-
|
|
|
|
|
420 |
|
421 |
|
422 |
-
def delete_ci_space(space_id: str, pr_num: int) -> None:
|
423 |
print(f"New task: delete ephemeral env for {space_id} (PR {pr_num})")
|
424 |
|
425 |
# Delete
|
@@ -431,7 +427,9 @@ def delete_ci_space(space_id: str, pr_num: int) -> None:
|
|
431 |
return
|
432 |
|
433 |
# Notify about deletion
|
434 |
-
|
|
|
|
|
435 |
|
436 |
|
437 |
def notify_pr(
|
@@ -464,57 +462,13 @@ def _get_ci_space_id(space_id: str, pr_num: int) -> str:
|
|
464 |
return f"{space_id}-ci-pr-{pr_num}"
|
465 |
|
466 |
|
467 |
-
def set_config(space_id: str, pr_num: str) -> None:
|
468 |
-
"""a function to set the ephemerial space config"""
|
469 |
-
variables: Dict[str, str] = EPHEMERAL_SPACES_CONFIG["variables"]
|
470 |
-
secrets: Dict[str, str] = EPHEMERAL_SPACES_CONFIG["secrets"]
|
471 |
-
hardware: Optional[SpaceHardware] = EPHEMERAL_SPACES_CONFIG["hardware"]
|
472 |
-
storage: Optional[SpaceHardware] = EPHEMERAL_SPACES_CONFIG["storage"]
|
473 |
-
ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
|
474 |
-
# Configure space
|
475 |
-
for key, value in variables.items():
|
476 |
-
add_space_variable(ci_space_id, key, value)
|
477 |
-
for key, value in secrets.items():
|
478 |
-
add_space_secret(ci_space_id, key, value)
|
479 |
-
|
480 |
-
# Request hardware/storage for space
|
481 |
-
if hardware is not None and hardware != SpaceHardware.CPU_BASIC:
|
482 |
-
request_space_hardware(ci_space_id, hardware, sleep_time=5 * 60) # sleep after 5min on PR Spaces with GPU
|
483 |
-
if storage is not None:
|
484 |
-
request_space_storage(ci_space_id, storage)
|
485 |
-
|
486 |
-
|
487 |
def rebuild_space(space_id: str, pr_num: int) -> None:
|
488 |
"a function to rebuild the ephemeral space without config"
|
489 |
# This is useful to cut down on resource usage and to remove tokens from
|
490 |
# the ephemeral space
|
491 |
-
|
492 |
-
|
493 |
-
|
494 |
-
except RepositoryNotFoundError:
|
495 |
-
pass
|
496 |
-
create_ephemeral_space(space_id=space_id, pr_num=pr_num)
|
497 |
-
# Download space codebase from PR revision
|
498 |
-
snapshot_path = Path(snapshot_download(repo_id=space_id, revision=f"refs/pr/{pr_num}", repo_type="space"))
|
499 |
-
|
500 |
-
# Overwrite README file in cache (/!\)
|
501 |
-
readme_path = snapshot_path / "README.md"
|
502 |
-
card = RepoCard.load(readme_path)
|
503 |
-
setattr(card.data, "synced_sha", snapshot_path.name) # latest sha
|
504 |
-
card.data.title = f"{card.data.title} (ephemeral #{pr_num})"
|
505 |
-
card.save(readme_path)
|
506 |
-
|
507 |
-
# Sync space codebase with PR revision
|
508 |
-
upload_folder(
|
509 |
-
repo_id=ci_space_id,
|
510 |
-
repo_type="space",
|
511 |
-
commit_message=f"Sync CI Space with PR {pr_num}.",
|
512 |
-
folder_path=snapshot_path,
|
513 |
-
delete_patterns="*",
|
514 |
-
)
|
515 |
-
|
516 |
-
# Delete readme file from cache (just in case)
|
517 |
-
readme_path.unlink(missing_ok=True)
|
518 |
|
519 |
|
520 |
def handle_modification(space_id: str, discussion: Any) -> None:
|
@@ -522,10 +476,13 @@ def handle_modification(space_id: str, discussion: Any) -> None:
|
|
522 |
if not repo_exists(ci_space_id):
|
523 |
return
|
524 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=discussion.num)
|
525 |
-
|
526 |
-
|
527 |
-
|
528 |
-
|
|
|
|
|
|
|
529 |
|
530 |
|
531 |
def handle_command(space_id: str, payload: WebhookPayload) -> None:
|
@@ -535,7 +492,7 @@ def handle_command(space_id: str, payload: WebhookPayload) -> None:
|
|
535 |
event_author = details.events[-1]._event["author"]["name"] # username of that event
|
536 |
if event_author in EPHEMERAL_SPACES_CONFIG["trusted_authors"]:
|
537 |
if payload.comment.content == "/trust_pr":
|
538 |
-
|
539 |
notify_pr(space_id=space_id, pr_num=pr_num, action="trusted_pr")
|
540 |
elif payload.comment.content == "/untrust_pr":
|
541 |
rebuild_space(space_id=space_id, pr_num=pr_num)
|
|
|
213 |
if repo_exists(repo_id=ci_space_id, repo_type="space"):
|
214 |
# Found a PR for which the CI space has not been deleted
|
215 |
print(f"Recovery. Found a closed PR with an active CI space: {discussion.url}. Deleting it.")
|
216 |
+
background_pool.submit(delete_ci_space, space_id=space_id, pr_num=discussion.num, notify=True)
|
217 |
|
218 |
|
219 |
###
|
|
|
258 |
and payload.discussion.isPullRequest
|
259 |
and payload.discussion.status in ACTIVE_PR_STATUS
|
260 |
):
|
261 |
+
# A comment, is it a command ?
|
262 |
if payload.event.scope == "discussion.comment":
|
263 |
handle_command(space_id=space_id, payload=payload)
|
264 |
# Always sync (in case the space was sleeping or building)
|
|
|
274 |
and payload.discussion.isPullRequest
|
275 |
and (payload.discussion.status == "merged" or payload.discussion.status == "closed")
|
276 |
):
|
277 |
+
task_queue.add_task(delete_ci_space, space_id=space_id, pr_num=payload.discussion.num, notify=True)
|
|
|
|
|
|
|
|
|
278 |
has_task = True
|
279 |
elif (
|
280 |
# Means "some content has been pushed to the Space" (any branch)
|
|
|
386 |
raise
|
387 |
|
388 |
|
389 |
+
def configure_ephemeral_space(space_id: str, pr_num: int, trusted_pr=False) -> bool:
|
390 |
# Config values
|
391 |
ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
|
392 |
trusted_authors: List[str] = EPHEMERAL_SPACES_CONFIG["trusted_authors"]
|
|
|
397 |
|
398 |
# Check if trusted author
|
399 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=pr_num)
|
400 |
+
if details.author in trusted_authors or trusted_pr is True:
|
401 |
+
# Configure space
|
402 |
+
for key, value in variables.items():
|
403 |
+
add_space_variable(ci_space_id, key, value)
|
404 |
+
for key, value in secrets.items():
|
405 |
+
add_space_secret(ci_space_id, key, value)
|
406 |
+
|
407 |
+
# Request hardware/storage for space
|
408 |
+
if hardware is not None and hardware != SpaceHardware.CPU_BASIC:
|
409 |
+
request_space_hardware(ci_space_id, hardware, sleep_time=5 * 60) # sleep after 5min on PR Spaces with GPU
|
410 |
+
if storage is not None:
|
411 |
+
request_space_storage(ci_space_id, storage)
|
|
|
|
|
412 |
|
413 |
+
return True
|
414 |
+
else:
|
415 |
+
return False
|
416 |
|
417 |
|
418 |
+
def delete_ci_space(space_id: str, pr_num: int, notify: bool = True) -> None:
|
419 |
print(f"New task: delete ephemeral env for {space_id} (PR {pr_num})")
|
420 |
|
421 |
# Delete
|
|
|
427 |
return
|
428 |
|
429 |
# Notify about deletion
|
430 |
+
if notify is True:
|
431 |
+
# This logic is adaped across multiple functions so we will not always notify the pr
|
432 |
+
notify_pr(space_id=space_id, pr_num=pr_num, action="deleted")
|
433 |
|
434 |
|
435 |
def notify_pr(
|
|
|
462 |
return f"{space_id}-ci-pr-{pr_num}"
|
463 |
|
464 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
465 |
def rebuild_space(space_id: str, pr_num: int) -> None:
|
466 |
"a function to rebuild the ephemeral space without config"
|
467 |
# This is useful to cut down on resource usage and to remove tokens from
|
468 |
# the ephemeral space
|
469 |
+
delete_ci_space(space_id=space_id, pr_num=pr_num, notify=False)
|
470 |
+
# create a new synced ephemeral space
|
471 |
+
sync_ci_space(space_id=space_id, pr_num=pr_num)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
472 |
|
473 |
|
474 |
def handle_modification(space_id: str, discussion: Any) -> None:
|
|
|
476 |
if not repo_exists(ci_space_id):
|
477 |
return
|
478 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=discussion.num)
|
479 |
+
# If last commit is not by a trusted author we rebuild it
|
480 |
+
for event in details.events[::-1]:
|
481 |
+
if event.type == "commit":
|
482 |
+
if event._event["author"]["name"] not in EPHEMERAL_SPACES_CONFIG["trusted_authors"]:
|
483 |
+
rebuild_space(space_id=space_id, pr_num=discussion.num)
|
484 |
+
else:
|
485 |
+
return
|
486 |
|
487 |
|
488 |
def handle_command(space_id: str, payload: WebhookPayload) -> None:
|
|
|
492 |
event_author = details.events[-1]._event["author"]["name"] # username of that event
|
493 |
if event_author in EPHEMERAL_SPACES_CONFIG["trusted_authors"]:
|
494 |
if payload.comment.content == "/trust_pr":
|
495 |
+
configure_ephemeral_space(space_id=space_id, pr_num=pr_num, trusted_pr=True)
|
496 |
notify_pr(space_id=space_id, pr_num=pr_num, action="trusted_pr")
|
497 |
elif payload.comment.content == "/untrust_pr":
|
498 |
rebuild_space(space_id=space_id, pr_num=pr_num)
|