Spaces:
Running
on
Zero
Running
on
Zero
Commit
·
7a6c881
1
Parent(s):
80ec937
Add keep-alive scripts and environment configuration for Tranception Space
Browse files- .claude/settings.local.json +17 -0
- .env.example +13 -0
- .github/workflows/keep_space_alive.yml +44 -0
- app.py +203 -17
- app_wrapper.py +62 -0
- app_zero_gpu_fixed.py +498 -0
- healthcheck.py +56 -0
- keep_alive_cron.sh +35 -0
- manual_keep_alive.py +47 -0
- monitor_space.py +156 -0
- setup_cron.sh +19 -0
- zero_gpu_monitor.py +92 -0
.claude/settings.local.json
ADDED
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"permissions": {
|
3 |
+
"allow": [
|
4 |
+
"Bash(find:*)",
|
5 |
+
"Bash(git add:*)",
|
6 |
+
"Bash(git commit:*)",
|
7 |
+
"Bash(git push:*)",
|
8 |
+
"Bash(grep:*)",
|
9 |
+
"Bash(git reset:*)",
|
10 |
+
"Bash(git rm:*)",
|
11 |
+
"Bash(python test:*)",
|
12 |
+
"Bash(rm:*)",
|
13 |
+
"Bash(chmod:*)"
|
14 |
+
],
|
15 |
+
"deny": []
|
16 |
+
}
|
17 |
+
}
|
.env.example
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Environment variables for Tranception app
|
2 |
+
|
3 |
+
# Set to 'true' to disable Zero GPU support (useful for debugging or when Zero GPU is unstable)
|
4 |
+
DISABLE_ZERO_GPU=false
|
5 |
+
|
6 |
+
# Set to 'true' to enable debug logging
|
7 |
+
DEBUG_MODE=false
|
8 |
+
|
9 |
+
# Maximum inference batch size (reduce for CPU mode)
|
10 |
+
MAX_BATCH_SIZE=50
|
11 |
+
|
12 |
+
# Timeout for Zero GPU operations (in seconds)
|
13 |
+
ZERO_GPU_TIMEOUT=300
|
.github/workflows/keep_space_alive.yml
ADDED
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
name: Keep Hugging Face Space Alive
|
2 |
+
|
3 |
+
on:
|
4 |
+
schedule:
|
5 |
+
# Run every 30 minutes
|
6 |
+
- cron: '*/30 * * * *'
|
7 |
+
workflow_dispatch: # Allow manual trigger
|
8 |
+
|
9 |
+
jobs:
|
10 |
+
keep-alive:
|
11 |
+
runs-on: ubuntu-latest
|
12 |
+
|
13 |
+
steps:
|
14 |
+
- name: Ping Tranception Space
|
15 |
+
run: |
|
16 |
+
SPACE_URL="https://huggingface.co/spaces/MoraxCheng/Transeption_iGEM_BASISCHINA_2025"
|
17 |
+
|
18 |
+
# Try to access the Space
|
19 |
+
response=$(curl -s -o /dev/null -w "%{http_code}" "$SPACE_URL" --max-time 30)
|
20 |
+
echo "Space returned HTTP $response"
|
21 |
+
|
22 |
+
if [ "$response" != "200" ]; then
|
23 |
+
echo "Space might be sleeping, sending wake-up request..."
|
24 |
+
|
25 |
+
# Send a minimal prediction request
|
26 |
+
curl -s -X POST "$SPACE_URL/api/predict" \
|
27 |
+
-H "Content-Type: application/json" \
|
28 |
+
-d '{
|
29 |
+
"fn_index": 0,
|
30 |
+
"data": ["MSKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFICTTGKLPVPWPTLVTTLSYGVQCFSRYPDHMKQHDFFKSAMPEGYVQERTIFFKDDGNYKTRAEVKFEGDTLVNRIELKGIDFKEDGNILGHKLEYNYNSHNVYIMADKQKNGIKVNFKIRHNIEDGSVQLADHYQQNTPIGDGPVLLPDNHYLSTQSALSKDPNEKRDHMVLLEFVTAAGITHGMDELYK",
|
31 |
+
1, 10, "Small", false, 10
|
32 |
+
]
|
33 |
+
}' \
|
34 |
+
--max-time 120 > /dev/null || true
|
35 |
+
|
36 |
+
echo "Wake-up request sent"
|
37 |
+
else
|
38 |
+
echo "Space is active!"
|
39 |
+
fi
|
40 |
+
|
41 |
+
- name: Log Status
|
42 |
+
if: always()
|
43 |
+
run: |
|
44 |
+
echo "Keep-alive job completed at $(date)"
|
app.py
CHANGED
@@ -17,10 +17,22 @@ import zipfile
|
|
17 |
import shutil
|
18 |
import uuid
|
19 |
import gc
|
|
|
|
|
|
|
|
|
20 |
|
21 |
# Check if Zero GPU should be disabled via environment variable
|
22 |
DISABLE_ZERO_GPU = os.environ.get('DISABLE_ZERO_GPU', 'false').lower() == 'true'
|
23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
# Create a mock spaces module for fallback
|
25 |
class MockSpaces:
|
26 |
"""Mock spaces module for when Zero GPU is not available"""
|
@@ -52,6 +64,79 @@ else:
|
|
52 |
spaces = MockSpaces()
|
53 |
print(f"Warning: Error with spaces module: {e}. Running without Zero GPU support.")
|
54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
55 |
# Add current directory to path
|
56 |
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
57 |
|
@@ -259,6 +344,9 @@ def get_mutated_protein(sequence,mutant):
|
|
259 |
return ''.join(mutated_sequence)
|
260 |
|
261 |
def score_and_create_matrix_all_singles_impl(sequence,mutation_range_start=None,mutation_range_end=None,model_type="Large",scoring_mirror=False,batch_size_inference=20,max_number_positions_per_heatmap=50,num_workers=0,AA_vocab=AA_vocab):
|
|
|
|
|
|
|
262 |
# Clean up old files periodically
|
263 |
cleanup_old_files()
|
264 |
|
@@ -295,6 +383,11 @@ def score_and_create_matrix_all_singles_impl(sequence,mutation_range_start=None,
|
|
295 |
|
296 |
# Device selection - Zero GPU will provide CUDA when decorated with @spaces.GPU
|
297 |
print(f"GPU Available: {torch.cuda.is_available()}")
|
|
|
|
|
|
|
|
|
|
|
298 |
if torch.cuda.is_available():
|
299 |
device = torch.device("cuda")
|
300 |
model = model.to(device)
|
@@ -308,7 +401,9 @@ def score_and_create_matrix_all_singles_impl(sequence,mutation_range_start=None,
|
|
308 |
device = torch.device("cpu")
|
309 |
model = model.to(device)
|
310 |
print("Inference will take place on CPU")
|
311 |
-
|
|
|
|
|
312 |
# Reduce batch size for CPU inference
|
313 |
batch_size_inference = min(batch_size_inference, 10)
|
314 |
|
@@ -366,15 +461,10 @@ def score_and_create_matrix_all_singles_impl(sequence,mutation_range_start=None,
|
|
366 |
torch.cuda.empty_cache()
|
367 |
|
368 |
# Apply Zero GPU decorator - will use real decorator if available, mock otherwise
|
369 |
-
|
370 |
score_and_create_matrix_all_singles = spaces.GPU(duration=300)(score_and_create_matrix_all_singles_impl)
|
371 |
-
|
372 |
-
|
373 |
-
try:
|
374 |
-
score_and_create_matrix_all_singles = spaces.GPU()(score_and_create_matrix_all_singles_impl)
|
375 |
-
except Exception as e2:
|
376 |
-
print(f"Warning: Failed to apply GPU decorator: {e2}")
|
377 |
-
score_and_create_matrix_all_singles = score_and_create_matrix_all_singles_impl
|
378 |
|
379 |
def extract_sequence(protein_id, taxon, sequence):
|
380 |
return sequence
|
@@ -385,6 +475,19 @@ def clear_inputs(protein_sequence_input,mutation_range_start,mutation_range_end)
|
|
385 |
mutation_range_end = None
|
386 |
return protein_sequence_input,mutation_range_start,mutation_range_end
|
387 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
388 |
# Create Gradio app
|
389 |
tranception_design = gr.Blocks()
|
390 |
|
@@ -393,7 +496,42 @@ with tranception_design:
|
|
393 |
gr.Markdown("## 🧬 BASIS-China iGEM Team 2025 - Protein Engineering Platform")
|
394 |
gr.Markdown("### Welcome to BASIS-China's implementation of Tranception on Hugging Face Spaces!")
|
395 |
gr.Markdown("We are the BASIS-China iGEM team, and we're excited to present our deployment of the Tranception model for protein fitness prediction. This tool enables in silico directed evolution to iteratively improve protein fitness through single amino acid substitutions. At each step, Tranception computes log likelihood ratios for all possible mutations compared to the starting sequence, generating fitness heatmaps and recommendations to guide protein engineering.")
|
396 |
-
gr.Markdown("**Technical Details**: This deployment leverages Hugging Face's Zero GPU infrastructure, which dynamically allocates H200 GPU resources when available. This allows for efficient inference while managing computational resources effectively.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
397 |
|
398 |
with gr.Tabs():
|
399 |
with gr.TabItem("Input"):
|
@@ -484,6 +622,15 @@ with tranception_design:
|
|
484 |
gr.Markdown("Links: <a href='https://proceedings.mlr.press/v162/notin22a.html' target='_blank'>Paper</a> <a href='https://github.com/OATML-Markslab/Tranception' target='_blank'>Code</a> <a href='https://sites.google.com/view/proteingym/substitutions' target='_blank'>ProteinGym</a> <a href='https://igem.org/teams/5247' target='_blank'>BASIS-China iGEM Team</a>")
|
485 |
|
486 |
if __name__ == "__main__":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
487 |
# Configure queue for better resource management
|
488 |
tranception_design.queue(
|
489 |
max_size=10, # Limit queue size
|
@@ -492,10 +639,49 @@ if __name__ == "__main__":
|
|
492 |
)
|
493 |
|
494 |
# Launch with appropriate settings for HF Spaces
|
495 |
-
#
|
496 |
-
|
497 |
-
|
498 |
-
|
499 |
-
|
500 |
-
|
501 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
17 |
import shutil
|
18 |
import uuid
|
19 |
import gc
|
20 |
+
import time
|
21 |
+
import signal
|
22 |
+
import threading
|
23 |
+
import datetime
|
24 |
|
25 |
# Check if Zero GPU should be disabled via environment variable
|
26 |
DISABLE_ZERO_GPU = os.environ.get('DISABLE_ZERO_GPU', 'false').lower() == 'true'
|
27 |
|
28 |
+
# Keep-alive settings
|
29 |
+
KEEP_ALIVE_INTERVAL = 300 # 5 minutes
|
30 |
+
last_activity_time = datetime.datetime.now()
|
31 |
+
keep_alive_thread = None
|
32 |
+
|
33 |
+
# Auto-refresh component to keep connection alive
|
34 |
+
AUTO_REFRESH_INTERVAL = 240 # 4 minutes
|
35 |
+
|
36 |
# Create a mock spaces module for fallback
|
37 |
class MockSpaces:
|
38 |
"""Mock spaces module for when Zero GPU is not available"""
|
|
|
64 |
spaces = MockSpaces()
|
65 |
print(f"Warning: Error with spaces module: {e}. Running without Zero GPU support.")
|
66 |
|
67 |
+
# Flag to track if we should avoid Zero GPU due to initialization errors
|
68 |
+
USE_ZERO_GPU = SPACES_AVAILABLE and not DISABLE_ZERO_GPU
|
69 |
+
|
70 |
+
# Global flag to track initialization state
|
71 |
+
INIT_FAILED = False
|
72 |
+
|
73 |
+
def handle_init_error(signum, frame):
|
74 |
+
"""Handle initialization errors gracefully"""
|
75 |
+
global INIT_FAILED
|
76 |
+
INIT_FAILED = True
|
77 |
+
print("Handling initialization error...")
|
78 |
+
sys.exit(1)
|
79 |
+
|
80 |
+
# Set up signal handler for graceful shutdown
|
81 |
+
signal.signal(signal.SIGTERM, handle_init_error)
|
82 |
+
|
83 |
+
def update_activity():
|
84 |
+
"""Update last activity time"""
|
85 |
+
global last_activity_time
|
86 |
+
last_activity_time = datetime.datetime.now()
|
87 |
+
|
88 |
+
def keep_alive_worker():
|
89 |
+
"""Background thread to keep the Space alive"""
|
90 |
+
while True:
|
91 |
+
try:
|
92 |
+
time.sleep(KEEP_ALIVE_INTERVAL)
|
93 |
+
current_time = datetime.datetime.now()
|
94 |
+
time_since_activity = (current_time - last_activity_time).total_seconds()
|
95 |
+
|
96 |
+
print(f"Keep-alive check: Last activity {time_since_activity:.0f} seconds ago")
|
97 |
+
|
98 |
+
# Update activity timestamp
|
99 |
+
if time_since_activity > KEEP_ALIVE_INTERVAL:
|
100 |
+
print("Updating activity timestamp...")
|
101 |
+
update_activity()
|
102 |
+
|
103 |
+
# Create a dummy inference to keep Zero GPU warm
|
104 |
+
try:
|
105 |
+
if USE_ZERO_GPU and hasattr(score_and_create_matrix_all_singles, '__wrapped__'):
|
106 |
+
print("Triggering keep-alive inference...")
|
107 |
+
# Use the smallest possible input
|
108 |
+
dummy_result = score_and_create_matrix_all_singles(
|
109 |
+
sequence="MSKGE",
|
110 |
+
mutation_range_start=1,
|
111 |
+
mutation_range_end=2,
|
112 |
+
model_type="Small",
|
113 |
+
scoring_mirror=False,
|
114 |
+
batch_size_inference=1
|
115 |
+
)
|
116 |
+
print("Keep-alive inference completed")
|
117 |
+
# Clean up results
|
118 |
+
if dummy_result and len(dummy_result) > 2:
|
119 |
+
for file in dummy_result[2]: # CSV files
|
120 |
+
try:
|
121 |
+
os.remove(file)
|
122 |
+
except:
|
123 |
+
pass
|
124 |
+
except Exception as e:
|
125 |
+
print(f"Keep-alive inference error (non-critical): {e}")
|
126 |
+
except Exception as e:
|
127 |
+
print(f"Keep-alive thread error: {e}")
|
128 |
+
time.sleep(60) # Wait a bit before retrying
|
129 |
+
|
130 |
+
def warm_up_zero_gpu():
|
131 |
+
"""Warm up Zero GPU after idle period"""
|
132 |
+
if not USE_ZERO_GPU:
|
133 |
+
return False
|
134 |
+
|
135 |
+
print("Warming up Zero GPU...")
|
136 |
+
# Note: Cannot reliably warm up Zero GPU outside of decorated functions
|
137 |
+
# This is a limitation of the Zero GPU system
|
138 |
+
return False
|
139 |
+
|
140 |
# Add current directory to path
|
141 |
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
142 |
|
|
|
344 |
return ''.join(mutated_sequence)
|
345 |
|
346 |
def score_and_create_matrix_all_singles_impl(sequence,mutation_range_start=None,mutation_range_end=None,model_type="Large",scoring_mirror=False,batch_size_inference=20,max_number_positions_per_heatmap=50,num_workers=0,AA_vocab=AA_vocab):
|
347 |
+
# Update activity time
|
348 |
+
update_activity()
|
349 |
+
|
350 |
# Clean up old files periodically
|
351 |
cleanup_old_files()
|
352 |
|
|
|
383 |
|
384 |
# Device selection - Zero GPU will provide CUDA when decorated with @spaces.GPU
|
385 |
print(f"GPU Available: {torch.cuda.is_available()}")
|
386 |
+
|
387 |
+
# Try to ensure GPU is available when using Zero GPU
|
388 |
+
if USE_ZERO_GPU and not torch.cuda.is_available():
|
389 |
+
print("Zero GPU enabled but CUDA not available - this is expected before GPU allocation")
|
390 |
+
|
391 |
if torch.cuda.is_available():
|
392 |
device = torch.device("cuda")
|
393 |
model = model.to(device)
|
|
|
401 |
device = torch.device("cpu")
|
402 |
model = model.to(device)
|
403 |
print("Inference will take place on CPU")
|
404 |
+
if USE_ZERO_GPU:
|
405 |
+
print("WARNING: Zero GPU is enabled but CUDA is not available!")
|
406 |
+
print("The Space may need to be restarted from the Hugging Face interface.")
|
407 |
# Reduce batch size for CPU inference
|
408 |
batch_size_inference = min(batch_size_inference, 10)
|
409 |
|
|
|
461 |
torch.cuda.empty_cache()
|
462 |
|
463 |
# Apply Zero GPU decorator - will use real decorator if available, mock otherwise
|
464 |
+
if USE_ZERO_GPU:
|
465 |
score_and_create_matrix_all_singles = spaces.GPU(duration=300)(score_and_create_matrix_all_singles_impl)
|
466 |
+
else:
|
467 |
+
score_and_create_matrix_all_singles = score_and_create_matrix_all_singles_impl
|
|
|
|
|
|
|
|
|
|
|
468 |
|
469 |
def extract_sequence(protein_id, taxon, sequence):
|
470 |
return sequence
|
|
|
475 |
mutation_range_end = None
|
476 |
return protein_sequence_input,mutation_range_start,mutation_range_end
|
477 |
|
478 |
+
# Health check endpoint
|
479 |
+
def health_check():
|
480 |
+
"""Simple health check that returns current status"""
|
481 |
+
update_activity()
|
482 |
+
status = {
|
483 |
+
"status": "healthy",
|
484 |
+
"zero_gpu": USE_ZERO_GPU,
|
485 |
+
"cuda_available": torch.cuda.is_available(),
|
486 |
+
"last_activity": last_activity_time.isoformat(),
|
487 |
+
"timestamp": datetime.datetime.now().isoformat()
|
488 |
+
}
|
489 |
+
return status
|
490 |
+
|
491 |
# Create Gradio app
|
492 |
tranception_design = gr.Blocks()
|
493 |
|
|
|
496 |
gr.Markdown("## 🧬 BASIS-China iGEM Team 2025 - Protein Engineering Platform")
|
497 |
gr.Markdown("### Welcome to BASIS-China's implementation of Tranception on Hugging Face Spaces!")
|
498 |
gr.Markdown("We are the BASIS-China iGEM team, and we're excited to present our deployment of the Tranception model for protein fitness prediction. This tool enables in silico directed evolution to iteratively improve protein fitness through single amino acid substitutions. At each step, Tranception computes log likelihood ratios for all possible mutations compared to the starting sequence, generating fitness heatmaps and recommendations to guide protein engineering.")
|
499 |
+
gr.Markdown("**Technical Details**: This deployment leverages Hugging Face's Zero GPU infrastructure, which dynamically allocates H200 GPU resources when available. This allows for efficient inference while managing computational resources effectively. The system includes automatic keep-alive mechanisms to maintain GPU availability.")
|
500 |
+
|
501 |
+
# Status indicator
|
502 |
+
with gr.Row():
|
503 |
+
with gr.Column(scale=1):
|
504 |
+
def get_gpu_status():
|
505 |
+
time_since = (datetime.datetime.now() - last_activity_time).total_seconds()
|
506 |
+
if USE_ZERO_GPU:
|
507 |
+
if torch.cuda.is_available():
|
508 |
+
gpu_name = torch.cuda.get_device_name(0)
|
509 |
+
return f"🔥 Zero GPU Active: {gpu_name} | Last activity: {int(time_since)}s ago"
|
510 |
+
else:
|
511 |
+
return f"⚠️ Zero GPU: Ready | Last activity: {int(time_since)}s ago"
|
512 |
+
else:
|
513 |
+
return "💻 Running on CPU"
|
514 |
+
|
515 |
+
gpu_status = gr.Textbox(
|
516 |
+
label="Compute Status",
|
517 |
+
value=get_gpu_status,
|
518 |
+
every=5, # Update every 5 seconds
|
519 |
+
interactive=False,
|
520 |
+
elem_id="gpu_status"
|
521 |
+
)
|
522 |
+
|
523 |
+
# Hidden components for keep-alive
|
524 |
+
with gr.Row(visible=False):
|
525 |
+
# Auto-refresh component to maintain WebSocket connection
|
526 |
+
keep_alive_refresh = gr.Number(value=0, visible=False)
|
527 |
+
|
528 |
+
def increment_counter():
|
529 |
+
update_activity()
|
530 |
+
return gr.update(value=time.time())
|
531 |
+
|
532 |
+
# This will trigger every 4 minutes to keep the connection alive
|
533 |
+
keep_alive_timer = gr.Timer(value=AUTO_REFRESH_INTERVAL)
|
534 |
+
keep_alive_timer.tick(increment_counter, outputs=[keep_alive_refresh])
|
535 |
|
536 |
with gr.Tabs():
|
537 |
with gr.TabItem("Input"):
|
|
|
622 |
gr.Markdown("Links: <a href='https://proceedings.mlr.press/v162/notin22a.html' target='_blank'>Paper</a> <a href='https://github.com/OATML-Markslab/Tranception' target='_blank'>Code</a> <a href='https://sites.google.com/view/proteingym/substitutions' target='_blank'>ProteinGym</a> <a href='https://igem.org/teams/5247' target='_blank'>BASIS-China iGEM Team</a>")
|
623 |
|
624 |
if __name__ == "__main__":
|
625 |
+
# Start keep-alive thread
|
626 |
+
if USE_ZERO_GPU:
|
627 |
+
print("Starting keep-alive thread for Zero GPU...")
|
628 |
+
keep_alive_thread = threading.Thread(target=keep_alive_worker, daemon=True)
|
629 |
+
keep_alive_thread.start()
|
630 |
+
|
631 |
+
# Schedule periodic dummy inferences to keep alive
|
632 |
+
print("Keep-alive system activated - will perform dummy inferences every 5 minutes")
|
633 |
+
|
634 |
# Configure queue for better resource management
|
635 |
tranception_design.queue(
|
636 |
max_size=10, # Limit queue size
|
|
|
639 |
)
|
640 |
|
641 |
# Launch with appropriate settings for HF Spaces
|
642 |
+
# Wrap launch in try-except to handle Zero GPU initialization errors gracefully
|
643 |
+
launch_retries = 0
|
644 |
+
max_launch_retries = 3
|
645 |
+
|
646 |
+
while launch_retries < max_launch_retries:
|
647 |
+
try:
|
648 |
+
# Add a small delay on retries to allow system to stabilize
|
649 |
+
if launch_retries > 0:
|
650 |
+
print(f"Retry attempt {launch_retries}/{max_launch_retries}...")
|
651 |
+
time.sleep(5)
|
652 |
+
|
653 |
+
tranception_design.launch(
|
654 |
+
max_threads=2, # Limit concurrent threads
|
655 |
+
show_error=True,
|
656 |
+
server_name="0.0.0.0",
|
657 |
+
server_port=7860,
|
658 |
+
quiet=False, # Show all logs
|
659 |
+
prevent_thread_lock=True, # Prevent thread locking issues
|
660 |
+
share=False, # Don't create public link
|
661 |
+
inbrowser=False # Don't open browser
|
662 |
+
)
|
663 |
+
break # If successful, exit the retry loop
|
664 |
+
|
665 |
+
except RuntimeError as e:
|
666 |
+
error_msg = str(e)
|
667 |
+
if "ZeroGPU" in error_msg or "Unknown" in error_msg:
|
668 |
+
print(f"Zero GPU initialization error: {e}")
|
669 |
+
launch_retries += 1
|
670 |
+
|
671 |
+
if launch_retries < max_launch_retries:
|
672 |
+
print("Retrying with Zero GPU after warm-up...")
|
673 |
+
# Wait longer before retry
|
674 |
+
time.sleep(10)
|
675 |
+
else:
|
676 |
+
print("Max retries reached. The Space may need to be restarted.")
|
677 |
+
print("Note: Zero GPU containers can crash after idle periods.")
|
678 |
+
print("Consider restarting the Space from the Hugging Face interface.")
|
679 |
+
sys.exit(1)
|
680 |
+
else:
|
681 |
+
# Non-Zero GPU error, re-raise
|
682 |
+
raise
|
683 |
+
except Exception as e:
|
684 |
+
print(f"Unexpected error during launch: {e}")
|
685 |
+
launch_retries += 1
|
686 |
+
if launch_retries >= max_launch_retries:
|
687 |
+
raise
|
app_wrapper.py
ADDED
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Wrapper script for Tranception app with Zero GPU error handling
|
4 |
+
"""
|
5 |
+
import os
|
6 |
+
import sys
|
7 |
+
import subprocess
|
8 |
+
import time
|
9 |
+
|
10 |
+
def run_app(disable_zero_gpu=False):
|
11 |
+
"""Run the main app with optional Zero GPU disable"""
|
12 |
+
env = os.environ.copy()
|
13 |
+
if disable_zero_gpu:
|
14 |
+
env['DISABLE_ZERO_GPU'] = 'true'
|
15 |
+
print("Running with Zero GPU disabled")
|
16 |
+
else:
|
17 |
+
print("Attempting to run with Zero GPU support")
|
18 |
+
|
19 |
+
cmd = [sys.executable, "app.py"]
|
20 |
+
return subprocess.run(cmd, env=env)
|
21 |
+
|
22 |
+
def main():
|
23 |
+
"""Main wrapper that handles Zero GPU initialization errors"""
|
24 |
+
max_retries = 3
|
25 |
+
retry_count = 0
|
26 |
+
|
27 |
+
while retry_count < max_retries:
|
28 |
+
try:
|
29 |
+
# First attempt with Zero GPU
|
30 |
+
if retry_count == 0:
|
31 |
+
result = run_app(disable_zero_gpu=False)
|
32 |
+
else:
|
33 |
+
# Subsequent attempts without Zero GPU
|
34 |
+
print(f"Retry {retry_count}: Running without Zero GPU")
|
35 |
+
result = run_app(disable_zero_gpu=True)
|
36 |
+
|
37 |
+
# Check if the app exited due to Zero GPU error
|
38 |
+
if result.returncode == 1:
|
39 |
+
retry_count += 1
|
40 |
+
if retry_count < max_retries:
|
41 |
+
print(f"App crashed. Waiting 5 seconds before retry...")
|
42 |
+
time.sleep(5)
|
43 |
+
continue
|
44 |
+
else:
|
45 |
+
# Normal exit or other error
|
46 |
+
break
|
47 |
+
|
48 |
+
except KeyboardInterrupt:
|
49 |
+
print("\nShutting down...")
|
50 |
+
break
|
51 |
+
except Exception as e:
|
52 |
+
print(f"Unexpected error: {e}")
|
53 |
+
retry_count += 1
|
54 |
+
if retry_count < max_retries:
|
55 |
+
time.sleep(5)
|
56 |
+
|
57 |
+
if retry_count >= max_retries:
|
58 |
+
print("Max retries reached. Please check the logs.")
|
59 |
+
sys.exit(1)
|
60 |
+
|
61 |
+
if __name__ == "__main__":
|
62 |
+
main()
|
app_zero_gpu_fixed.py
ADDED
@@ -0,0 +1,498 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Tranception Design App - Hugging Face Spaces Version (Zero GPU Fixed)
|
4 |
+
"""
|
5 |
+
import os
|
6 |
+
import sys
|
7 |
+
import torch
|
8 |
+
import transformers
|
9 |
+
from transformers import PreTrainedTokenizerFast
|
10 |
+
import numpy as np
|
11 |
+
import pandas as pd
|
12 |
+
import matplotlib.pyplot as plt
|
13 |
+
import seaborn as sns
|
14 |
+
import gradio as gr
|
15 |
+
from huggingface_hub import hf_hub_download
|
16 |
+
import shutil
|
17 |
+
import uuid
|
18 |
+
import gc
|
19 |
+
import time
|
20 |
+
import datetime
|
21 |
+
|
22 |
+
# Simplified Zero GPU handling
|
23 |
+
try:
|
24 |
+
import spaces
|
25 |
+
SPACES_AVAILABLE = True
|
26 |
+
print("Zero GPU support detected")
|
27 |
+
except ImportError:
|
28 |
+
SPACES_AVAILABLE = False
|
29 |
+
print("Running without Zero GPU support")
|
30 |
+
|
31 |
+
# Add current directory to path
|
32 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
33 |
+
|
34 |
+
# Check if we need to download and extract the tranception module
|
35 |
+
if not os.path.exists("tranception"):
|
36 |
+
print("Downloading Tranception repository...")
|
37 |
+
try:
|
38 |
+
# Clone the repository structure
|
39 |
+
result = os.system("git clone https://github.com/OATML-Markslab/Tranception.git temp_tranception")
|
40 |
+
if result != 0:
|
41 |
+
raise Exception("Failed to clone Tranception repository")
|
42 |
+
# Move the tranception module to current directory
|
43 |
+
shutil.move("temp_tranception/tranception", "tranception")
|
44 |
+
# Clean up
|
45 |
+
shutil.rmtree("temp_tranception")
|
46 |
+
except Exception as e:
|
47 |
+
print(f"Error setting up Tranception: {e}")
|
48 |
+
if os.path.exists("temp_tranception"):
|
49 |
+
shutil.rmtree("temp_tranception")
|
50 |
+
raise
|
51 |
+
|
52 |
+
import tranception
|
53 |
+
from tranception import config, model_pytorch
|
54 |
+
|
55 |
+
# Download model checkpoints if not present
|
56 |
+
def download_model_from_hf(model_name):
|
57 |
+
"""Download model from Hugging Face Hub if not present locally"""
|
58 |
+
model_path = f"./{model_name}"
|
59 |
+
if not os.path.exists(model_path):
|
60 |
+
print(f"Loading {model_name} model from Hugging Face Hub...")
|
61 |
+
# All models are available on HF Hub
|
62 |
+
return f"PascalNotin/{model_name}"
|
63 |
+
return model_path
|
64 |
+
|
65 |
+
AA_vocab = "ACDEFGHIKLMNPQRSTVWY"
|
66 |
+
tokenizer = PreTrainedTokenizerFast(tokenizer_file="./tranception/utils/tokenizers/Basic_tokenizer",
|
67 |
+
unk_token="[UNK]",
|
68 |
+
sep_token="[SEP]",
|
69 |
+
pad_token="[PAD]",
|
70 |
+
cls_token="[CLS]",
|
71 |
+
mask_token="[MASK]"
|
72 |
+
)
|
73 |
+
|
74 |
+
def create_all_single_mutants(sequence,AA_vocab=AA_vocab,mutation_range_start=None,mutation_range_end=None):
|
75 |
+
all_single_mutants={}
|
76 |
+
sequence_list=list(sequence)
|
77 |
+
if mutation_range_start is None: mutation_range_start=1
|
78 |
+
if mutation_range_end is None: mutation_range_end=len(sequence)
|
79 |
+
for position,current_AA in enumerate(sequence[mutation_range_start-1:mutation_range_end]):
|
80 |
+
for mutated_AA in AA_vocab:
|
81 |
+
if current_AA!=mutated_AA:
|
82 |
+
mutated_sequence = sequence_list.copy()
|
83 |
+
mutated_sequence[mutation_range_start + position - 1] = mutated_AA
|
84 |
+
all_single_mutants[current_AA+str(mutation_range_start+position)+mutated_AA]="".join(mutated_sequence)
|
85 |
+
all_single_mutants = pd.DataFrame.from_dict(all_single_mutants,columns=['mutated_sequence'],orient='index')
|
86 |
+
all_single_mutants.reset_index(inplace=True)
|
87 |
+
all_single_mutants.columns = ['mutant','mutated_sequence']
|
88 |
+
return all_single_mutants
|
89 |
+
|
90 |
+
def create_scoring_matrix_visual(scores,sequence,image_index=0,mutation_range_start=None,mutation_range_end=None,AA_vocab=AA_vocab,annotate=True,fontsize=20,unique_id=None):
|
91 |
+
if unique_id is None:
|
92 |
+
unique_id = str(uuid.uuid4())
|
93 |
+
|
94 |
+
filtered_scores=scores.copy()
|
95 |
+
filtered_scores=filtered_scores[filtered_scores.position.isin(range(mutation_range_start,mutation_range_end+1))]
|
96 |
+
piv=filtered_scores.pivot(index='position',columns='target_AA',values='avg_score').round(4)
|
97 |
+
|
98 |
+
# Save CSV file
|
99 |
+
csv_path = 'fitness_scoring_substitution_matrix_{}_{}.csv'.format(unique_id, image_index)
|
100 |
+
|
101 |
+
# Create a more detailed CSV with mutation info
|
102 |
+
csv_data = []
|
103 |
+
for position in range(mutation_range_start,mutation_range_end+1):
|
104 |
+
for target_AA in list(AA_vocab):
|
105 |
+
mutant = sequence[position-1]+str(position)+target_AA
|
106 |
+
if mutant in set(filtered_scores.mutant):
|
107 |
+
score_value = filtered_scores.loc[filtered_scores.mutant==mutant,'avg_score']
|
108 |
+
if isinstance(score_value, pd.Series):
|
109 |
+
score = float(score_value.iloc[0])
|
110 |
+
else:
|
111 |
+
score = float(score_value)
|
112 |
+
else:
|
113 |
+
score = 0.0
|
114 |
+
|
115 |
+
csv_data.append({
|
116 |
+
'position': position,
|
117 |
+
'original_AA': sequence[position-1],
|
118 |
+
'target_AA': target_AA,
|
119 |
+
'mutation': mutant,
|
120 |
+
'fitness_score': score
|
121 |
+
})
|
122 |
+
|
123 |
+
csv_df = pd.DataFrame(csv_data)
|
124 |
+
csv_df.to_csv(csv_path, index=False)
|
125 |
+
|
126 |
+
# Continue with visualization
|
127 |
+
mutation_range_len = mutation_range_end - mutation_range_start + 1
|
128 |
+
# Limit figure size to prevent memory issues
|
129 |
+
fig_width = min(50, len(AA_vocab) * 0.8)
|
130 |
+
fig_height = min(mutation_range_len, 50)
|
131 |
+
fig, ax = plt.subplots(figsize=(fig_width, fig_height))
|
132 |
+
scores_dict = {}
|
133 |
+
valid_mutant_set=set(filtered_scores.mutant)
|
134 |
+
ax.tick_params(bottom=True, top=True, left=True, right=True)
|
135 |
+
ax.tick_params(labelbottom=True, labeltop=True, labelleft=True, labelright=True)
|
136 |
+
if annotate:
|
137 |
+
for position in range(mutation_range_start,mutation_range_end+1):
|
138 |
+
for target_AA in list(AA_vocab):
|
139 |
+
mutant = sequence[position-1]+str(position)+target_AA
|
140 |
+
if mutant in valid_mutant_set:
|
141 |
+
score_value = filtered_scores.loc[filtered_scores.mutant==mutant,'avg_score']
|
142 |
+
if isinstance(score_value, pd.Series):
|
143 |
+
scores_dict[mutant] = float(score_value.iloc[0])
|
144 |
+
else:
|
145 |
+
scores_dict[mutant] = float(score_value)
|
146 |
+
else:
|
147 |
+
scores_dict[mutant]=0.0
|
148 |
+
labels = (np.asarray(["{} \n {:.4f}".format(symb,value) for symb, value in scores_dict.items() ])).reshape(mutation_range_len,len(AA_vocab))
|
149 |
+
heat = sns.heatmap(piv,annot=labels,fmt="",cmap='RdYlGn',linewidths=0.30,ax=ax,vmin=np.percentile(scores.avg_score,2),vmax=np.percentile(scores.avg_score,98),\
|
150 |
+
cbar_kws={'label': 'Log likelihood ratio (mutant / starting sequence)'},annot_kws={"size": fontsize})
|
151 |
+
else:
|
152 |
+
heat = sns.heatmap(piv,cmap='RdYlGn',linewidths=0.30,ax=ax,vmin=np.percentile(scores.avg_score,2),vmax=np.percentile(scores.avg_score,98),\
|
153 |
+
cbar_kws={'label': 'Log likelihood ratio (mutant / starting sequence)'},annot_kws={"size": fontsize})
|
154 |
+
heat.figure.axes[-1].yaxis.label.set_size(fontsize=int(fontsize*1.5))
|
155 |
+
heat.set_title("Higher predicted scores (green) imply higher protein fitness",fontsize=fontsize*2, pad=40)
|
156 |
+
heat.set_ylabel("Sequence position", fontsize = fontsize*2)
|
157 |
+
heat.set_xlabel("Amino Acid mutation", fontsize = fontsize*2)
|
158 |
+
|
159 |
+
# Set y-axis labels (positions)
|
160 |
+
yticklabels = [str(pos)+' ('+sequence[pos-1]+')' for pos in range(mutation_range_start,mutation_range_end+1)]
|
161 |
+
heat.set_yticklabels(yticklabels, fontsize=fontsize, rotation=0)
|
162 |
+
|
163 |
+
# Set x-axis labels (amino acids) - ensuring correct number
|
164 |
+
heat.set_xticklabels(list(AA_vocab), fontsize=fontsize)
|
165 |
+
try:
|
166 |
+
plt.tight_layout()
|
167 |
+
image_path = 'fitness_scoring_substitution_matrix_{}_{}.png'.format(unique_id, image_index)
|
168 |
+
plt.savefig(image_path,dpi=100)
|
169 |
+
return image_path, csv_path
|
170 |
+
finally:
|
171 |
+
plt.close('all') # Ensure all figures are closed
|
172 |
+
plt.clf() # Clear the current figure
|
173 |
+
plt.cla() # Clear the current axes
|
174 |
+
|
175 |
+
def suggest_mutations(scores):
|
176 |
+
intro_message = "The following mutations may be sensible options to improve fitness: \n\n"
|
177 |
+
#Best mutants
|
178 |
+
top_mutants=list(scores.sort_values(by=['avg_score'],ascending=False).head(5).mutant)
|
179 |
+
top_mutants_fitness=list(scores.sort_values(by=['avg_score'],ascending=False).head(5).avg_score)
|
180 |
+
top_mutants_recos = [top_mutant+" ("+str(round(top_mutant_fitness,4))+")" for (top_mutant,top_mutant_fitness) in zip(top_mutants,top_mutants_fitness)]
|
181 |
+
mutant_recos = "The single mutants with highest predicted fitness are (positive scores indicate fitness increase Vs starting sequence, negative scores indicate fitness decrease):\n {} \n\n".format(", ".join(top_mutants_recos))
|
182 |
+
#Best positions
|
183 |
+
positive_scores = scores[scores.avg_score > 0]
|
184 |
+
if len(positive_scores) > 0:
|
185 |
+
# Only select numeric columns for groupby mean
|
186 |
+
positive_scores_position_avg = positive_scores.groupby(['position'])['avg_score'].mean().reset_index()
|
187 |
+
top_positions=list(positive_scores_position_avg.sort_values(by=['avg_score'],ascending=False).head(5)['position'].astype(str))
|
188 |
+
position_recos = "The positions with the highest average fitness increase are (only positions with at least one fitness increase are considered):\n {}".format(", ".join(top_positions))
|
189 |
+
else:
|
190 |
+
position_recos = "No positions with positive fitness effects found."
|
191 |
+
return intro_message+mutant_recos+position_recos
|
192 |
+
|
193 |
+
def check_valid_mutant(sequence,mutant,AA_vocab=AA_vocab):
|
194 |
+
valid = True
|
195 |
+
try:
|
196 |
+
from_AA, position, to_AA = mutant[0], int(mutant[1:-1]), mutant[-1]
|
197 |
+
except:
|
198 |
+
valid = False
|
199 |
+
if valid and position > 0 and position <= len(sequence):
|
200 |
+
if sequence[position-1]!=from_AA: valid=False
|
201 |
+
else:
|
202 |
+
valid = False
|
203 |
+
if to_AA not in AA_vocab: valid=False
|
204 |
+
return valid
|
205 |
+
|
206 |
+
def cleanup_old_files(max_age_minutes=30):
|
207 |
+
"""Clean up old inference files"""
|
208 |
+
import glob
|
209 |
+
current_time = time.time()
|
210 |
+
patterns = ["fitness_scoring_substitution_matrix_*.png",
|
211 |
+
"fitness_scoring_substitution_matrix_*.csv",
|
212 |
+
"all_mutations_fitness_scores_*.csv"]
|
213 |
+
|
214 |
+
cleaned_count = 0
|
215 |
+
for pattern in patterns:
|
216 |
+
for file_path in glob.glob(pattern):
|
217 |
+
try:
|
218 |
+
file_age = current_time - os.path.getmtime(file_path)
|
219 |
+
if file_age > max_age_minutes * 60:
|
220 |
+
os.remove(file_path)
|
221 |
+
cleaned_count += 1
|
222 |
+
except Exception as e:
|
223 |
+
# Log error but continue cleaning other files
|
224 |
+
print(f"Warning: Could not remove {file_path}: {e}")
|
225 |
+
|
226 |
+
if cleaned_count > 0:
|
227 |
+
print(f"Cleaned up {cleaned_count} old files")
|
228 |
+
|
229 |
+
def get_mutated_protein(sequence,mutant):
|
230 |
+
if not check_valid_mutant(sequence,mutant):
|
231 |
+
return "The mutant is not valid"
|
232 |
+
mutated_sequence = list(sequence)
|
233 |
+
mutated_sequence[int(mutant[1:-1])-1]=mutant[-1]
|
234 |
+
return ''.join(mutated_sequence)
|
235 |
+
|
236 |
+
def score_and_create_matrix_all_singles_impl(sequence,mutation_range_start=None,mutation_range_end=None,model_type="Large",scoring_mirror=False,batch_size_inference=20,max_number_positions_per_heatmap=50,num_workers=0,AA_vocab=AA_vocab):
|
237 |
+
# Clean up old files periodically
|
238 |
+
cleanup_old_files()
|
239 |
+
|
240 |
+
# Generate unique ID for this request
|
241 |
+
unique_id = str(uuid.uuid4())
|
242 |
+
|
243 |
+
if mutation_range_start is None: mutation_range_start=1
|
244 |
+
if mutation_range_end is None: mutation_range_end=len(sequence)
|
245 |
+
|
246 |
+
# Clean sequence
|
247 |
+
sequence = sequence.strip().upper()
|
248 |
+
|
249 |
+
# Validate
|
250 |
+
assert len(sequence) > 0, "no sequence entered"
|
251 |
+
assert mutation_range_start <= mutation_range_end, "mutation range is invalid"
|
252 |
+
assert mutation_range_end <= len(sequence), f"End position ({mutation_range_end}) exceeds sequence length ({len(sequence)})"
|
253 |
+
|
254 |
+
# Load model with HF Space compatibility
|
255 |
+
try:
|
256 |
+
if model_type=="Small":
|
257 |
+
model_path = download_model_from_hf("Tranception_Small")
|
258 |
+
model = tranception.model_pytorch.TranceptionLMHeadModel.from_pretrained(pretrained_model_name_or_path=model_path)
|
259 |
+
elif model_type=="Medium":
|
260 |
+
model_path = download_model_from_hf("Tranception_Medium")
|
261 |
+
model = tranception.model_pytorch.TranceptionLMHeadModel.from_pretrained(pretrained_model_name_or_path=model_path)
|
262 |
+
elif model_type=="Large":
|
263 |
+
model_path = download_model_from_hf("Tranception_Large")
|
264 |
+
model = tranception.model_pytorch.TranceptionLMHeadModel.from_pretrained(pretrained_model_name_or_path=model_path)
|
265 |
+
except Exception as e:
|
266 |
+
print(f"Error loading {model_type} model: {e}")
|
267 |
+
print("Falling back to Medium model...")
|
268 |
+
model_path = download_model_from_hf("Tranception_Medium")
|
269 |
+
model = tranception.model_pytorch.TranceptionLMHeadModel.from_pretrained(pretrained_model_name_or_path=model_path)
|
270 |
+
|
271 |
+
# Device selection - Zero GPU will provide CUDA when decorated with @spaces.GPU
|
272 |
+
print(f"GPU Available: {torch.cuda.is_available()}")
|
273 |
+
|
274 |
+
if torch.cuda.is_available():
|
275 |
+
device = torch.device("cuda")
|
276 |
+
model = model.to(device)
|
277 |
+
gpu_name = torch.cuda.get_device_name(0)
|
278 |
+
gpu_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3
|
279 |
+
print(f"Inference will take place on {gpu_name}")
|
280 |
+
print(f"GPU Memory: {gpu_memory:.2f} GB")
|
281 |
+
# Increase batch size for GPU inference
|
282 |
+
batch_size_inference = min(batch_size_inference, 50)
|
283 |
+
else:
|
284 |
+
device = torch.device("cpu")
|
285 |
+
model = model.to(device)
|
286 |
+
print("Inference will take place on CPU")
|
287 |
+
# Reduce batch size for CPU inference
|
288 |
+
batch_size_inference = min(batch_size_inference, 10)
|
289 |
+
|
290 |
+
try:
|
291 |
+
model.eval()
|
292 |
+
model.config.tokenizer = tokenizer
|
293 |
+
|
294 |
+
all_single_mutants = create_all_single_mutants(sequence,AA_vocab,mutation_range_start,mutation_range_end)
|
295 |
+
|
296 |
+
with torch.no_grad():
|
297 |
+
scores = model.score_mutants(DMS_data=all_single_mutants,
|
298 |
+
target_seq=sequence,
|
299 |
+
scoring_mirror=scoring_mirror,
|
300 |
+
batch_size_inference=batch_size_inference,
|
301 |
+
num_workers=num_workers,
|
302 |
+
indel_mode=False
|
303 |
+
)
|
304 |
+
|
305 |
+
scores = pd.merge(scores,all_single_mutants,on="mutated_sequence",how="left")
|
306 |
+
scores["position"]=scores["mutant"].map(lambda x: int(x[1:-1]))
|
307 |
+
scores["target_AA"] = scores["mutant"].map(lambda x: x[-1])
|
308 |
+
|
309 |
+
score_heatmaps = []
|
310 |
+
csv_files = []
|
311 |
+
mutation_range = mutation_range_end - mutation_range_start + 1
|
312 |
+
number_heatmaps = int((mutation_range - 1) / max_number_positions_per_heatmap) + 1
|
313 |
+
image_index = 0
|
314 |
+
window_start = mutation_range_start
|
315 |
+
window_end = min(mutation_range_end,mutation_range_start+max_number_positions_per_heatmap-1)
|
316 |
+
|
317 |
+
for image_index in range(number_heatmaps):
|
318 |
+
image_path, csv_path = create_scoring_matrix_visual(scores,sequence,image_index,window_start,window_end,AA_vocab,unique_id=unique_id)
|
319 |
+
score_heatmaps.append(image_path)
|
320 |
+
csv_files.append(csv_path)
|
321 |
+
window_start += max_number_positions_per_heatmap
|
322 |
+
window_end = min(mutation_range_end,window_start+max_number_positions_per_heatmap-1)
|
323 |
+
|
324 |
+
# Also save a comprehensive CSV with all mutations
|
325 |
+
comprehensive_csv_path = 'all_mutations_fitness_scores_{}.csv'.format(unique_id)
|
326 |
+
scores_export = scores[['mutant', 'position', 'target_AA', 'avg_score', 'mutated_sequence']].copy()
|
327 |
+
scores_export['original_AA'] = scores_export['mutant'].str[0]
|
328 |
+
scores_export = scores_export.rename(columns={'avg_score': 'fitness_score'})
|
329 |
+
scores_export = scores_export[['position', 'original_AA', 'target_AA', 'mutant', 'fitness_score', 'mutated_sequence']]
|
330 |
+
scores_export.to_csv(comprehensive_csv_path, index=False)
|
331 |
+
csv_files.append(comprehensive_csv_path)
|
332 |
+
|
333 |
+
return score_heatmaps, suggest_mutations(scores), csv_files
|
334 |
+
|
335 |
+
finally:
|
336 |
+
# Always clean up model from memory
|
337 |
+
if 'model' in locals():
|
338 |
+
del model
|
339 |
+
gc.collect()
|
340 |
+
if torch.cuda.is_available():
|
341 |
+
torch.cuda.empty_cache()
|
342 |
+
|
343 |
+
# Apply Zero GPU decorator if available
|
344 |
+
if SPACES_AVAILABLE:
|
345 |
+
score_and_create_matrix_all_singles = spaces.GPU(duration=300)(score_and_create_matrix_all_singles_impl)
|
346 |
+
else:
|
347 |
+
score_and_create_matrix_all_singles = score_and_create_matrix_all_singles_impl
|
348 |
+
|
349 |
+
def extract_sequence(protein_id, taxon, sequence):
|
350 |
+
return sequence
|
351 |
+
|
352 |
+
def clear_inputs(protein_sequence_input,mutation_range_start,mutation_range_end):
|
353 |
+
protein_sequence_input = ""
|
354 |
+
mutation_range_start = None
|
355 |
+
mutation_range_end = None
|
356 |
+
return protein_sequence_input,mutation_range_start,mutation_range_end
|
357 |
+
|
358 |
+
# Create Gradio app
|
359 |
+
tranception_design = gr.Blocks()
|
360 |
+
|
361 |
+
with tranception_design:
|
362 |
+
gr.Markdown("# In silico directed evolution for protein redesign with Tranception")
|
363 |
+
gr.Markdown("## 🧬 BASIS-China iGEM Team 2025 - Protein Engineering Platform")
|
364 |
+
gr.Markdown("### Welcome to BASIS-China's implementation of Tranception on Hugging Face Spaces!")
|
365 |
+
gr.Markdown("We are the BASIS-China iGEM team, and we're excited to present our deployment of the Tranception model for protein fitness prediction. This tool enables in silico directed evolution to iteratively improve protein fitness through single amino acid substitutions. At each step, Tranception computes log likelihood ratios for all possible mutations compared to the starting sequence, generating fitness heatmaps and recommendations to guide protein engineering.")
|
366 |
+
gr.Markdown("**Technical Details**: This deployment leverages Hugging Face's Zero GPU infrastructure, which dynamically allocates H200 GPU resources when available. This allows for efficient inference while managing computational resources effectively.")
|
367 |
+
|
368 |
+
# Status indicator
|
369 |
+
with gr.Row():
|
370 |
+
with gr.Column(scale=1):
|
371 |
+
def get_gpu_status():
|
372 |
+
if SPACES_AVAILABLE:
|
373 |
+
if torch.cuda.is_available():
|
374 |
+
gpu_name = torch.cuda.get_device_name(0)
|
375 |
+
return f"🔥 Zero GPU Active: {gpu_name}"
|
376 |
+
else:
|
377 |
+
return "⚠️ Zero GPU: Ready (GPU allocated on inference)"
|
378 |
+
else:
|
379 |
+
return "💻 Running on CPU"
|
380 |
+
|
381 |
+
gpu_status = gr.Textbox(
|
382 |
+
label="Compute Status",
|
383 |
+
value=get_gpu_status,
|
384 |
+
every=5, # Update every 5 seconds
|
385 |
+
interactive=False,
|
386 |
+
elem_id="gpu_status"
|
387 |
+
)
|
388 |
+
|
389 |
+
with gr.Tabs():
|
390 |
+
with gr.TabItem("Input"):
|
391 |
+
with gr.Row():
|
392 |
+
protein_sequence_input = gr.Textbox(lines=1,
|
393 |
+
label="Protein sequence",
|
394 |
+
placeholder = "Input the sequence of amino acids representing the starting protein of interest or select one from the list of examples below. You may enter the full sequence or just a subdomain (providing full context typically leads to better results, but is slower at inference)"
|
395 |
+
)
|
396 |
+
|
397 |
+
with gr.Row():
|
398 |
+
mutation_range_start = gr.Number(label="Start of mutation window (first position indexed at 1)", value=1, precision=0)
|
399 |
+
mutation_range_end = gr.Number(label="End of mutation window (leave empty for full lenth)", value=10, precision=0)
|
400 |
+
|
401 |
+
with gr.TabItem("Parameters"):
|
402 |
+
with gr.Row():
|
403 |
+
model_size_selection = gr.Radio(label="Tranception model size (larger models are more accurate but are slower at inference)",
|
404 |
+
choices=["Small","Medium","Large"],
|
405 |
+
value="Small")
|
406 |
+
with gr.Row():
|
407 |
+
scoring_mirror = gr.Checkbox(label="Score protein from both directions (leads to more robust fitness predictions, but doubles inference time)")
|
408 |
+
with gr.Row():
|
409 |
+
batch_size_inference = gr.Number(label="Model batch size at inference time (reduce for CPU)",value = 10, precision=0)
|
410 |
+
with gr.Row():
|
411 |
+
gr.Markdown("Note: the current version does not leverage retrieval of homologs at inference time to increase fitness prediction performance.")
|
412 |
+
|
413 |
+
with gr.Row():
|
414 |
+
clear_button = gr.Button(value="Clear",variant="secondary")
|
415 |
+
run_button = gr.Button(value="Predict fitness",variant="primary")
|
416 |
+
|
417 |
+
protein_ID = gr.Textbox(label="Uniprot ID", visible=False)
|
418 |
+
taxon = gr.Textbox(label="Taxon", visible=False)
|
419 |
+
|
420 |
+
examples = gr.Examples(
|
421 |
+
inputs=[protein_ID, taxon, protein_sequence_input],
|
422 |
+
outputs=[protein_sequence_input],
|
423 |
+
fn=extract_sequence,
|
424 |
+
examples=[
|
425 |
+
['ADRB2_HUMAN' ,'Human', 'MGQPGNGSAFLLAPNGSHAPDHDVTQERDEVWVVGMGIVMSLIVLAIVFGNVLVITAIAKFERLQTVTNYFITSLACADLVMGLAVVPFGAAHILMKMWTFGNFWCEFWTSIDVLCVTASIETLCVIAVDRYFAITSPFKYQSLLTKNKARVIILMVWIVSGLTSFLPIQMHWYRATHQEAINCYANETCCDFFTNQAYAIASSIVSFYVPLVIMVFVYSRVFQEAKRQLQKIDKSEGRFHVQNLSQVEQDGRTGHGLRRSSKFCLKEHKALKTLGIIMGTFTLCWLPFFIVNIVHVIQDNLIRKEVYILLNWIGYVNSGFNPLIYCRSPDFRIAFQELLCLRRSSLKAYGNGYSSNGNTGEQSGYHVEQEKENKLLCEDLPGTEDFVGHQGTVPSDNIDSQGRNCSTNDSLL'],
|
426 |
+
['IF1_ECOLI' ,'Prokaryote', 'MAKEDNIEMQGTVLETLPNTMFRVELENGHVVTAHISGKMRKNYIRILTGDKVTVELTPYDLSKGRIVFRSR'],
|
427 |
+
['P53_HUMAN' ,'Human', 'MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGPDEAPRMPEAAPRVAPAPAAPTPAAPAPAPSWPLSSSVPSQKTYQGSYGFRLGFLHSGTAKSVTCTYSPALNKMFCQLAKTCPVQLWVDSTPPPGTRVRAMAIYKQSQHMTEVVRRCPHHERCSDSDGLAPPQHLIRVEGNLRVEYLDDRNTFRHSVVVPYEPPEVGSDCTTIHYNYMCNSSCMGGMNRRPILTIITLEDSSGNLLGRNSFEVRVCACPGRDRRTEEENLRKKGEPHHELPPGSTKRALPNNTSSSPQPKKKPLDGEYFTLQIRGRERFEMFRELNEALELKDAQAGKEPGGSRAHSSHLKSKKGQSTSRHKKLMFKTEGPDSD'],
|
428 |
+
['BLAT_ECOLX' ,'Prokaryote', 'MSIQHFRVALIPFFAAFCLPVFAHPETLVKVKDAEDQLGARVGYIELDLNSGKILESFRPEERFPMMSTFKVLLCGAVLSRVDAGQEQLGRRIHYSQNDLVEYSPVTEKHLTDGMTVRELCSAAITMSDNTAANLLLTTIGGPKELTAFLHNMGDHVTRLDRWEPELNEAIPNDERDTTMPAAMATTLRKLLTGELLTLASRQQLIDWMEADKVAGPLLRSALPAGWFIADKSGAGERGSRGIIAALGPDGKPSRIVVIYTTGSQATMDERNRQIAEIGASLIKHW'],
|
429 |
+
['BRCA1_HUMAN' ,'Human', 'MDLSALRVEEVQNVINAMQKILECPICLELIKEPVSTKCDHIFCKFCMLKLLNQKKGPSQCPLCKNDITKRSLQESTRFSQLVEELLKIICAFQLDTGLEYANSYNFAKKENNSPEHLKDEVSIIQSMGYRNRAKRLLQSEPENPSLQETSLSVQLSNLGTVRTLRTKQRIQPQKTSVYIELGSDSSEDTVNKATYCSVGDQELLQITPQGTRDEISLDSAKKAACEFSETDVTNTEHHQPSNNDLNTTEKRAAERHPEKYQGSSVSNLHVEPCGTNTHASSLQHENSSLLLTKDRMNVEKAEFCNKSKQPGLARSQHNRWAGSKETCNDRRTPSTEKKVDLNADPLCERKEWNKQKLPCSENPRDTEDVPWITLNSSIQKVNEWFSRSDELLGSDDSHDGESESNAKVADVLDVLNEVDEYSGSSEKIDLLASDPHEALICKSERVHSKSVESNIEDKIFGKTYRKKASLPNLSHVTENLIIGAFVTEPQIIQERPLTNKLKRKRRPTSGLHPEDFIKKADLAVQKTPEMINQGTNQTEQNGQVMNITNSGHENKTKGDSIQNEKNPNPIESLEKESAFKTKAEPISSSISNMELELNIHNSKAPKKNRLRRKSSTRHIHALELVVSRNLSPPNCTELQIDSCSSSEEIKKKKYNQMPVRHSRNLQLMEGKEPATGAKKSNKPNEQTSKRHDSDTFPELKLTNAPGSFTKCSNTSELKEFVNPSLPREEKEEKLETVKVSNNAEDPKDLMLSGERVLQTERSVESSSISLVPGTDYGTQESISLLEVSTLGKAKTEPNKCVSQCAAFENPKGLIHGCSKDNRNDTEGFKYPLGHEVNHSRETSIEMEESELDAQYLQNTFKVSKRQSFAPFSNPGNAEEECATFSAHSGSLKKQSPKVTFECEQKEENQGKNESNIKPVQTVNITAGFPVVGQKDKPVDNAKCSIKGGSRFCLSSQFRGNETGLITPNKHGLLQNPYRIPPLFPIKSFVKTKCKKNLLEENFEEHSMSPEREMGNENIPSTVSTISRNNIRENVFKEASSSNINEVGSSTNEVGSSINEIGSSDENIQAELGRNRGPKLNAMLRLGVLQPEVYKQSLPGSNCKHPEIKKQEYEEVVQTVNTDFSPYLISDNLEQPMGSSHASQVCSETPDDLLDDGEIKEDTSFAENDIKESSAVFSKSVQKGELSRSPSPFTHTHLAQGYRRGAKKLESSEENLSSEDEELPCFQHLLFGKVNNIPSQSTRHSTVATECLSKNTEENLLSLKNSLNDCSNQVILAKASQEHHLSEETKCSASLFSSQCSELEDLTANTNTQDPFLIGSSKQMRHQSESQGVGLSDKELVSDDEERGTGLEENNQEEQSMDSNLGEAASGCESETSVSEDCSGLSSQSDILTTQQRDTMQHNLIKLQQEMAELEAVLEQHGSQPSNSYPSIISDSSALEDLRNPEQSTSEKAVLTSQKSSEYPISQNPEGLSADKFEVSADSSTSKNKEPGVERSSPSKCPSLDDRWYMHSCSGSLQNRNYPSQEELIKVVDVEEQQLEESGPHDLTETSYLPRQDLEGTPYLESGISLFSDDPESDPSEDRAPESARVGNIPSSTSALKVPQLKVAESAQSPAAAHTTDTAGYNAMEESVSREKPELTASTERVNKRMSMVVSGLTPEEFMLVYKFARKHHITLTNLITEETTHVVMKTDAEFVCERTLKYFLGIAGGKWVVSYFWVTQSIKERKMLNEHDFEVRGDVVNGRNHQGPKRARESQDRKIFRGLEICCYGPFTNMPTDQLEWMVQLCGASVVKELSSFTLGTGVHPIVVVQPDAWTEDNGFHAIGQMCEAPVVTREWVLDSVALYQCQELDTYLIPQIPHSHY'],
|
430 |
+
['CALM1_HUMAN' ,'Human', 'MADQLTEEQIAEFKEAFSLFDKDGDGTITTKELGTVMRSLGQNPTEAELQDMINEVDADGNGTIDFPEFLTMMARKMKDTDSEEEIREAFRVFDKDGNGYISAAELRHVMTNLGEKLTDEEVDEMIREADIDGDGQVNYEEFVQMMTAK'],
|
431 |
+
['CCDB_ECOLI' ,'Prokaryote', 'MQFKVYTYKRESRYRLFVDVQSDIIDTPGRRMVIPLASARLLSDKVSRELYPVVHIGDESWRMMTTDMASVPVSVIGEEVADLSHRENDIKNAINLMFWGI'],
|
432 |
+
['GFP_AEQVI' ,'Other eukaryote', 'MSKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFICTTGKLPVPWPTLVTTLSYGVQCFSRYPDHMKQHDFFKSAMPEGYVQERTIFFKDDGNYKTRAEVKFEGDTLVNRIELKGIDFKEDGNILGHKLEYNYNSHNVYIMADKQKNGIKVNFKIRHNIEDGSVQLADHYQQNTPIGDGPVLLPDNHYLSTQSALSKDPNEKRDHMVLLEFVTAAGITHGMDELYK'],
|
433 |
+
['GRB2_HUMAN' ,'Human', 'MEAIAKYDFKATADDELSFKRGDILKVLNEECDQNWYKAELNGKDGFIPKNYIEMKPHPWFFGKIPRAKAEEMLSKQRHDGAFLIRESESAPGDFSLSVKFGNDVQHFKVLRDGAGKYFLWVVKFNSLNELVDYHRSTSVSRNQQIFLRDIEQVPQQPTYVQALFDFDPQEDGELGFRRGDFIHVMDNSDPNWWKGACHGQTGMFPRNYVTPVNRNV'],
|
434 |
+
],
|
435 |
+
)
|
436 |
+
|
437 |
+
gr.Markdown("<br>")
|
438 |
+
gr.Markdown("# Fitness predictions for all single amino acid substitutions in mutation range")
|
439 |
+
gr.Markdown("Inference may take a few seconds for short proteins & mutation ranges to several minutes for longer ones")
|
440 |
+
output_image = gr.Gallery(label="Fitness predictions for all single amino acid substitutions in mutation range") #Using Gallery to break down large scoring matrices into smaller images
|
441 |
+
|
442 |
+
output_recommendations = gr.Textbox(label="Mutation recommendations")
|
443 |
+
|
444 |
+
with gr.Row():
|
445 |
+
gr.Markdown("## Download CSV Files")
|
446 |
+
output_csv_files = gr.File(label="Download CSV files with fitness scores", file_count="multiple", interactive=False)
|
447 |
+
|
448 |
+
clear_button.click(
|
449 |
+
inputs = [protein_sequence_input,mutation_range_start,mutation_range_end],
|
450 |
+
outputs = [protein_sequence_input,mutation_range_start,mutation_range_end],
|
451 |
+
fn=clear_inputs
|
452 |
+
)
|
453 |
+
run_button.click(
|
454 |
+
fn=score_and_create_matrix_all_singles,
|
455 |
+
inputs=[protein_sequence_input,mutation_range_start,mutation_range_end,model_size_selection,scoring_mirror,batch_size_inference],
|
456 |
+
outputs=[output_image,output_recommendations,output_csv_files],
|
457 |
+
)
|
458 |
+
|
459 |
+
gr.Markdown("# Mutate the starting protein sequence")
|
460 |
+
with gr.Row():
|
461 |
+
mutation_triplet = gr.Textbox(lines=1,label="Selected mutation", placeholder = "Input the mutation triplet for the selected mutation (eg., M1A)")
|
462 |
+
mutate_button = gr.Button(value="Apply mutation to starting protein", variant="primary")
|
463 |
+
mutated_protein_sequence = gr.Textbox(lines=1,label="Mutated protein sequence")
|
464 |
+
mutate_button.click(
|
465 |
+
fn = get_mutated_protein,
|
466 |
+
inputs = [protein_sequence_input,mutation_triplet],
|
467 |
+
outputs = mutated_protein_sequence
|
468 |
+
)
|
469 |
+
|
470 |
+
gr.Markdown("<p>You may now use the output mutated sequence above as the starting sequence for another round of in silico directed evolution.</p>")
|
471 |
+
gr.Markdown("### About BASIS-China iGEM Team")
|
472 |
+
gr.Markdown("We are a high school synthetic biology team participating in the International Genetically Engineered Machine (iGEM) competition. Our 2025 project focuses on protein engineering and computational biology applications. This Tranception deployment is part of our broader effort to make advanced protein design tools accessible to the synthetic biology community.")
|
473 |
+
gr.Markdown("### About Tranception")
|
474 |
+
gr.Markdown("<p><b>Tranception: Protein Fitness Prediction with Autoregressive Transformers and Inference-time Retrieval</b><br>Pascal Notin, Mafalda Dias, Jonathan Frazer, Javier Marchena-Hurtado, Aidan N. Gomez, Debora S. Marks<sup>*</sup>, Yarin Gal<sup>*</sup><br><sup>* equal senior authorship</sup></p>")
|
475 |
+
gr.Markdown("Links: <a href='https://proceedings.mlr.press/v162/notin22a.html' target='_blank'>Paper</a> <a href='https://github.com/OATML-Markslab/Tranception' target='_blank'>Code</a> <a href='https://sites.google.com/view/proteingym/substitutions' target='_blank'>ProteinGym</a> <a href='https://igem.org/teams/5247' target='_blank'>BASIS-China iGEM Team</a>")
|
476 |
+
|
477 |
+
if __name__ == "__main__":
|
478 |
+
# Configure queue for better resource management
|
479 |
+
tranception_design.queue(
|
480 |
+
max_size=10, # Limit queue size
|
481 |
+
status_update_rate="auto", # Show status updates
|
482 |
+
api_open=False # Disable API to prevent external requests
|
483 |
+
)
|
484 |
+
|
485 |
+
# Launch with settings optimized for HF Spaces
|
486 |
+
try:
|
487 |
+
tranception_design.launch(
|
488 |
+
max_threads=2, # Limit concurrent threads
|
489 |
+
show_error=True,
|
490 |
+
server_name="0.0.0.0",
|
491 |
+
server_port=7860,
|
492 |
+
quiet=False, # Show all logs
|
493 |
+
prevent_thread_lock=True # Prevent thread locking issues
|
494 |
+
)
|
495 |
+
except Exception as e:
|
496 |
+
print(f"Launch error: {e}")
|
497 |
+
# If launch fails, try again with minimal settings
|
498 |
+
tranception_design.launch()
|
healthcheck.py
ADDED
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Health check script for Tranception app on Hugging Face Spaces
|
4 |
+
"""
|
5 |
+
import os
|
6 |
+
import sys
|
7 |
+
import torch
|
8 |
+
|
9 |
+
def check_environment():
|
10 |
+
"""Check if the environment is properly configured"""
|
11 |
+
print("=== Tranception Health Check ===")
|
12 |
+
|
13 |
+
# Check Python version
|
14 |
+
print(f"Python version: {sys.version}")
|
15 |
+
|
16 |
+
# Check PyTorch
|
17 |
+
print(f"PyTorch version: {torch.__version__}")
|
18 |
+
print(f"CUDA available: {torch.cuda.is_available()}")
|
19 |
+
if torch.cuda.is_available():
|
20 |
+
print(f"CUDA version: {torch.version.cuda}")
|
21 |
+
print(f"GPU: {torch.cuda.get_device_name(0)}")
|
22 |
+
|
23 |
+
# Check environment variables
|
24 |
+
print(f"\nEnvironment variables:")
|
25 |
+
print(f"DISABLE_ZERO_GPU: {os.environ.get('DISABLE_ZERO_GPU', 'not set')}")
|
26 |
+
print(f"SPACE_ID: {os.environ.get('SPACE_ID', 'not set')}")
|
27 |
+
|
28 |
+
# Check if running on Hugging Face Spaces
|
29 |
+
if os.environ.get('SPACE_ID'):
|
30 |
+
print("\nRunning on Hugging Face Spaces")
|
31 |
+
|
32 |
+
# Try to import spaces module
|
33 |
+
try:
|
34 |
+
import spaces
|
35 |
+
print("✓ spaces module available")
|
36 |
+
# Try to create a GPU decorator
|
37 |
+
try:
|
38 |
+
test_decorator = spaces.GPU()
|
39 |
+
print("✓ Zero GPU decorator can be created")
|
40 |
+
except Exception as e:
|
41 |
+
print(f"✗ Zero GPU decorator error: {e}")
|
42 |
+
except ImportError:
|
43 |
+
print("✗ spaces module not available")
|
44 |
+
else:
|
45 |
+
print("\nNot running on Hugging Face Spaces")
|
46 |
+
|
47 |
+
# Check model files
|
48 |
+
print(f"\nChecking model availability on Hugging Face Hub:")
|
49 |
+
models = ["Tranception_Small", "Tranception_Medium", "Tranception_Large"]
|
50 |
+
for model in models:
|
51 |
+
print(f"- PascalNotin/{model}: Available on HF Hub")
|
52 |
+
|
53 |
+
print("\n=== Health check complete ===")
|
54 |
+
|
55 |
+
if __name__ == "__main__":
|
56 |
+
check_environment()
|
keep_alive_cron.sh
ADDED
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
# Keep-alive script for Tranception Hugging Face Space
|
3 |
+
# Add to crontab: */5 * * * * /path/to/keep_alive_cron.sh
|
4 |
+
|
5 |
+
SPACE_URL="https://huggingface.co/spaces/MoraxCheng/Transeption_iGEM_BASISCHINA_2025"
|
6 |
+
LOG_FILE="/var/log/tranception_keep_alive.log"
|
7 |
+
|
8 |
+
# Function to log with timestamp
|
9 |
+
log_message() {
|
10 |
+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >> "$LOG_FILE"
|
11 |
+
}
|
12 |
+
|
13 |
+
# Ping the Space
|
14 |
+
response=$(curl -s -o /dev/null -w "%{http_code}" "$SPACE_URL" --max-time 30)
|
15 |
+
|
16 |
+
if [ "$response" = "200" ]; then
|
17 |
+
log_message "SUCCESS: Space is alive (HTTP $response)"
|
18 |
+
else
|
19 |
+
log_message "WARNING: Space returned HTTP $response"
|
20 |
+
|
21 |
+
# Try to wake it up with a simple request
|
22 |
+
curl -s -X POST "$SPACE_URL/api/predict" \
|
23 |
+
-H "Content-Type: application/json" \
|
24 |
+
-d '{"fn_index": 0, "data": ["MSKGEELFT", 1, 5, "Small", false, 10]}' \
|
25 |
+
--max-time 60 > /dev/null
|
26 |
+
|
27 |
+
log_message "Sent wake-up request"
|
28 |
+
fi
|
29 |
+
|
30 |
+
# Keep log file size under control (max 1MB)
|
31 |
+
if [ -f "$LOG_FILE" ] && [ $(stat -f%z "$LOG_FILE" 2>/dev/null || stat -c%s "$LOG_FILE" 2>/dev/null) -gt 1048576 ]; then
|
32 |
+
tail -n 1000 "$LOG_FILE" > "$LOG_FILE.tmp"
|
33 |
+
mv "$LOG_FILE.tmp" "$LOG_FILE"
|
34 |
+
log_message "Log file rotated"
|
35 |
+
fi
|
manual_keep_alive.py
ADDED
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Manual keep-alive script for Tranception Space
|
4 |
+
Run this locally to keep your Space active
|
5 |
+
"""
|
6 |
+
import requests
|
7 |
+
import time
|
8 |
+
from datetime import datetime
|
9 |
+
|
10 |
+
SPACE_URL = "https://huggingface.co/spaces/MoraxCheng/Transeption_iGEM_BASISCHINA_2025"
|
11 |
+
PING_INTERVAL = 300 # 5 minutes
|
12 |
+
|
13 |
+
def ping_space():
|
14 |
+
"""Ping the Space to keep it alive"""
|
15 |
+
try:
|
16 |
+
print(f"[{datetime.now()}] Pinging Space...")
|
17 |
+
response = requests.get(SPACE_URL, timeout=30)
|
18 |
+
|
19 |
+
if response.status_code == 200:
|
20 |
+
print(f"✓ Space is alive (HTTP {response.status_code})")
|
21 |
+
return True
|
22 |
+
else:
|
23 |
+
print(f"⚠ Space returned HTTP {response.status_code}")
|
24 |
+
return False
|
25 |
+
except requests.exceptions.Timeout:
|
26 |
+
print("⚠ Request timed out - Space might be starting up")
|
27 |
+
return False
|
28 |
+
except Exception as e:
|
29 |
+
print(f"✗ Error: {e}")
|
30 |
+
return False
|
31 |
+
|
32 |
+
def main():
|
33 |
+
print(f"Starting keep-alive for: {SPACE_URL}")
|
34 |
+
print(f"Ping interval: {PING_INTERVAL} seconds")
|
35 |
+
print("Press Ctrl+C to stop\n")
|
36 |
+
|
37 |
+
while True:
|
38 |
+
try:
|
39 |
+
ping_space()
|
40 |
+
print(f"Next ping in {PING_INTERVAL} seconds...\n")
|
41 |
+
time.sleep(PING_INTERVAL)
|
42 |
+
except KeyboardInterrupt:
|
43 |
+
print("\nStopped by user")
|
44 |
+
break
|
45 |
+
|
46 |
+
if __name__ == "__main__":
|
47 |
+
main()
|
monitor_space.py
ADDED
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Monitor and keep alive your Hugging Face Space
|
4 |
+
Run this on your local computer or a server
|
5 |
+
"""
|
6 |
+
import requests
|
7 |
+
import time
|
8 |
+
from datetime import datetime
|
9 |
+
import json
|
10 |
+
|
11 |
+
# Your Space details
|
12 |
+
SPACE_URL = "https://huggingface.co/spaces/MoraxCheng/Transeption_iGEM_BASISCHINA_2025"
|
13 |
+
EMBED_URL = "https://moraxcheng-transeption-igem-basischina-2025.hf.space"
|
14 |
+
API_URL = f"{EMBED_URL}/api/predict"
|
15 |
+
|
16 |
+
# Monitoring settings
|
17 |
+
CHECK_INTERVAL = 300 # 5 minutes
|
18 |
+
WAKE_UP_TIMEOUT = 180 # 3 minutes max wait for wake up
|
19 |
+
|
20 |
+
def check_space_status():
|
21 |
+
"""Check if Space is responding"""
|
22 |
+
try:
|
23 |
+
print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Checking Space status...")
|
24 |
+
response = requests.get(EMBED_URL, timeout=10)
|
25 |
+
|
26 |
+
if response.status_code == 200:
|
27 |
+
print("✓ Space is online")
|
28 |
+
return True
|
29 |
+
else:
|
30 |
+
print(f"⚠ Space returned status {response.status_code}")
|
31 |
+
return False
|
32 |
+
except requests.exceptions.Timeout:
|
33 |
+
print("⚠ Request timed out - Space might be sleeping")
|
34 |
+
return False
|
35 |
+
except Exception as e:
|
36 |
+
print(f"✗ Error checking status: {e}")
|
37 |
+
return False
|
38 |
+
|
39 |
+
def wake_up_space():
|
40 |
+
"""Send a minimal request to wake up the Space"""
|
41 |
+
try:
|
42 |
+
print("Attempting to wake up Space...")
|
43 |
+
|
44 |
+
# Send a minimal prediction request
|
45 |
+
payload = {
|
46 |
+
"fn_index": 0,
|
47 |
+
"data": [
|
48 |
+
"MSKGE", # Minimal sequence
|
49 |
+
1, # Start position
|
50 |
+
2, # End position
|
51 |
+
"Small", # Model size
|
52 |
+
False, # No mirror scoring
|
53 |
+
1 # Minimal batch size
|
54 |
+
]
|
55 |
+
}
|
56 |
+
|
57 |
+
headers = {
|
58 |
+
"Content-Type": "application/json",
|
59 |
+
}
|
60 |
+
|
61 |
+
response = requests.post(
|
62 |
+
API_URL,
|
63 |
+
json=payload,
|
64 |
+
headers=headers,
|
65 |
+
timeout=30
|
66 |
+
)
|
67 |
+
|
68 |
+
if response.status_code in [200, 202]:
|
69 |
+
print("✓ Wake-up request sent successfully")
|
70 |
+
return True
|
71 |
+
else:
|
72 |
+
print(f"⚠ Wake-up request returned status {response.status_code}")
|
73 |
+
return False
|
74 |
+
|
75 |
+
except Exception as e:
|
76 |
+
print(f"✗ Error sending wake-up request: {e}")
|
77 |
+
return False
|
78 |
+
|
79 |
+
def wait_for_space_ready(max_wait=WAKE_UP_TIMEOUT):
|
80 |
+
"""Wait for Space to become ready"""
|
81 |
+
print(f"Waiting for Space to become ready (max {max_wait}s)...")
|
82 |
+
start_time = time.time()
|
83 |
+
|
84 |
+
while time.time() - start_time < max_wait:
|
85 |
+
if check_space_status():
|
86 |
+
print(f"✓ Space is ready after {int(time.time() - start_time)}s")
|
87 |
+
return True
|
88 |
+
|
89 |
+
print(".", end="", flush=True)
|
90 |
+
time.sleep(10)
|
91 |
+
|
92 |
+
print(f"\n✗ Space did not become ready after {max_wait}s")
|
93 |
+
return False
|
94 |
+
|
95 |
+
def monitor_loop():
|
96 |
+
"""Main monitoring loop"""
|
97 |
+
print("="*60)
|
98 |
+
print("Hugging Face Space Monitor")
|
99 |
+
print(f"Space: {SPACE_URL}")
|
100 |
+
print(f"Check interval: {CHECK_INTERVAL}s")
|
101 |
+
print("Press Ctrl+C to stop")
|
102 |
+
print("="*60)
|
103 |
+
|
104 |
+
consecutive_failures = 0
|
105 |
+
|
106 |
+
while True:
|
107 |
+
try:
|
108 |
+
# Check if Space is alive
|
109 |
+
if check_space_status():
|
110 |
+
consecutive_failures = 0
|
111 |
+
print(f"Next check in {CHECK_INTERVAL}s...\n")
|
112 |
+
else:
|
113 |
+
consecutive_failures += 1
|
114 |
+
print(f"Space appears to be down (failure #{consecutive_failures})")
|
115 |
+
|
116 |
+
# Try to wake it up
|
117 |
+
if wake_up_space():
|
118 |
+
# Wait for it to become ready
|
119 |
+
if wait_for_space_ready():
|
120 |
+
consecutive_failures = 0
|
121 |
+
print("✓ Space successfully revived!\n")
|
122 |
+
else:
|
123 |
+
print("✗ Failed to revive Space\n")
|
124 |
+
else:
|
125 |
+
print("✗ Could not send wake-up request\n")
|
126 |
+
|
127 |
+
if consecutive_failures >= 3:
|
128 |
+
print("⚠️ ATTENTION: Space has been down for multiple checks!")
|
129 |
+
print("You may need to manually restart it from the Hugging Face interface.")
|
130 |
+
print(f"Go to: {SPACE_URL}/settings\n")
|
131 |
+
|
132 |
+
# Wait before next check
|
133 |
+
time.sleep(CHECK_INTERVAL)
|
134 |
+
|
135 |
+
except KeyboardInterrupt:
|
136 |
+
print("\n\nMonitoring stopped by user")
|
137 |
+
break
|
138 |
+
except Exception as e:
|
139 |
+
print(f"\nUnexpected error: {e}")
|
140 |
+
print("Continuing monitoring...\n")
|
141 |
+
time.sleep(60)
|
142 |
+
|
143 |
+
if __name__ == "__main__":
|
144 |
+
# Test connection first
|
145 |
+
print("Testing connection to Space...")
|
146 |
+
if check_space_status():
|
147 |
+
print("✓ Initial connection successful\n")
|
148 |
+
else:
|
149 |
+
print("⚠ Space appears to be sleeping, attempting to wake...")
|
150 |
+
if wake_up_space() and wait_for_space_ready():
|
151 |
+
print("✓ Space is now ready\n")
|
152 |
+
else:
|
153 |
+
print("✗ Could not wake Space. It may need manual restart.\n")
|
154 |
+
|
155 |
+
# Start monitoring
|
156 |
+
monitor_loop()
|
setup_cron.sh
ADDED
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
# Setup script for keep-alive cron job
|
3 |
+
|
4 |
+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
5 |
+
CRON_SCRIPT="$SCRIPT_DIR/keep_alive_cron.sh"
|
6 |
+
|
7 |
+
# Make the script executable
|
8 |
+
chmod +x "$CRON_SCRIPT"
|
9 |
+
|
10 |
+
# Add to crontab (runs every 5 minutes)
|
11 |
+
(crontab -l 2>/dev/null; echo "*/5 * * * * $CRON_SCRIPT") | crontab -
|
12 |
+
|
13 |
+
echo "Keep-alive cron job installed!"
|
14 |
+
echo "It will ping your Space every 5 minutes."
|
15 |
+
echo ""
|
16 |
+
echo "To view current cron jobs: crontab -l"
|
17 |
+
echo "To remove the cron job: crontab -e (then delete the line)"
|
18 |
+
echo ""
|
19 |
+
echo "Space URL: https://huggingface.co/spaces/MoraxCheng/Transeption_iGEM_BASISCHINA_2025"
|
zero_gpu_monitor.py
ADDED
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Zero GPU Monitor - Keeps the Space alive and monitors health
|
4 |
+
"""
|
5 |
+
import os
|
6 |
+
import time
|
7 |
+
import requests
|
8 |
+
import sys
|
9 |
+
from datetime import datetime
|
10 |
+
|
11 |
+
# Configuration
|
12 |
+
SPACE_URL = os.environ.get('SPACE_URL', 'http://localhost:7860')
|
13 |
+
CHECK_INTERVAL = 180 # 3 minutes
|
14 |
+
HEALTH_ENDPOINT = '/health'
|
15 |
+
MAX_FAILURES = 3
|
16 |
+
|
17 |
+
def check_space_health():
|
18 |
+
"""Check if the Space is responding"""
|
19 |
+
try:
|
20 |
+
response = requests.get(SPACE_URL, timeout=10)
|
21 |
+
return response.status_code == 200
|
22 |
+
except Exception as e:
|
23 |
+
print(f"Health check failed: {e}")
|
24 |
+
return False
|
25 |
+
|
26 |
+
def keep_space_warm():
|
27 |
+
"""Send a dummy request to keep the Space warm"""
|
28 |
+
try:
|
29 |
+
# Send a simple request to the API
|
30 |
+
payload = {
|
31 |
+
"fn_index": 0,
|
32 |
+
"data": ["MSKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFICTTGKLPVPWPTLVTTLSYGVQCFSRYPDHMKQHDFFKSAMPEGYVQERTIFFKDDGNYKTRAEVKFEGDTLVNRIELKGIDFKEDGNILGHKLEYNYNSHNVYIMADKQKNGIKVNFKIRHNIEDGSVQLADHYQQNTPIGDGPVLLPDNHYLSTQSALSKDPNEKRDHMVLLEFVTAAGITHGMDELYK", 1, 10, "Small", False, 10]
|
33 |
+
}
|
34 |
+
response = requests.post(f"{SPACE_URL}/api/predict", json=payload, timeout=30)
|
35 |
+
return response.status_code in [200, 202]
|
36 |
+
except Exception as e:
|
37 |
+
print(f"Keep-warm request failed: {e}")
|
38 |
+
return False
|
39 |
+
|
40 |
+
def monitor_loop():
|
41 |
+
"""Main monitoring loop"""
|
42 |
+
print(f"Starting Zero GPU Space monitor...")
|
43 |
+
print(f"Space URL: {SPACE_URL}")
|
44 |
+
print(f"Check interval: {CHECK_INTERVAL} seconds")
|
45 |
+
|
46 |
+
consecutive_failures = 0
|
47 |
+
last_warm_up = datetime.now()
|
48 |
+
|
49 |
+
while True:
|
50 |
+
try:
|
51 |
+
current_time = datetime.now()
|
52 |
+
print(f"\n[{current_time}] Performing health check...")
|
53 |
+
|
54 |
+
# Check if Space is healthy
|
55 |
+
if check_space_health():
|
56 |
+
print("✓ Space is healthy")
|
57 |
+
consecutive_failures = 0
|
58 |
+
|
59 |
+
# Send keep-warm request every check
|
60 |
+
time_since_warmup = (current_time - last_warm_up).total_seconds()
|
61 |
+
if time_since_warmup > CHECK_INTERVAL:
|
62 |
+
print("Sending keep-warm request...")
|
63 |
+
if keep_space_warm():
|
64 |
+
print("✓ Keep-warm successful")
|
65 |
+
last_warm_up = current_time
|
66 |
+
else:
|
67 |
+
print("✗ Keep-warm failed")
|
68 |
+
else:
|
69 |
+
consecutive_failures += 1
|
70 |
+
print(f"✗ Space is not responding (failure {consecutive_failures}/{MAX_FAILURES})")
|
71 |
+
|
72 |
+
if consecutive_failures >= MAX_FAILURES:
|
73 |
+
print("ERROR: Space appears to be down!")
|
74 |
+
print("Please restart the Space from Hugging Face interface")
|
75 |
+
# Could add notification logic here
|
76 |
+
|
77 |
+
# Wait before next check
|
78 |
+
time.sleep(CHECK_INTERVAL)
|
79 |
+
|
80 |
+
except KeyboardInterrupt:
|
81 |
+
print("\nMonitor stopped by user")
|
82 |
+
break
|
83 |
+
except Exception as e:
|
84 |
+
print(f"Monitor error: {e}")
|
85 |
+
time.sleep(60) # Wait a minute before retrying
|
86 |
+
|
87 |
+
if __name__ == "__main__":
|
88 |
+
# Get Space URL from environment or command line
|
89 |
+
if len(sys.argv) > 1:
|
90 |
+
SPACE_URL = sys.argv[1]
|
91 |
+
|
92 |
+
monitor_loop()
|