More logging

app.py

@@ -7,11 +7,25 @@ import spaces
 import torch
 import logging
 from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers.utils import logging as transformers_logging
 
-# Set up standard logging
-logging.basicConfig(level=logging.DEBUG)
+# Set up comprehensive logging
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
 logger = logging.getLogger(__name__)
 
+# Enable transformers logging
+transformers_logging.set_verbosity_debug()
+transformers_logging.enable_default_handler()
+transformers_logging.enable_explicit_format()
+
+# Enable accelerate and torch logging
+logging.getLogger("accelerate").setLevel(logging.DEBUG)
+logging.getLogger("torch").setLevel(logging.DEBUG)
+logging.getLogger("transformers").setLevel(logging.DEBUG)
+
 import huggingface_hub
 
 import prep_decompiled
@@ -36,8 +50,14 @@ print(f"GPU memory after vardecoder:")
 print(f"Allocated: {torch.cuda.memory_allocated() / 1024**3:.2f} GB")
 print(f"Reserved: {torch.cuda.memory_reserved() / 1024**3:.2f} GB")
 
+# Add more detailed debugging before loading the second model
 try:
     logger.info("Loading fielddecoder model...")
+    print(f"CUDA available: {torch.cuda.is_available()}")
+    print(f"CUDA device count: {torch.cuda.device_count()}")
+    print(f"Current device: {torch.cuda.current_device()}")
+    print(f"Device name: {torch.cuda.get_device_name()}")
+    
     fielddecoder_model = AutoModelForCausalLM.from_pretrained(
         "ejschwartz/resym-fielddecoder", 
         torch_dtype=torch.bfloat16,