Update app.py

app.py

@@ -6,6 +6,7 @@ import logging
 from typing import Tuple, Literal
 import functools
 import spaces
+from cashews import cache
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -16,6 +17,8 @@ model = None
 tokenizer = None
 device = None
 
+cache.setup("mem://", size_limit="4gb")
+
 def load_model():
     global model, tokenizer, device
     logger.info("Loading model and tokenizer...")
@@ -48,8 +51,8 @@ def get_card_info(hub_id: str) -> Tuple[str, str]:
             raise ValueError(f"Could not find model or dataset with id {hub_id}")
 
 @spaces.GPU
-def generate_summary(card_text: str, card_type: str) -> str:
-    """Generate a summary for the given card text."""
+def _generate_summary_gpu(card_text: str, card_type: str) -> str:
+    """Internal function that runs on GPU."""
     # Determine prefix based on card type
     prefix = "<MODEL_CARD>" if card_type == "model" else "<DATASET_CARD>"
 
@@ -84,6 +87,11 @@ def generate_summary(card_text: str, card_type: str) -> str:
 
     return summary
 
+@cache(ttl="6h")
+def generate_summary(card_text: str, card_type: str) -> str:
+    """Cached wrapper for generate_summary."""
+    return _generate_summary_gpu(card_text, card_type)
+
 def summarize(hub_id: str = "", card_type: str = "model", content: str = "") -> str:
     """Interface function for Gradio."""
     try:
@@ -100,6 +108,7 @@ def summarize(hub_id: str = "", card_type: str = "model", content: str = "") ->
         else:
             return "Error: Either hub_id or content must be provided"
 
+        # Use the cached wrapper
         summary = generate_summary(card_text, card_type)
         return summary