Update app.py

app.py

@@ -7,6 +7,36 @@ import torch
 # Load model and tokenizer
 model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
 tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+# Add this workaround for the RoPE scaling issue
+from transformers.utils import WEIGHTS_NAME, CONFIG_NAME
+import os
+import json
+
+# Fix the rope_scaling configuration before loading the model
+config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.json")
+if not os.path.exists(config_path):
+    # Download the config file if it doesn't exist
+    from huggingface_hub import hf_hub_download
+    config_path = hf_hub_download(repo_id=model_id, filename=CONFIG_NAME)
+
+# Load and modify the config
+with open(config_path, 'r') as f:
+    config = json.load(f)
+
+# Fix the rope_scaling format
+if 'rope_scaling' in config and not (isinstance(config['rope_scaling'], dict) and 'type' in config['rope_scaling'] and 'factor' in config['rope_scaling']):
+    # Convert to the expected format
+    old_scaling = config['rope_scaling']
+    config['rope_scaling'] = {
+        'type': 'dynamic',
+        'factor': old_scaling.get('factor', 8.0)
+    }
+    # Save the modified config
+    with open(config_path, 'w') as f:
+        json.dump(config, f)
+
+# Now load the model with the fixed config
 model = AutoModelForCausalLM.from_pretrained(
     model_id, 
     torch_dtype=torch.bfloat16,