add cpu

app.py

@@ -70,7 +70,7 @@ def load_model():
     tokenizer.padding_side = "right"
     
     # model = AutoModel.from_pretrained(CUR_MODEL, max_memory={"cpu": "12GiB"}, torch_dtype=torch.bfloat16, offload_state_dict=True)
-    base_model_instance = AutoModel.from_pretrained(BASE_MODEL)
+    base_model_instance = AutoModel.from_pretrained(BASE_MODEL).cpu()
     model = PeftModel.from_pretrained(base_model_instance, CUR_MODEL)
     model = model.merge_and_unload()
     model.eval()