handraise-dev
/

gguf-inference

Text Generation

Model card Files Files and versions Community

syberWolf commited on Jul 4, 2024

Commit

52afa01

·

1 Parent(s): eff3ac4

forgot comma

Files changed (1) hide show

handler.py +1 -1

handler.py CHANGED Viewed

@@ -10,7 +10,7 @@ class EndpointHandler:
         tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-1.5B-Instruct")
         model = AutoModelForCausalLM.from_pretrained(
             "Qwen/Qwen2-1.5B-Instruct",
-            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
             device_map="cuda" # for single instance one GPU
         )

         tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-1.5B-Instruct")
         model = AutoModelForCausalLM.from_pretrained(
             "Qwen/Qwen2-1.5B-Instruct",
+            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
             device_map="cuda" # for single instance one GPU
         )