Update: input tokens are on the same device as a model

model.py

@@ -18,7 +18,7 @@ async def get_answer_from_llm(question: str = None):
     messages = [{"role": "user", "content": f"{question}"}]
     input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
     gen_tokens = model.generate(
-        input_ids,
+        input_ids.to(device),
         max_new_tokens=100,
         do_sample=True,
         temperature=0.3,