maybe fixed qwen3

utils/models.py

@@ -3,7 +3,7 @@ os.environ['MKL_THREADING_LAYER'] = 'GNU'
 import spaces
 
 import torch
-from transformers import pipeline, AutoTokenizer, StoppingCriteria, StoppingCriteriaList
+from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM, StoppingCriteria, StoppingCriteriaList
 from .prompts import format_rag_prompt
 from .shared import generation_interrupt
 
@@ -142,7 +142,7 @@ def run_inference(model_name, context, question):
             "text-generation",
             model=model_name,
             tokenizer=tokenizer,
-            device_map='auto',
+            device_map='cuda',
             trust_remote_code=True,
             torch_dtype=torch.bfloat16,
         )
@@ -153,16 +153,15 @@ def run_inference(model_name, context, question):
             tokenize=False,
             **tokenizer_kwargs,
         )
-            
-
-
+        
+        input_length = len(formatted)
         # Check interrupt before generation
         if generation_interrupt.is_set():
             return ""
 
-        outputs = pipe(formatted, skip_special_tokens=True, **generation_kwargs, )
+        outputs = pipe(formatted, **generation_kwargs)
         #print(outputs[0]['generated_text'])
-        result = outputs[0]['generated_text']
+        result = outputs[0]['generated_text'][input_length:]
 
     except Exception as e:
         print(f"Error in inference for {model_name}: {e}")