Spaces:
Running
Running
File size: 2,924 Bytes
4b9f7d2 03f9f6e 4b9f7d2 703f171 4b9f7d2 d51553a 7557f38 4b9f7d2 d51553a 4b9f7d2 d51553a 4b9f7d2 d51553a 4b9f7d2 d51553a 4b9f7d2 d51553a 4b9f7d2 d51553a 4b9f7d2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
import torch, traceback
from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification
from log import log
from pydantic import BaseModel
model = None
tokenizer = None
eos_token_id = None
class Message(BaseModel):
user_input: str
def setup_model(s_config):
global model, tokenizer, eos_token_id
try:
log("🧠 setup_model() başladı")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
log(f"📡 Kullanılan cihaz: {device}")
tokenizer = AutoTokenizer.from_pretrained(s_config.MODEL_BASE, use_fast=False)
log("📦 Tokenizer yüklendi. Ana model indiriliyor...")
model = AutoModelForCausalLM.from_pretrained(s_config.MODEL_BASE, torch_dtype=torch.float32).to(device)
log("📦 Ana model indirildi ve yüklendi. eval() çağırılıyor...")
tokenizer.pad_token = tokenizer.pad_token or tokenizer.eos_token
model.config.pad_token_id = tokenizer.pad_token_id
eos_token_id = tokenizer("<|im_end|>", add_special_tokens=False)["input_ids"][0]
model.eval()
log("✅ Ana model eval() çağrıldı")
log(f"📦 Intent modeli indiriliyor: {s_config.INTENT_MODEL_ID}")
_ = AutoTokenizer.from_pretrained(s_config.INTENT_MODEL_ID)
_ = AutoModelForSequenceClassification.from_pretrained(s_config.INTENT_MODEL_ID)
log("✅ Intent modeli önbelleğe alındı.")
log("✔️ Model başarıyla yüklendi ve sohbet için hazır.")
except Exception as e:
log(f"❌ setup_model() hatası: {e}")
traceback.print_exc()
async def generate_response(text, app_config):
messages = [{"role": "user", "content": text}]
encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True)
eos_token = tokenizer("<|im_end|>", add_special_tokens=False)["input_ids"][0]
input_ids = encodeds.to(model.device)
attention_mask = (input_ids != tokenizer.pad_token_id).long()
with torch.no_grad():
output = model.generate(
input_ids=input_ids,
attention_mask=attention_mask,
max_new_tokens=128,
do_sample=app_config.USE_SAMPLING,
eos_token_id=eos_token,
pad_token_id=tokenizer.pad_token_id,
return_dict_in_generate=True,
output_scores=True
)
if not app_config.USE_SAMPLING:
scores = torch.stack(output.scores, dim=1)
probs = torch.nn.functional.softmax(scores[0], dim=-1)
top_conf = probs.max().item()
else:
top_conf = None
decoded = tokenizer.decode(output.sequences[0], skip_special_tokens=True).strip()
for tag in ["assistant", "<|im_start|>assistant"]:
start = decoded.find(tag)
if start != -1:
decoded = decoded[start + len(tag):].strip()
break
return decoded, top_conf |