llm_model.py

import torch, traceback
from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification
from log import log
from pydantic import BaseModel

_model = None
_tokenizer = None
_eos_token_id = None

def get_model():
    return _model

def get_tokenizer():
    return _tokenizer

def get_eos_token_id():
    return _eos_token_id

class Message(BaseModel):
    user_input: str

def setup_model(s_config):
    global _model, _tokenizer, _eos_token_id
    try:
        log("🧠 setup_model() başladı")
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        log(f"📡 Kullanılan cihaz: {device}")
        _tokenizer = AutoTokenizer.from_pretrained(s_config.MODEL_BASE, use_fast=False)
        log("📦 Tokenizer yüklendi. Ana model indiriliyor...")
        _model = AutoModelForCausalLM.from_pretrained(s_config.MODEL_BASE, torch_dtype=torch.float32).to(device)
        log("📦 Ana model indirildi ve yüklendi. eval() çağırılıyor...")
        _tokenizer.pad_token = _tokenizer.pad_token or _tokenizer.eos_token
        _model.config.pad_token_id = _tokenizer.pad_token_id
        _eos_token_id = _tokenizer("<|im_end|>", add_special_tokens=False)["input_ids"][0]
        _model.eval()
        log("✅ Ana model eval() çağrıldı")
        log(f"📦 Intent modeli indiriliyor: {s_config.INTENT_MODEL_ID}")
        _ = AutoTokenizer.from_pretrained(s_config.INTENT_MODEL_ID)
        _ = AutoModelForSequenceClassification.from_pretrained(s_config.INTENT_MODEL_ID)
        log("✅ Intent modeli önbelleğe alındı.")
        log("✔️ Model başarıyla yüklendi ve sohbet için hazır.")
    except Exception as e:
        log(f"❌ setup_model() hatası: {e}")
        traceback.print_exc()

async def generate_response(text, app_config):
    model = get_model()
    tokenizer = get_tokenizer()
    eos_token_id = get_eos_token_id()

    messages = [{"role": "user", "content": text}]
    encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True)
    input_ids = encodeds.to(model.device)
    attention_mask = (input_ids != tokenizer.pad_token_id).long()

    with torch.no_grad():
        output = model.generate(
            input_ids=input_ids,
            attention_mask=attention_mask,
            max_new_tokens=128,
            do_sample=app_config.USE_SAMPLING,
            eos_token_id=eos_token_id,
            pad_token_id=tokenizer.pad_token_id,
            return_dict_in_generate=True,
            output_scores=True
        )

    if not app_config.USE_SAMPLING:
        scores = torch.stack(output.scores, dim=1)
        probs = torch.nn.functional.softmax(scores[0], dim=-1)
        top_conf = probs.max().item()
    else:
        top_conf = None

    decoded = tokenizer.decode(output.sequences[0], skip_special_tokens=True).strip()
    for tag in ["assistant", "<|im_start|>assistant"]:
        start = decoded.find(tag)
        if start != -1:
            decoded = decoded[start + len(tag):].strip()
            break
    return decoded, top_conf