Update app.py
Browse files
app.py
CHANGED
@@ -75,19 +75,26 @@ def generate(req: UserInputRequest):
|
|
75 |
input_ids = input_data['input_ids']
|
76 |
attention_mask = input_data['attention_mask']
|
77 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
78 |
terminators = [
|
79 |
tokenizer.eos_token_id,
|
80 |
-
tokenizer.convert_tokens_to_ids("<|eot_id|>")
|
81 |
]
|
82 |
|
83 |
outputs = model.generate(
|
84 |
input_ids=input_ids,
|
85 |
attention_mask=attention_mask,
|
86 |
-
max_new_tokens=
|
87 |
eos_token_id=terminators
|
88 |
)
|
89 |
|
90 |
-
response = outputs[0][
|
91 |
answer = tokenizer.decode(response, skip_special_tokens=True)
|
92 |
|
93 |
end_time = time.time()
|
|
|
75 |
input_ids = input_data['input_ids']
|
76 |
attention_mask = input_data['attention_mask']
|
77 |
|
78 |
+
# === Dinamik token hesaplama
|
79 |
+
total_ctx = model.config.max_position_embeddings if hasattr(model.config, 'max_position_embeddings') else 4096
|
80 |
+
input_len = input_ids.shape[-1]
|
81 |
+
max_new_tokens = max(1, total_ctx - input_len)
|
82 |
+
|
83 |
+
log(f"ℹ️ Input uzunluğu: {input_len}, max_new_tokens ayarlandı: {max_new_tokens}")
|
84 |
+
|
85 |
terminators = [
|
86 |
tokenizer.eos_token_id,
|
87 |
+
tokenizer.convert_tokens_to_ids("<|eot_id|>") if "<|eot_id|>" in tokenizer.get_vocab() else tokenizer.eos_token_id
|
88 |
]
|
89 |
|
90 |
outputs = model.generate(
|
91 |
input_ids=input_ids,
|
92 |
attention_mask=attention_mask,
|
93 |
+
max_new_tokens=max_new_tokens,
|
94 |
eos_token_id=terminators
|
95 |
)
|
96 |
|
97 |
+
response = outputs[0][input_len:]
|
98 |
answer = tokenizer.decode(response, skip_special_tokens=True)
|
99 |
|
100 |
end_time = time.time()
|