ciyidogan commited on
Commit
8c976f2
·
verified ·
1 Parent(s): 25cd8e8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +10 -3
app.py CHANGED
@@ -75,19 +75,26 @@ def generate(req: UserInputRequest):
75
  input_ids = input_data['input_ids']
76
  attention_mask = input_data['attention_mask']
77
 
 
 
 
 
 
 
 
78
  terminators = [
79
  tokenizer.eos_token_id,
80
- tokenizer.convert_tokens_to_ids("<|eot_id|>")
81
  ]
82
 
83
  outputs = model.generate(
84
  input_ids=input_ids,
85
  attention_mask=attention_mask,
86
- max_new_tokens=200,
87
  eos_token_id=terminators
88
  )
89
 
90
- response = outputs[0][input_ids.shape[-1]:]
91
  answer = tokenizer.decode(response, skip_special_tokens=True)
92
 
93
  end_time = time.time()
 
75
  input_ids = input_data['input_ids']
76
  attention_mask = input_data['attention_mask']
77
 
78
+ # === Dinamik token hesaplama
79
+ total_ctx = model.config.max_position_embeddings if hasattr(model.config, 'max_position_embeddings') else 4096
80
+ input_len = input_ids.shape[-1]
81
+ max_new_tokens = max(1, total_ctx - input_len)
82
+
83
+ log(f"ℹ️ Input uzunluğu: {input_len}, max_new_tokens ayarlandı: {max_new_tokens}")
84
+
85
  terminators = [
86
  tokenizer.eos_token_id,
87
+ tokenizer.convert_tokens_to_ids("<|eot_id|>") if "<|eot_id|>" in tokenizer.get_vocab() else tokenizer.eos_token_id
88
  ]
89
 
90
  outputs = model.generate(
91
  input_ids=input_ids,
92
  attention_mask=attention_mask,
93
+ max_new_tokens=max_new_tokens,
94
  eos_token_id=terminators
95
  )
96
 
97
+ response = outputs[0][input_len:]
98
  answer = tokenizer.decode(response, skip_special_tokens=True)
99
 
100
  end_time = time.time()