Tim Luka Horstmann commited on
Commit
e112ae1
·
1 Parent(s): 3bbf0cd

Fix contxt

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -150,7 +150,7 @@ async def stream_response(query, history):
150
 
151
  total_tokens = system_tokens + query_tokens + sum(history_tokens) + len(history) * 10 + 10 # Rough estimate for formatting
152
 
153
- max_allowed_tokens = generator.n_ctx - 512 - 100 # max_tokens=512, safety_margin=100
154
 
155
  while total_tokens > max_allowed_tokens and history:
156
  removed_msg = history.pop(0)
 
150
 
151
  total_tokens = system_tokens + query_tokens + sum(history_tokens) + len(history) * 10 + 10 # Rough estimate for formatting
152
 
153
+ max_allowed_tokens = generator.n_ctx() - 512 - 100 # max_tokens=512, safety_margin=100
154
 
155
  while total_tokens > max_allowed_tokens and history:
156
  removed_msg = history.pop(0)