Tim Luka Horstmann
commited on
Commit
·
e112ae1
1
Parent(s):
3bbf0cd
Fix contxt
Browse files
app.py
CHANGED
@@ -150,7 +150,7 @@ async def stream_response(query, history):
|
|
150 |
|
151 |
total_tokens = system_tokens + query_tokens + sum(history_tokens) + len(history) * 10 + 10 # Rough estimate for formatting
|
152 |
|
153 |
-
max_allowed_tokens = generator.n_ctx - 512 - 100 # max_tokens=512, safety_margin=100
|
154 |
|
155 |
while total_tokens > max_allowed_tokens and history:
|
156 |
removed_msg = history.pop(0)
|
|
|
150 |
|
151 |
total_tokens = system_tokens + query_tokens + sum(history_tokens) + len(history) * 10 + 10 # Rough estimate for formatting
|
152 |
|
153 |
+
max_allowed_tokens = generator.n_ctx() - 512 - 100 # max_tokens=512, safety_margin=100
|
154 |
|
155 |
while total_tokens > max_allowed_tokens and history:
|
156 |
removed_msg = history.pop(0)
|