Tim Luka Horstmann
commited on
Commit
·
e112ae1
1
Parent(s):
3bbf0cd
Fix contxt
Browse files
app.py
CHANGED
|
@@ -150,7 +150,7 @@ async def stream_response(query, history):
|
|
| 150 |
|
| 151 |
total_tokens = system_tokens + query_tokens + sum(history_tokens) + len(history) * 10 + 10 # Rough estimate for formatting
|
| 152 |
|
| 153 |
-
max_allowed_tokens = generator.n_ctx - 512 - 100 # max_tokens=512, safety_margin=100
|
| 154 |
|
| 155 |
while total_tokens > max_allowed_tokens and history:
|
| 156 |
removed_msg = history.pop(0)
|
|
|
|
| 150 |
|
| 151 |
total_tokens = system_tokens + query_tokens + sum(history_tokens) + len(history) * 10 + 10 # Rough estimate for formatting
|
| 152 |
|
| 153 |
+
max_allowed_tokens = generator.n_ctx() - 512 - 100 # max_tokens=512, safety_margin=100
|
| 154 |
|
| 155 |
while total_tokens > max_allowed_tokens and history:
|
| 156 |
removed_msg = history.pop(0)
|