Update app.py
Browse files
app.py
CHANGED
@@ -6,6 +6,8 @@ import os
|
|
6 |
import platform
|
7 |
import psutil
|
8 |
import multiprocessing
|
|
|
|
|
9 |
|
10 |
|
11 |
app = FastAPI()
|
@@ -34,8 +36,8 @@ else:
|
|
34 |
llm = Llama(
|
35 |
model_path=model_path,
|
36 |
n_ctx=1024,
|
37 |
-
n_threads=
|
38 |
-
n_batch=
|
39 |
)
|
40 |
|
41 |
@app.get("/")
|
|
|
6 |
import platform
|
7 |
import psutil
|
8 |
import multiprocessing
|
9 |
+
import time
|
10 |
+
|
11 |
|
12 |
|
13 |
app = FastAPI()
|
|
|
36 |
llm = Llama(
|
37 |
model_path=model_path,
|
38 |
n_ctx=1024,
|
39 |
+
n_threads=8, # Adjust for your CPU
|
40 |
+
n_batch=32
|
41 |
)
|
42 |
|
43 |
@app.get("/")
|