File size: 314 Bytes
539ee6f 93a8ab9 539ee6f 93a8ab9 539ee6f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
from llama_cpp import Llama
import os
model_path = "llama-3.2-1B-it-Ecommerce-ChatBot-merged-F16.gguf"
n_threads = os.cpu_count()
print(f"number of cpu cores: {n_threads}")
llm = Llama(
model_path=model_path,
n_ctx=512,
n_batch=512,
n_threads=64,
n_gpu_layers=-1,
chat_format="llama-3"
)
|