debug

Dockerfile

@@ -3,8 +3,8 @@ FROM nvidia/cuda:12.2.0-base-ubuntu22.04 AS base
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"
-#ENV CUDA_VISIBLE_DEVICES=0
-#ENV LLAMA_CUBLAS=1
+ENV CUDA_VISIBLE_DEVICES=0
+ENV LLAMA_CUBLAS=1
 
 # Install necessary dependencies and musl
 RUN apt-get update && apt-get install -y \

app.py

@@ -351,7 +351,7 @@ try:
         logger.info("### Initializing LLM.")
         llm = Llama(model_path,
                     #*,
-                    n_gpu_layers=0,
+                    n_gpu_layers=-1,
                     split_mode=llama_cpp.LLAMA_SPLIT_MODE_LAYER,
                     main_gpu=0,
                     tensor_split=None,