Update Dockerfile

Dockerfile

@@ -1,63 +1,57 @@
-# Using the specified base image that's suited for llama-cpp-python
-# Define the image argument and provide a default value
-ARG IMAGE=python:3-slim-bullseye
-
-# Use the image as specified
-FROM ${IMAGE}
-
-# Re-declare the ARG after FROM
-ARG IMAGE
-
-# Update and upgrade the existing packages 
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
-    python3 \
-    python3-pip \
-    ninja-build \
-    build-essential
-
-RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context
-
-# Perform the conditional installations based on the image
-RUN echo "Image: ${IMAGE}" && \
-    if [ "${IMAGE}" = "python:3-slim-bullseye" ] ; then \
-    echo "OpenBLAS install:" && \
-    apt-get install -y --no-install-recommends libopenblas-dev && \
-    LLAMA_OPENBLAS=1 pip install llama-cpp-python --verbose; \
-else \
-    echo "CuBLAS install:" && \
-    LLAMA_CUBLAS=1 pip install llama-cpp-python --verbose; \
-fi
-
-VOLUME ["/models"]
-
-# Environment variables for model details
-ENV MODEL_NAME="llava-1.6-mistral-7b-gguf"
-ENV DEFAULT_MODEL_FILE="llava-v1.6-mistral-7b.Q3_K_XS.gguf"
-ENV MODEL_USER="cjpais"
-ENV DEFAULT_MODEL_BRANCH="main"
-ENV DEFAULT_CLIP_MODEL_FILE="mmproj-model-f16.gguf"
-ENV MODEL_URL="https://huggingface.co/${MODEL_USER}/${MODEL_NAME}/resolve/${DEFAULT_MODEL_BRANCH}/${DEFAULT_MODEL_FILE}"
-ENV CLIP_MODEL_URL="https://huggingface.co/${MODEL_USER}/${MODEL_NAME}/resolve/${DEFAULT_MODEL_BRANCH}/${DEFAULT_CLIP_MODEL_FILE}"
-
-# Set up the working directory
+# Base image
+FROM nvidia/cuda:12.2.0-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Update and install necessary dependencies
+RUN apt update && \
+    apt install --no-install-recommends -y \
+        build-essential \
+        python3 \
+        python3-pip \
+        wget \
+        curl \
+        git \
+        cmake \
+        zlib1g-dev \
+        libblas-dev && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+    
+# Setting up CUDA environment variables (this may not be necessary since you're using the official nvidia/cuda image, but it's good to be explicit)
+ENV PATH="/usr/local/cuda/bin:$PATH" \
+    LD_LIBRARY_PATH="/usr/local/cuda/lib64:$LD_LIBRARY_PATH" \
+    CUDA_HOME="/usr/local/cuda"
+
 WORKDIR /app
 
-# Ensure curl is available for downloading the models
-RUN apt-get update && apt-get install -y curl && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+# Download ggml and mmproj models from HuggingFace
+RUN wget https://huggingface.co/mys/ggml_bakllava-1/resolve/main/ggml-model-q4_k.gguf && \
+    wget https://huggingface.co/mys/ggml_bakllava-1/resolve/main/mmproj-model-f16.gguf 
+
+# Clone and build llava-server with CUDA support
+RUN git clone https://github.com/ggerganov/llama.cpp.git && \
+    cd llama.cpp && \
+    git submodule init && \
+    git submodule update && \
+    make LLAMA_CUBLAS=1
+
+# Create a non-root user for security reasons
+RUN useradd -m -u 1000 user && \
+    mkdir -p /home/user/app && \
+    cp /app/ggml-model-q4_k.gguf /home/user/app && \
+    cp /app/mmproj-model-f16.gguf /home/user/app
 
-# Create a directory for the models
-RUN mkdir -p /models
+RUN chown user:user /home/user/app/ggml-model-q4_k.gguf && \
+    chown user:user /home/user/app/mmproj-model-f16.gguf
 
-# Download the models
-RUN curl -L "${MODEL_URL}" -o /models/${DEFAULT_MODEL_FILE} && \
-    curl -L "${CLIP_MODEL_URL}" -o /models/${DEFAULT_CLIP_MODEL_FILE}
+USER user
+ENV HOME=/home/user
 
+WORKDIR $HOME/app
 
-ENV HOST=0.0.0.0
-ENV PORT=8000
-# Expose the port the server will run on
-EXPOSE 8000
+# Expose the port
+EXPOSE 8080
 
-# Command to run the server, using environment variables for model paths
-CMD ["python3", "-m", "llama_cpp.server", "--model", "/models/llava-v1.6-mistral-7b.Q3_K_XS.gguf", "--clip_model_path", "/models/mmproj-model-f16.gguf", "--chat_format", "llava-1-5"]
+# Start the llava-server with models
+CMD ["/app/llama.cpp/server", "--model", "ggml-model-q4_k.gguf", "--mmproj", "mmproj-model-f16.gguf", "--threads", "4"]