Spaces:
Paused
Paused
| @echo off | |
| @rem main code execution | |
| call :print_highlight "Starting inference server for llama-cpp" | |
| cd /D "%~dp0\.." | |
| echo "Change the current directory to: %cd%" | |
| call :path_sanity_check | |
| call :deactivate_environment | |
| @rem config | |
| set ENV_NAME=llama-cpp-python-server | |
| set PYTHON_VERSION=3.10 | |
| set CONDA_ROOT_PREFIX=%cd%\install_dir\conda | |
| set INSTALL_ENV_DIR=%cd%\install_dir\server_envs\%ENV_NAME% | |
| echo "Python version: %PYTHON_VERSION%" | |
| echo "Conda prefix: %CONDA_ROOT_PREFIX%" | |
| echo "Environment path: %INSTALL_ENV_DIR%" | |
| @rem handle conda environment | |
| call :check_conda_existence | |
| call :create_conda_environment | |
| call :isolate_environment | |
| call :activate_environment | |
| @rem install dependencies | |
| @rem ver 0.2.56 produces segment error for /embeddings on MacOS | |
| call python -m pip install llama-cpp-python[server]==0.2.55 | |
| @REM @rem start the server with passed params | |
| call python -m llama_cpp.server %* | |
| call conda deactivate | |
| goto :end | |
| @rem the end of main code execution | |
| @rem below are the functions used in the above execution | |
| :print_highlight | |
| echo. | |
| echo ****************************************************** | |
| echo %~1 | |
| echo ****************************************************** | |
| echo. | |
| goto :eof | |
| :path_sanity_check | |
| echo "Path sanity checking" | |
| echo "%cd%"| findstr /C:" " >nul ^ | |
| && (call :print_highlight "This script relies on Miniconda which can not be silently installed under a path with spaces." ^ | |
| && goto :end) | |
| goto :eof | |
| :deactivate_environment | |
| echo "Deactivate existing environment(s)" | |
| (call conda deactivate && call conda deactivate && call conda deactivate) 2>nul | |
| goto :eof | |
| :check_conda_existence | |
| echo "Check for conda existence" | |
| set conda_exists=F | |
| @rem figure out whether conda exists | |
| call "%CONDA_ROOT_PREFIX%\_conda.exe" --version >nul 2>&1 | |
| if "%ERRORLEVEL%" EQU "0" set conda_exists=T | |
| @rem verify if conda is installed by the main app, if not then raise error | |
| if "%conda_exists%" == "F" ( | |
| call :print_highlight "conda is not installed, seems like the app wasn't installed correctly." | |
| goto :end | |
| ) | |
| goto :eof | |
| :create_conda_environment | |
| @rem create the environment if needed | |
| if not exist "%INSTALL_ENV_DIR%" ( | |
| echo "Create conda environment" | |
| call "%CONDA_ROOT_PREFIX%\_conda.exe" create ^ | |
| --no-shortcuts -y -k --prefix "%INSTALL_ENV_DIR%" python="%PYTHON_VERSION%" || ^ | |
| ( echo. && call :print_highlight "Conda environment creation failed." && goto :end ) | |
| ) | |
| @rem check if conda environment was actually created | |
| if not exist "%INSTALL_ENV_DIR%\python.exe" ( | |
| call :print_highlight "Conda environment was not correctly created." | |
| goto :end | |
| ) | |
| goto :eof | |
| :isolate_environment | |
| echo "Isolate environment" | |
| set PYTHONNOUSERSITE=1 | |
| set PYTHONPATH= | |
| set PYTHONHOME= | |
| goto :eof | |
| :activate_environment | |
| echo "Activate conda environment" | |
| call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ^ | |
| ( echo. && call :print_highlight "Miniconda hook not found." && goto :end ) | |
| goto :eof | |
| :end | |