debug

Dockerfile

@@ -48,8 +48,8 @@ RUN FORCE_CMAKE=1 CMAKE_SYSTEM_PROCESSOR=AMD64 pip3 install --break-system-packa
 RUN pip3 install --break-system-packages cffi
 # Install text2vec-transformers
 WORKDIR /app/text2vec-transformers
-COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /app /app/text2vec-transformers
-COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /usr/local/bin /app/text2vec-transformers/bin
+#COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /app /app/text2vec-transformers
+#COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /usr/local/bin /app/text2vec-transformers/bin
 COPY ./multi-qa-MiniLM-L6-cos-v1 /app/text2vec-transformers
 RUN ./custom_prerequisites.py
 

README.md

@@ -10,6 +10,7 @@ app_port: 8501
 #app_file: app.py
 pinned: true
 startup_duration_timeout: 3 hours
+hardware: gpu
 ---
 
 # POC for Retrieval Augmented Generation with Large Language Models

app.py

@@ -346,7 +346,7 @@ try:
         logger.info("### Initializing LLM.")
         llm = Llama(model_path,
                     #*,
-                    n_gpu_layers=0,
+                    n_gpu_layers=-1,
                     split_mode=llama_cpp.LLAMA_SPLIT_MODE_LAYER,
                     main_gpu=0,
                     tensor_split=None,
@@ -398,7 +398,7 @@ try:
         ###############################################################################
         # Initial the the sentence transformer and encode the query prompt.
         logger.debug(f"#### Encode text query prompt to create vectors. {promptText}")
-        model = SentenceTransformer('/app/multi-qa-MiniLM-L6-cos-v1')
+        model = SentenceTransformer('/app/text2vec-transformers/multi-qa-MiniLM-L6-cos-v1')
         vector = model.encode(promptText)
         
         logLevel = logger.getEffectiveLevel()
@@ -539,7 +539,7 @@ try:
             placeHolder = st.empty()
         else:
             st.session_state.spinGenMsg = False;
-            with st.spinner('Generating Completion (but slowly. 40+ seconds.)...'):
+            with st.spinner('Generating Completion...'):
                 st.session_state.sysTAtext = st.session_state.sysTA
                 logger.debug(f"sysTAtext: {st.session_state.sysTAtext}") 
                 wrklist = setPrompt(st.session_state.userpTA,st.selectRag)