add support for NVIDIA llm (#1645)

### What problem does this PR solve?

add support for NVIDIA llm
### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Co-authored-by: Zhedong Cen <[email protected]>

conf/llm_factories.json

@@ -1918,6 +1918,290 @@
                     "model_type": "chat"
                 }
             ]
+        },
+        {
+            "name": "NVIDIA",
+            "logo": "",
+            "tags": "LLM,TEXT EMBEDDING, TEXT RE-RANK",
+            "status": "1",
+            "llm": [
+                {
+                    "llm_name": "nvidia/nemotron-4-340b-reward",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "aisingapore/sea-lion-7b-instruct",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "databricks/dbrx-instruct",
+                    "tags": "LLM,CHAT,16K",
+                    "max_tokens": 16384,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "google/gemma-7b",
+                    "tags": "LLM,CHAT,32K",
+                    "max_tokens": 32768,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "google/gemma-2b",
+                    "tags": "LLM,CHAT,16K",
+                    "max_tokens": 16384,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "google/gemma-2-9b-it",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "google/gemma-2-27b-it",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "google/recurrentgemma-2b",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mediatek/breeze-7b-instruct",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "meta/llama2-70b",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "meta/llama3-8b",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "meta/llama3-70b",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "microsoft/phi-3-medium-128k-instruct",
+                    "tags": "LLM,CHAT,128K",
+                    "max_tokens": 131072,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "microsoft/phi-3-medium-4k-instruct",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "microsoftphi-3-mini-128k-instruct",
+                    "tags": "LLM,CHAT,128K",
+                    "max_tokens": 131072,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "microsoft/phi-3-mini-4k-instruct",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "microsoft/phi-3-small-128k-instruct",
+                    "tags": "LLM,CHAT,128K",
+                    "max_tokens": 131072,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "microsoft/phi-3-small-8k-instruct",
+                    "tags": "LLM,CHAT,8K",
+                    "max_tokens": 8192,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mistralai/mistral-7b-instruct",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mistralai/mistral-7b-instruct-v0.3",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mistralai/mixtral-8x7b-instruct",
+                    "tags": "LLM,CHAT,32K",
+                    "max_tokens": 32768,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mistralai/mixtral-8x22b-instruct",
+                    "tags": "LLM,CHAT,64K",
+                    "max_tokens": 65536,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "mistralai/mistral-large",
+                    "tags": "LLM,CHAT,32K",
+                    "max_tokens": 32768,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "nv-mistralai/mistral-nemo-12b-instruct",
+                    "tags": "LLM,CHAT,128K",
+                    "max_tokens": 131072,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "nvidia/llama3-chatqa-1.5-70b",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "nvidia/llama3-chatqa-1.5-8b",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "nvidia/nemotron-4-340b-instruct",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "seallms/seallm-7b-v2.5",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "snowflake/arctic",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "upstage/solar-10.7b-instruct",
+                    "tags": "LLM,CHAT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "chat"
+                },
+                {
+                    "llm_name": "baai/bge-m3",
+                    "tags": "TEXT EMBEDDING,8K",
+                    "max_tokens": 8192,
+                    "model_type": "embedding"
+                },
+                {
+                    "llm_name": "nvidia/embed-qa-4",
+                    "tags": "TEXT EMBEDDING,512",
+                    "max_tokens": 512,
+                    "model_type": "embedding"
+                },
+                {
+                    "llm_name": "nvidia/nv-embed-v1",
+                    "tags": "TEXT EMBEDDING,32K",
+                    "max_tokens": 32768,
+                    "model_type": "embedding"
+                },
+                {
+                    "llm_name": "nvidia/nv-embedqa-e5-v5",
+                    "tags": "TEXT EMBEDDING,512",
+                    "max_tokens": 512,
+                    "model_type": "embedding"
+                },
+                {
+                    "llm_name": "nvidia/nv-embedqa-mistral-7b-v2",
+                    "tags": "TEXT EMBEDDING,512",
+                    "max_tokens": 512,
+                    "model_type": "embedding"
+                },
+                {
+                    "llm_name": "nvidia/nv-rerankqa-mistral-4b-v3",
+                    "tags": "RE-RANK,512",
+                    "max_tokens": 512,
+                    "model_type": "rerank"
+                },
+                {
+                    "llm_name": "nvidia/rerank-qa-mistral-4b",
+                    "tags": "RE-RANK,512",
+                    "max_tokens": 512,
+                    "model_type": "rerank"
+                },
+                {
+                    "llm_name": "snowflake/arctic-embed-l",
+                    "tags": "TEXT EMBEDDING,512",
+                    "max_tokens": 512,
+                    "model_type": "embedding"
+                },
+                {
+                    "llm_name": "adept/fuyu-8b",
+                    "tags": "LLM,IMAGE2TEXT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "image2text"
+                },
+                {
+                    "llm_name": "google/deplot",
+                    "tags": "LLM,IMAGE2TEXT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "image2text"
+                },
+                {
+                    "llm_name": "google/paligemma",
+                    "tags": "LLM,IMAGE2TEXT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "image2text"
+                },
+                {
+                    "llm_name": "Iiuhaotian/Ilava-v1.6-34b",
+                    "tags": "LLM,IMAGE2TEXT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "image2text"
+                },
+                {
+                    "llm_name": "Iiuhaotian/Ilava-v1.6-mistral-7b",
+                    "tags": "LLM,IMAGE2TEXT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "image2text"
+                },
+                {
+                    "llm_name": "microsoft/kosmos-2",
+                    "tags": "LLM,IMAGE2TEXT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "image2text"
+                },
+                {
+                    "llm_name": "microsoft/phi-3-vision-128k-instruct",
+                    "tags": "LLM,IMAGE2TEXT,128K",
+                    "max_tokens": 131072,
+                    "model_type": "image2text"
+                },
+                {
+                    "llm_name": "nvidia/neva-22b",
+                    "tags": "LLM,IMAGE2TEXT,4K",
+                    "max_tokens": 4096,
+                    "model_type": "image2text"
+                }
+            ]
         }
     ]
 }

rag/llm/__init__.py

@@ -34,7 +34,8 @@ EmbeddingModel = {
     "BAAI": DefaultEmbedding,
     "Mistral": MistralEmbed,
     "Bedrock": BedrockEmbed,
-    "Gemini":GeminiEmbed
+    "Gemini":GeminiEmbed,
+    "NVIDIA":NvidiaEmbed
 }
 
 
@@ -48,7 +49,8 @@ CvModel = {
     "Moonshot": LocalCV,
     'Gemini':GeminiCV,
     'OpenRouter':OpenRouterCV,
-    "LocalAI":LocalAICV
+    "LocalAI":LocalAICV,
+    "NVIDIA":NvidiaCV
 }
 
 
@@ -71,7 +73,8 @@ ChatModel = {
     "Bedrock": BedrockChat,
     "Groq": GroqChat,
     'OpenRouter':OpenRouterChat,
-    "StepFun":StepFunChat
+    "StepFun":StepFunChat,
+    "NVIDIA":NvidiaChat
 }
 
 
@@ -79,7 +82,8 @@ RerankModel = {
     "BAAI": DefaultRerank,
     "Jina": JinaRerank,
     "Youdao": YoudaoRerank,
-    "Xinference": XInferenceRerank
+    "Xinference": XInferenceRerank,
+    "NVIDIA":NvidiaRerank
 }
 
 

rag/llm/chat_model.py

@@ -581,7 +581,6 @@ class MiniMaxChat(Base):
             response = requests.request(
                 "POST", url=self.base_url, headers=headers, data=payload
             )
-            print(response, flush=True)
             response = response.json()
             ans = response["choices"][0]["message"]["content"].strip()
             if response["choices"][0]["finish_reason"] == "length":
@@ -902,4 +901,79 @@ class StepFunChat(Base):
     def __init__(self, key, model_name, base_url="https://api.stepfun.com/v1/chat/completions"):
         if not base_url:
             base_url = "https://api.stepfun.com/v1/chat/completions"
-        super().__init__(key, model_name, base_url)
+        super().__init__(key, model_name, base_url)
+
+
+class NvidiaChat(Base):
+    def __init__(
+        self,
+        key,
+        model_name,
+        base_url="https://integrate.api.nvidia.com/v1/chat/completions",
+    ):
+        if not base_url:
+            base_url = "https://integrate.api.nvidia.com/v1/chat/completions"
+        self.base_url = base_url
+        self.model_name = model_name
+        self.api_key = key
+        self.headers = {
+            "accept": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def chat(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        for k in list(gen_conf.keys()):
+            if k not in ["temperature", "top_p", "max_tokens"]:
+                del gen_conf[k]
+        payload = {"model": self.model_name, "messages": history, **gen_conf}
+        try:
+            response = requests.post(
+                url=self.base_url, headers=self.headers, json=payload
+            )
+            response = response.json()
+            ans = response["choices"][0]["message"]["content"].strip()
+            return ans, response["usage"]["total_tokens"]
+        except Exception as e:
+            return "**ERROR**: " + str(e), 0
+
+    def chat_streamly(self, system, history, gen_conf):
+        if system:
+            history.insert(0, {"role": "system", "content": system})
+        for k in list(gen_conf.keys()):
+            if k not in ["temperature", "top_p", "max_tokens"]:
+                del gen_conf[k]
+        ans = ""
+        total_tokens = 0
+        payload = {
+            "model": self.model_name,
+            "messages": history,
+            "stream": True,
+            **gen_conf,
+        }
+
+        try:
+            response = requests.post(
+                url=self.base_url,
+                headers=self.headers,
+                json=payload,
+            )
+            for resp in response.text.split("\n\n"):
+                if "choices" not in resp:
+                    continue
+                resp = json.loads(resp[6:])
+                if "content" in resp["choices"][0]["delta"]:
+                    text = resp["choices"][0]["delta"]["content"]
+                else:
+                    continue
+                ans += text
+                if "usage" in resp:
+                    total_tokens = resp["usage"]["total_tokens"]
+                yield ans
+
+        except Exception as e:
+            yield ans + "\n**ERROR**: " + str(e)
+
+        yield total_tokens

rag/llm/cv_model.py

@@ -137,7 +137,6 @@ class Base(ABC):
         ]
 
 
-
 class GptV4(Base):
     def __init__(self, key, model_name="gpt-4-vision-preview", lang="Chinese", base_url="https://api.openai.com/v1"):
         if not base_url: base_url="https://api.openai.com/v1"
@@ -619,3 +618,65 @@ class LocalCV(Base):
 
     def describe(self, image, max_tokens=1024):
         return "", 0
+
+
+class NvidiaCV(Base):
+    def __init__(
+        self,
+        key,
+        model_name,
+        lang="Chinese",
+        base_url="https://ai.api.nvidia.com/v1/vlm",
+    ):
+        if not base_url:
+            base_url = ("https://ai.api.nvidia.com/v1/vlm",)
+        self.lang = lang
+        factory, llm_name = model_name.split("/")
+        if factory != "liuhaotian":
+            self.base_url = os.path.join(base_url, factory, llm_name)
+        else:
+            self.base_url = os.path.join(
+                base_url, "community", llm_name.replace("-v1.6", "16")
+            )
+        self.key = key
+
+    def describe(self, image, max_tokens=1024):
+        b64 = self.image2base64(image)
+        response = requests.post(
+            url=self.base_url,
+            headers={
+                "accept": "application/json",
+                "content-type": "application/json",
+                "Authorization": f"Bearer {self.key}",
+            },
+            json={
+                "messages": self.prompt(b64),
+                "max_tokens": max_tokens,
+            },
+        )
+        response = response.json()
+        return (
+            response["choices"][0]["message"]["content"].strip(),
+            response["usage"]["total_tokens"],
+        )
+
+    def prompt(self, b64):
+        return [
+            {
+                "role": "user",
+                "content": (
+                    "请用中文详细描述一下图中的内容，比如时间，地点，人物，事情，人物心情等，如果有数据请提取出数据。"
+                    if self.lang.lower() == "chinese"
+                    else "Please describe the content of this picture, like where, when, who, what happen. If it has number data, please extract them out."
+                )
+                + f' <img src="data:image/jpeg;base64,{b64}"/>',
+            }
+        ]
+
+    def chat_prompt(self, text, b64):
+        return [
+            {
+                "role": "user",
+                "content": text + f' <img src="data:image/jpeg;base64,{b64}"/>',
+            }
+        ]

rag/llm/embedding_model.py

@@ -462,3 +462,41 @@ class GeminiEmbed(Base):
             title="Embedding of single string")
         token_count = num_tokens_from_string(text)
         return np.array(result['embedding']),token_count
+
+class NvidiaEmbed(Base):
+    def __init__(
+        self, key, model_name, base_url="https://integrate.api.nvidia.com/v1/embeddings"
+    ):
+        if not base_url:
+            base_url = "https://integrate.api.nvidia.com/v1/embeddings"
+        self.api_key = key
+        self.base_url = base_url
+        self.headers = {
+            "accept": "application/json",
+            "Content-Type": "application/json",
+            "authorization": f"Bearer {self.api_key}",
+        }
+        self.model_name = model_name
+        if model_name == "nvidia/embed-qa-4":
+            self.base_url = "https://ai.api.nvidia.com/v1/retrieval/nvidia/embeddings"
+            self.model_name = "NV-Embed-QA"
+        if model_name == "snowflake/arctic-embed-l":
+            self.base_url = "https://ai.api.nvidia.com/v1/retrieval/snowflake/arctic-embed-l/embeddings"
+
+    def encode(self, texts: list, batch_size=None):
+        payload = {
+            "input": texts,
+            "input_type": "query",
+            "model": self.model_name,
+            "encoding_format": "float",
+            "truncate": "END",
+        }
+        res = requests.post(self.base_url, headers=self.headers, json=payload).json()
+        return (
+            np.array([d["embedding"] for d in res["data"]]),
+            res["usage"]["total_tokens"],
+        )
+
+    def encode_queries(self, text):
+        embds, cnt = self.encode([text])
+        return np.array(embds[0]), cnt

rag/llm/rerank_model.py

@@ -164,3 +164,41 @@ class LocalAIRerank(Base):
 
     def similarity(self, query: str, texts: list):
         raise NotImplementedError("The LocalAIRerank has not been implement")
+
+
+class NvidiaRerank(Base):
+    def __init__(
+        self, key, model_name, base_url="https://ai.api.nvidia.com/v1/retrieval/nvidia/"
+    ):
+        if not base_url:
+            base_url = "https://ai.api.nvidia.com/v1/retrieval/nvidia/"
+        self.model_name = model_name
+
+        if self.model_name == "nvidia/nv-rerankqa-mistral-4b-v3":
+            self.base_url = os.path.join(
+                base_url, "nv-rerankqa-mistral-4b-v3", "reranking"
+            )
+
+        if self.model_name == "nvidia/rerank-qa-mistral-4b":
+            self.base_url = os.path.join(base_url, "reranking")
+            self.model_name = "nv-rerank-qa-mistral-4b:1"
+
+        self.headers = {
+            "accept": "application/json",
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {key}",
+        }
+
+    def similarity(self, query: str, texts: list):
+        token_count = num_tokens_from_string(query) + sum(
+            [num_tokens_from_string(t) for t in texts]
+        )
+        data = {
+            "model": self.model_name,
+            "query": {"text": query},
+            "passages": [{"text": text} for text in texts],
+            "truncate": "END",
+            "top_n": len(texts),
+        }
+        res = requests.post(self.base_url, headers=self.headers, json=data).json()
+        return (np.array([d["logit"] for d in res["rankings"]]), token_count)

web/src/pages/user-setting/setting-model/constant.ts

@@ -20,6 +20,7 @@ export const IconMap = {
   OpenRouter: 'open-router',
   LocalAI: 'local-ai',
   StepFun: 'stepfun',
+  NVIDIA:'nvidia'
 };
 
 export const BedrockRegionList = [