Update litellm_config.yaml

litellm_config.yaml

@@ -1,126 +1,33 @@
+# LiteLLM Proxy config for Replicate
+# Save as config.yaml and run:  litellm --config ./config.yaml --debug
+
 model_list:
-  - model_name: hf-mistral-7b-instruct-v03
-    litellm_params:
-      model: huggingface/mistralai/Mistral-7B-Instruct-v0.3
-      api_key: os.environ/HUGGINGFACE_API_KEY
-  - model_name: hf-llama3-8b-instruct
-    litellm_params:
-      model: huggingface/meta-llama/Meta-Llama-3-8B-Instruct
-      api_key: os.environ/HUGGINGFACE_API_KEY
-  - model_name: awan-llama3-8b
-    litellm_params:
-      model: openai/Meta-Llama-3-8B-Instruct
-      api_key: os.environ/OTHER_AWANLLM_API_KEY
-      api_base: https://api.awanllm.com/v1
-      rpm: 10
-  - model_name: awan-mistral-7b-instruct-v03
-    litellm_params:
-      model: openai/Mistral-7B-Instruct
-      api_key: os.environ/OTHER_AWANLLM_API_KEY
-      api_base: https://api.awanllm.com/v1
-      rpm: 10
-  - model_name: groq-llama3-70b
-    litellm_params:
-      model: openai/llama3-70b-8192
-      api_key: os.environ/GROQ_API_KEY
-      api_base: https://api.groq.com/openai/v1
-      rpm: 30
-  - model_name: groq-llama3-8b
-    litellm_params:
-      model: openai/llama3-8b-8192
-      api_key: os.environ/GROQ_API_KEY
-      api_base: https://api.groq.com/openai/v1
-      rpm: 30
-  - model_name: groq-mixtral-8x7b
-    litellm_params:
-      model: openai/mixtral-8x7b-32768
-      api_key: os.environ/GROQ_API_KEY
-      api_base: https://api.groq.com/openai/v1
-      rpm: 30
-  - model_name: or-llama3-8b
-    litellm_params:
-      model: openai/meta-llama/llama-3-8b-instruct:free
-      api_key: os.environ/OPENROUTER_API_KEY
-      api_base: https://openrouter.ai/api/v1
-      rpm: 6
-  - model_name: or-openchat-7b
-    litellm_params:
-      model: openai/openchat/openchat-7b:free
-      api_key: os.environ/OPENROUTER_API_KEY
-      api_base: https://openrouter.ai/api/v1
-      rpm: 6
-  - model_name: or-zephyr-7b-beta
-    litellm_params:
-      model: openai/huggingfaceh4/zephyr-7b-beta:free
-      api_key: os.environ/OPENROUTER_API_KEY
-      api_base: https://openrouter.ai/api/v1
-      rpm: 6
-  - model_name: or-nous-capybara-7b
-    litellm_params:
-      model: openai/nousresearch/nous-capybara-7b:free
-      api_key: os.environ/OPENROUTER_API_KEY
-      api_base: https://openrouter.ai/api/v1
-      rpm: 6
-  - model_name: or-mythomist-7b
-    litellm_params:
-      model: "openrouter/gryphe/mythomist-7b:free"
-      api_key: os.environ/OPENROUTER_API_KEY
-      #api_base: https://openrouter.ai/api/v1
-      rpm: 6
-  - model_name: or-phi3-medium-128k
-    litellm_params:
-      model: openai/microsoft/phi-3-medium-128k-instruct:free
-      api_key: os.environ/OPENROUTER_API_KEY
-      api_base: https://openrouter.ai/api/v1
-      rpm: 6
-  - model_name: or-mistral-7b-v01
-    litellm_params:
-      model: openai/mistralai/mistral-7b-instruct:free
-      api_key: os.environ/OPENROUTER_API_KEY
-      api_base: https://openrouter.ai/api/v1
-      rpm: 6
-  - model_name: cf-qwen1-5-7b
-    litellm_params:
-      model: cloudflare/@cf/qwen/qwen1.5-7b-chat-awq
-      api_key: os.environ/CLOUDFLARE_API_KEY
-  - model_name: cf-hermes-2-pro-mistral-7b
-    litellm_params:
-      model: cloudflare/@hf/nousresearch/hermes-2-pro-mistral-7b
-      api_key: os.environ/CLOUDFLARE_API_KEY
-  - model_name: cf-llama3-8b
-    litellm_params:
-      model: cloudflare/@cf/meta/llama-3-8b-instruct
-      api_key: os.environ/CLOUDFLARE_API_KEY
-  - model_name: cf-deepseek-coder-6-7b
-    litellm_params:
-      model: cloudflare/@hf/thebloke/deepseek-coder-6.7b-instruct-awq
-      api_key: os.environ/CLOUDFLARE_API_KEY
-  - model_name: cf-sqlcoder-7b-2
-    litellm_params:
-      model: cloudflare/@cf/defog/sqlcoder-7b-2
-      api_key: os.environ/CLOUDFLARE_API_KEY
-  - model_name: cf-mistral-7b-v02
-    litellm_params:
-      model: cloudflare/@hf/mistral/mistral-7b-instruct-v0.2
-      api_key: os.environ/CLOUDFLARE_API_KEY
-  - model_name: gemini-pro-vision
-    litellm_params:
-      model: gemini/gemini-pro-vision
-      rpm: 15
-  - model_name: gemini-1.5-flash
+  # Meta Llama 3 Instruct (8B) via Replicate
+  - model_name: openai/gpt-5-nano
+    litellm_params:
+      model: replicate/openai/gpt-5-nano
+      api_key: os.environ/REPLICATE_API_KEY
+      # optional common params
+      temperature: 0.7
+      max_tokens: 1024
+
+  # Llama-2-70B chat with a pinned version ID (replace the example hash with the current one you use)
+  - model_name: openai/gpt-5-mini
     litellm_params:
-      model: gemini/gemini-1.5-flash
-      rpm: 15
-  - model_name: gemini-1.5-pro
+      model: replicate/openai/gpt-5-mini
+      api_key: os.environ/REPLICATE_API_KEY
+
+  # Call a Replicate *deployment* you own (replace owner/name)
+  - model_name: openai/gpt-5
     litellm_params:
-      model: gemini/gemini-1.5-pro
-      rpm: 2
-      tpm: 32000
+      model: replicate/openai/gpt-5
+      api_key: os.environ/REPLICATE_API_KEY
+
+# Optional: proxy/router tuning
+router_settings:
+  timeout: 600           # Replicate can cold-boot, so give it time
+  num_retries: 1
 
-litellm_settings:
-  success_callback: ["langfuse"]
-  failure_callback: ["langfuse"]
-  num_retries: 3
-  request_timeout: 20
-  allowed_fails: 3
-  drop_params: true
+# (Optional) enable simple logging callbacks, etc.
+# litellm_settings:
+#   success_callback: ["generic"]