File size: 806 Bytes
9a88164 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 |
"""This simple script shows how to interact with an OpenAI-compatible server from a client."""
# import argparse
# import modal
from openai import OpenAI
import os
client = OpenAI(api_key=os.getenv("modal_api"))
client.base_url = (
"https://alexprincecursor--example-vllm-openai-compatible-serve.modal.run/v1/"
)
response = client.chat.completions.create(
model="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16", # GPT-4.1 mini
messages=[
{"role": "system", "content": "You are a rockstar lyric generator. You are given a song and you need to generate a lyric for it."},
{"role": "user", "content":"The song is 'Bohemian Rhapsody' by Queen."}
],
max_tokens=512,
temperature=0.7
)
print(response.choices[0].message.content) |