Spaces:
Running
Running
File size: 5,879 Bytes
913f308 3d462e4 913f308 3d462e4 913f308 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
#
# SPDX-FileCopyrightText: Hadad <[email protected]>
# SPDX-License-Identifier: Apache-2.0
#
import json # Import JSON module for encoding and decoding JSON data
from src.tools.audio import AudioGeneration # Import AudioGeneration class to handle audio creation
# Asynchronous handler for audio generation command
async def audio_integration(
input, # User input containing the /audio command and instruction
new_history, # Conversation history in message format
session_id, # Session ID for conversation context
selected_model, # Selected AI model for generation
jarvis, # AI backend function for generating responses
mode, # Mode for AI response generation
temperature, # Temperature parameter for AI
top_k, # Top-k parameter for AI
min_p, # Min-p parameter for AI
top_p, # Top-p parameter for AI
repetition_penalty # Repetition penalty for AI
):
# Extract the audio instruction text after the '/audio' command prefix and strip whitespace
audio_instruction = input[6:].strip() # Get instruction after /audio
# If no instruction text is provided after the command, yield empty and exit early
if not audio_instruction: # Check if instruction is empty
yield [] # Yield empty list for missing instruction
return # Exit function
try: # Try block for audio generation
# Asynchronously create audio content based on the instruction using AudioGeneration class
audio = await AudioGeneration.create_audio(audio_instruction) # Generate audio
# Serialize the audio data and instruction into a JSON formatted string for processing
audio_generation_content = json.dumps({
"audio": audio, # Audio content or URL
"audio_instruction": audio_instruction # Instruction for audio generation
})
# Construct the conversation history including the audio generation result and formatting instructions
audio_generation_result = (
new_history
+ [
{
"role": "system",
"content": (
"Audio generation result:\n\n" + audio_generation_content + "\n\n\n"
"Show the audio using the following HTML audio tag format, where '{audio_link}' is the URL of the generated audio:\n\n"
"<audio controls src='{audio_link}' style='width:100%; max-width:100%;'></audio>\n\n"
"Please replace '{audio_link}' with the actual audio URL provided in the context.\n\n"
"Then, describe the generated audio based on the above information.\n\n\n"
"Use the same language as the previous user input or user request.\n"
"For example, if the previous user input or user request is in Indonesian, explain in Indonesian.\n"
"If it is in English, explain in English. This also applies to other languages.\n\n\n"
)
}
]
)
# Use async generator to get descriptive text about the generated audio from AI
async for audio_description in jarvis(
session_id=session_id, # Session ID
model=selected_model, # Selected model
history=audio_generation_result, # Updated history with audio result
user_message=input, # User input
mode=mode, # Mode for AI response
temperature=temperature, # temperature parameter
top_k=top_k, # top_k parameter
min_p=min_p, # min_p parameter
top_p=top_p, # top_p parameter
repetition_penalty=repetition_penalty # repetition_penalty parameter
):
yield [{"role": "tool", "content": audio_description}] # Yield audio description in tool role
return # Exit after handling audio
except Exception: # Exception handling for audio generation failure
# If audio generation fails, let AI generate a contextual error message
generation_failed = (
new_history
+ [
{
"role": "system",
"content": (
"Audio generation failed for the user's request. The user tried to generate audio with the instruction: '"
+ audio_instruction + "'\n\n\n"
"Please explain to the user that audio generation failed and suggest they wait 15 seconds before trying again.\n"
"Be helpful and empathetic in your response.\n\n\n"
"Use the same language as the previous user input or user request.\n"
"For example, if the previous user input or user request is in Indonesian, explain in Indonesian.\n"
"If it is in English, explain in English. This also applies to other languages.\n\n\n"
)
}
]
)
# Use AI to generate a contextual error message
async for error_response in jarvis(
session_id=session_id, # Session ID
model=selected_model, # Selected model
history=generation_failed, # History with error context
user_message=input, # User input
mode="/no_think", # Use non-reasoning mode for error handling
temperature=0.7, # Fixed temperature for more consistent error messages
top_k=20, # Limit token sampling
min_p=0, # Minimum probability threshold
top_p=0.8, # Nucleus sampling threshold
repetition_penalty=1 # No repetition penalty
):
yield [{"role": "tool", "content": error_response}] # Yield error response in tool role
return # Exit after error handling |