Spaces:
Paused
Paused
File size: 5,771 Bytes
913f308 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
#
# SPDX-FileCopyrightText: Hadad <[email protected]>
# SPDX-License-Identifier: Apache-2.0
#
import json # Import JSON module for encoding and decoding JSON data
from src.tools.audio import AudioGeneration # Import AudioGeneration class to handle audio creation
# Asynchronous handler for audio generation command
async def audio_integration(
input, # User input containing the /audio command and instruction
new_history, # Conversation history in message format
session_id, # Session ID for conversation context
selected_model, # Selected AI model for generation
jarvis # AI backend function for generating responses
):
# Extract the audio instruction text after the '/audio' command prefix and strip whitespace
audio_instruction = input[6:].strip() # Get instruction after /audio
# If no instruction text is provided after the command, yield empty and exit early
if not audio_instruction: # Check if instruction is empty
yield [] # Yield empty list for missing instruction
return # Exit function
try: # Try block for audio generation
# Asynchronously create audio content based on the instruction using AudioGeneration class
audio = await AudioGeneration.create_audio(audio_instruction) # Generate audio
# Serialize the audio data and instruction into a JSON formatted string for processing
audio_generation_content = json.dumps({
"audio": audio, # Audio content or URL
"audio_instruction": audio_instruction # Instruction for audio generation
})
# Construct the conversation history including the audio generation result and formatting instructions
audio_generation_result = (
new_history
+ [
{
"role": "system",
"content": (
"Audio generation result:\n\n" + audio_generation_content + "\n\n\n"
"Show the audio using the following HTML audio tag format, where '{audio_link}' is the URL of the generated audio:\n\n"
"<audio controls src='{audio_link}' style='width:100%; max-width:100%;'></audio>\n\n"
"Please replace '{audio_link}' with the actual audio URL provided in the context.\n\n"
"Then, describe the generated audio based on the above information.\n\n\n"
"Use the same language as the previous user input or user request.\n"
"For example, if the previous user input or user request is in Indonesian, explain in Indonesian.\n"
"If it is in English, explain in English. This also applies to other languages.\n\n\n"
)
}
]
)
# Use async generator to get descriptive text about the generated audio from AI
async for audio_description in jarvis(
session_id=session_id, # Session ID
model=selected_model, # Selected model
history=audio_generation_result, # Updated history with audio result
user_message=input, # User input
mode="/no_think", # Use non-reasoning mode to avoid extra processing overhead
temperature=0.7, # Fixed temperature for consistent audio description generation
top_k=20, # Limit token sampling to top 20 most probable tokens
min_p=0, # Minimum probability threshold set to zero
top_p=0.8, # Nucleus sampling threshold for quality control
repetition_penalty=1 # No repetition penalty for this step
):
yield [{"role": "tool", "content": audio_description}] # Yield audio description in tool role
return # Exit after handling audio
except Exception: # Exception handling for audio generation failure
# If audio generation fails, let AI generate a contextual error message
generation_failed = (
new_history
+ [
{
"role": "system",
"content": (
"Audio generation failed for the user's request. The user tried to generate audio with the instruction: '"
+ audio_instruction + "'\n\n\n"
"Please explain to the user that audio generation failed and suggest they wait 15 seconds before trying again.\n"
"Be helpful and empathetic in your response.\n\n\n"
"Use the same language as the previous user input or user request.\n"
"For example, if the previous user input or user request is in Indonesian, explain in Indonesian.\n"
"If it is in English, explain in English. This also applies to other languages.\n\n\n"
)
}
]
)
# Use AI to generate a contextual error message
async for error_response in jarvis(
session_id=session_id, # Session ID
model=selected_model, # Selected model
history=generation_failed, # History with error context
user_message=input, # User input
mode="/no_think", # Use non-reasoning mode for error handling
temperature=0.7, # Fixed temperature for more consistent error messages
top_k=20, # Limit token sampling
min_p=0, # Minimum probability threshold
top_p=0.8, # Nucleus sampling threshold
repetition_penalty=1 # No repetition penalty
):
yield [{"role": "tool", "content": error_response}] # Yield error response in tool role
return # Exit after error handling |