Duplicate from simonraj/OralCoachDemo

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: OralCoachDemo
+emoji: 📚
+colorFrom: purple
+colorTo: yellow
+sdk: gradio
+sdk_version: 3.42.0
+app_file: app.py
+pinned: false
+license: openrail
+duplicated_from: simonraj/OralCoachDemo
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,112 @@
+import os
+import openai
+import gradio as gr
+import base64
+from data4 import strategy_text, description, questions
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") 
+openai.api_key = OPENAI_API_KEY
+
+def transcribe_audio(audio_file_path):
+    # Use OpenAI's Whisper to transcribe the audio
+    audio_file = open(audio_file_path, "rb")
+    transcript = openai.Audio.transcribe("whisper-1", audio_file)
+    return transcript["text"]
+
+def get_base64_image():
+    with open("SBC4.jpg", "rb") as img_file:
+        return base64.b64encode(img_file.read()).decode("utf-8")
+    
+def get_image_html():
+    return (
+        f"<img src='data:image/jpeg;base64,{get_base64_image()}' style='display: block; margin-left: auto; margin-right: auto; padding-bottom: 15px; width: 300px;'>"
+    )
+
+current_question_index = 0
+user_input_counter = 0
+conversation_history = []
+
+def intelligent_tutor(audio_file, provide_hints=False):
+    global current_question_index
+    global questions
+    global user_input_counter
+    global conversation_history
+
+    input_text = transcribe_audio(audio_file)
+    current_question = questions[current_question_index]
+
+    if provide_hints:
+        # If hints are requested, provide guidance on how to answer using the strategy text
+        hint_message = f"Consider using the {strategy_text[current_question_index]} to answer the question: '{questions[current_question_index]}'."
+        return f"Respond to this Question: {questions[current_question_index]}", hint_message
+
+    conversation = [
+        {
+            "role": "system",
+            "content": f"You are an expert English Language Teacher in a Singapore Primary school, directly guiding a Primary 6 student in Singapore. The student is answering the question: '{questions[current_question_index]}'. Based on their response, provide direct feedback to help them improve their spoken skills. Emphasize areas of strength, suggest areas of improvement, and guide them on how to better answer using the {strategy_text[current_question_index]} strategy. The feedback should be in second person, addressing the student directly."
+        },
+        {"role": "user", "content": input_text}
+    ]
+
+    # Append the user's response to the conversation history
+    conversation_history.append(input_text)
+
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=conversation,
+        max_tokens=400
+    )
+
+    if not response.choices:
+        return "No response from the model.", ""
+
+    text_response = response.choices[0]['message']['content'].strip()
+    text_response = text_response.replace('\n', '<br>')
+
+    user_input_counter += 1
+
+    if user_input_counter % 2 == 0:
+        if current_question_index + 1 < len(questions):
+            current_question_index += 1
+            next_question = questions[current_question_index]
+            text_response += f"\n\nNext question ({current_question_index + 1}): {next_question}"
+        else:
+            # All questions have been answered, provide a summary
+            summary_prompt = {
+                "role": "system",
+                "content": f"Based on the entire conversation, provide a detailed feedback summary highlighting the overall performance, strengths, and areas of improvement. Reference the student's responses and evaluate how well they used the {strategy_text[current_question_index]} strategy to structure their answers. Format the feedback in bullet points."
+            }
+            summary_conversation = [summary_prompt, {"role": "user", "content": " ".join(conversation_history)}]
+            
+            summary_response = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=summary_conversation,
+                max_tokens=600  # Increased token limit for detailed summary
+            )
+            
+            if not summary_response.choices:
+                return "No response from the model.", ""
+            
+            text_response = summary_response.choices[0]['message']['content'].strip()
+            text_response = text_response.replace('\n', '<br>')
+
+    wrapped_output_text = f'<div style="height: 300px; overflow-y: scroll;">{text_response}</div>'
+    return f"Current Question: {questions[current_question_index]}", wrapped_output_text
+
+iface = gr.Interface(
+    fn=intelligent_tutor,
+    inputs=[
+        gr.Audio(source="microphone", type="filepath", label="Record audio", sampling_rate=16000),
+        gr.inputs.Checkbox(label="Provide Summary of Conversation"),  # Checkbox for hints
+    ],
+    outputs=[
+        gr.outputs.HTML(label="Question"),
+        gr.outputs.HTML(label="Output Text"),
+    ],
+    title="Oral Coach for Stimulus Based Conversation",
+    description=(get_image_html() + 
+                 "<br> " + questions[0] +
+                 "<br>You have two attempts for each question.<br>" +
+                 "<b>Please answer the displayed question at the output screen after the 1st Question.</b>"),
+)
+iface.launch(share=False)

data4.py

@@ -0,0 +1,20 @@
+# data4.py
+
+strategy_text = {
+    0: "Use the PEEL strategy (Point, Evidence, Experience(s), Link back to the question).",
+    1: "Use the 5W1H thinking frame (Who, What, Where, When, Why, How) to generate ideas.",
+    2: "Use the OREO thinking frame (Opening Statement, Reasons, Elaborate, Opinion) to guide the student."
+}
+
+description = (
+    "Poster calling for nomination for the Supportive Friend Award. It shows 3 pictures of 2 students exhibiting acts of friendship. "
+    "The first pictures shows 2 students and 1 of them seems to helping the other with their homework, "
+    "the other images shows two students playing a game of chess and the 3rd pictures shows 2 friends having a conversation with "
+    "the 3 respective captions: helps me, plays with me and listens to me."
+)
+
+questions = [
+    "1. Look at the picture. Would you like to have a schoolmate like this? Why? / Why not?",
+    "2. Do you like making new friends? Why? / Why not?",
+    "3. Would you like to be a buddy to a younger pupil? Why? / Why not?",
+]

requirements.txt

@@ -0,0 +1 @@
+openai