Update quiz_processing.py

quiz_processing.py

@@ -1,201 +1,229 @@
-import re
-import json
-import os
-from langchain_google_genai import ChatGoogleGenerativeAI
-from transformers import AutoTokenizer
-from huggingface_hub import login
-
-hf_token = os.environ.get('HF_TOKEN', None)
-login(token=hf_token)
-
-tokenizer = AutoTokenizer.from_pretrained("answerdotai/ModernBERT-base", use_auth_token=hf_token)
-
-def clean_text(text):
-    text = re.sub(r'\[speaker_\d+\]', '', text)
-    text = re.sub(r'\s+', ' ', text).strip()
-    return text
-
-def split_text_by_tokens(text, max_tokens=8000):
-    text = clean_text(text)
-    tokens = tokenizer.encode(text)
-
-    if len(tokens) <= max_tokens:
-        return [text]
-
-    split_point = len(tokens) // 2
-
-    sentences = re.split(r'(?<=[.!?])\s+', text)
-
-    first_half = []
-    second_half = []
-
-    current_tokens = 0
-    for sentence in sentences:
-        sentence_tokens = len(tokenizer.encode(sentence))
-
-        if current_tokens + sentence_tokens <= split_point:
-            first_half.append(sentence)
-            current_tokens += sentence_tokens
-        else:
-            second_half.append(sentence)
-
-    return [" ".join(first_half), " ".join(second_half)]
-
-def analyze_segment_with_gemini(segment_text, google_api_key):
-    os.environ["GOOGLE_API_KEY"] = google_api_key
-
-    llm = ChatGoogleGenerativeAI(
-        model="gemini-1.5-flash",
-        temperature=0.7,
-        max_tokens=None,
-        timeout=None,
-        max_retries=3
-    )
-
-    prompt = f"""
-        Analyze the following text and identify distinct segments within it and do text segmentation:
-        1. Segments should be STRICTLY max=15
-        2. For each segment/topic you identify:
-           - Provide a SPECIFIC and UNIQUE topic name (3-5 words) that clearly differentiates it from other segments
-           - List 3-5 key concepts discussed in that segment (be precise and avoid repetition between segments)
-           - Write a brief summary of that segment (3-5 sentences)
-           - Create 5 high-quality, meaningful quiz questions based DIRECTLY on the content in that segment only
-           - Questions and answers should be only from the content of the segment
-
-        For each quiz question:
-        - Create one correct answer that comes DIRECTLY from the text
-        - Create two plausible but incorrect answers
-        - IMPORTANT: Ensure all answer options have similar length (± 3 words)
-        - Ensure the correct answer is clearly indicated with a ✓ symbol
-       - Questions should **require actual understanding**, NOT just basic fact recall.
-        - Questions Are **non-trivial**, encourage deeper thinking, and **avoid surface-level facts**.
-        - Are **directly based on the segment's content** (not inferred from the summary).
-        - Do **not include questions about document structure** (e.g., title, number of paragraphs).
-        - Do **not generate overly generic or obvious questions** (e.g., "What is mentioned in the text?").
-        - Focus on **core ideas, logical reasoning, and conceptual understanding**.
-
-
-        Text:
-        {segment_text}
-
-        Format your response as JSON with the following structure:
-        {{
-            "segments": [
-                {{
-                    "topic_name": "Unique and Specific Topic Name",
-                    "key_concepts": ["concept1", "concept2", "concept3"],
-                    "summary": "Brief summary of this segment.",
-                    "quiz_questions": [
-                        {{
-                            "question": "Question text?",
-                            "options": [
-                                {{
-                                    "text": "Option A",
-                                    "correct": false
-                                }},
-                                {{
-                                    "text": "Option B",
-                                    "correct": true
-                                }},
-                                {{
-                                    "text": "Option C",
-                                    "correct": false
-                                }}
-                            ]
-                        }}
-                    ]
-                }}
-            ]
-        }}
-
-        IMPORTANT: Each segment must have a DISTINCT topic name that clearly differentiates it from others.
-    - **Do NOT repeat** key concepts across multiple segments unless absolutely necessary.
-    - **Ensure the quiz questions challenge the reader** and **are not easily guessable**.
-
-    """
-
-    try:
-        response = llm.invoke(prompt)
-        response_text = response.content
-
-        json_match = re.search(r'\{[\s\S]*\}', response_text)
-        if json_match:
-            return json.loads(json_match.group(0))
-        else:
-            return json.loads(response_text)
-    except Exception as e:
-        print(f"Error in Gemini analysis: {e}")
-        return {
-            "segments": [
-                {
-                    "topic_name": "Analysis Error",
-                    "key_concepts": ["Could not process text"],
-                    "summary": "An error occurred during text analysis.",
-                    "quiz_questions": []
-                }
-            ]
-        }
-
-def format_quiz_for_display(results):
-    output = []
-
-    for segment in results:
-        topic = segment["topic_name"]
-        segment_num = segment.get("segment_number", 1)
-
-        output.append(f"\n\n{'='*40}")
-        output.append(f"SEGMENT {segment_num}: {topic}")
-        output.append(f"{'='*40}\n")
-
-        output.append("KEY CONCEPTS:")
-        for concept in segment["key_concepts"]:
-            output.append(f"• {concept}")
-
-        output.append("\nSUMMARY:")
-        output.append(segment["summary"])
-
-        output.append("\nQUIZ QUESTIONS:")
-        for i, q in enumerate(segment["quiz_questions"]):
-            output.append(f"\n{i+1}. {q['question']}")
-
-            for j, option in enumerate(q['options']):
-                letter = chr(97 + j).upper()
-                correct_marker = " ✓" if option["correct"] else ""
-                output.append(f"   {letter}. {option['text']}{correct_marker}")
-
-    return "\n".join(output)
-
-def save_quiz_json(all_segments):
-    json_data = {"segments": all_segments}
-    json_filename = "generated_quiz.json"
-    with open(json_filename, "w", encoding="utf-8") as f:
-        json.dump(json_data, f, indent=2)
-    return json_filename
-
-
-def process_text(transcript_text, google_api_key):
-    if not transcript_text:
-        return "No text to analyze", None, None
-    text_parts = split_text_by_tokens(transcript_text)
-
-    all_segments = []
-    segment_counter = 1
-
-    for part in text_parts:
-        analysis = analyze_segment_with_gemini(part, google_api_key)
-
-        if "segments" in analysis:
-            for segment in analysis["segments"]:
-                segment["segment_number"] = segment_counter
-                all_segments.append(segment)
-                segment_counter += 1
-
-    formatted_quiz = format_quiz_for_display(all_segments)
-
-    quiz_filename = "generated_quiz.txt"
-    with open(quiz_filename, "w", encoding="utf-8") as f:
-        f.write(formatted_quiz)
-        
-    json_filename = save_quiz_json(all_segments)
-
-    return formatted_quiz, quiz_filename, json_filename
+import re
+import json
+import os
+from langchain_google_genai import ChatGoogleGenerativeAI
+from transformers import AutoTokenizer
+from huggingface_hub import login
+
+hf_token = os.environ.get('HF_TOKEN', None)
+login(token=hf_token)
+
+tokenizer = AutoTokenizer.from_pretrained("answerdotai/ModernBERT-base", use_auth_token=hf_token)
+
+def clean_text(text):
+    text = re.sub(r'\[speaker_\d+\]', '', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def split_text_by_tokens(text, max_tokens=12000):
+    text = clean_text(text)
+    tokens = tokenizer.encode(text)
+
+    if len(tokens) <= max_tokens:
+        return [text]
+
+    split_point = len(tokens) // 2
+
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+
+    first_half = []
+    second_half = []
+
+    current_tokens = 0
+    for sentence in sentences:
+        sentence_tokens = len(tokenizer.encode(sentence))
+
+        if current_tokens + sentence_tokens <= split_point:
+            first_half.append(sentence)
+            current_tokens += sentence_tokens
+        else:
+            second_half.append(sentence)
+
+    return [" ".join(first_half), " ".join(second_half)]
+
+def analyze_segment_with_gemini(segment_text, google_api_key, course_name="", section_name="", lesson_name=""):
+    os.environ["GOOGLE_API_KEY"] = google_api_key
+
+    llm = ChatGoogleGenerativeAI(
+        model="gemini-2.0-flash",
+        temperature=0.7,
+        max_tokens=None,
+        timeout=None,
+        max_retries=3
+    )
+
+    prompt = f"""
+        Analyze the following text and identify distinct segments within it and do text segmentation:
+        1. Segments should be STRICTLY max=15
+        2. For each segment/topic you identify:
+           - Provide a SPECIFIC and UNIQUE topic name (3-5 words) that clearly differentiates it from other segments
+           - List 3-5 key concepts discussed in that segment (be precise and avoid repetition between segments)
+           - Write a brief summary of that segment (3-5 sentences)
+           - Create 5 high-quality, meaningful quiz questions based DIRECTLY on the content in that segment only
+           - Questions and answers should be only from the content of the segment
+        For each quiz question:
+        - Create one correct answer that comes DIRECTLY from the text
+        - Create two plausible but incorrect answers
+        - IMPORTANT: Ensure all answer options have similar length (± 3 words)
+        - Ensure the correct answer is clearly indicated with a ✓ symbol
+       - Questions should **require actual understanding**, NOT just basic fact recall.
+        - Questions Are **non-trivial**, encourage deeper thinking, and **avoid surface-level facts**.
+        - Are **directly based on the segment's content** (not inferred from the summary).
+        - Do **not include questions about document structure** (e.g., title, number of paragraphs).
+        - Do **not generate overly generic or obvious questions** (e.g., "What is mentioned in the text?").
+        - Focus on **core ideas, logical reasoning, and conceptual understanding**
+    ADDITIONAL REQUIREMENT:
+    - First, detect the language of the original text.
+    - Generate all questions and answers in the same language as the original text.
+    - If the text is in Russian, generate questions and answers in Russian.
+    - If the text is in another language, generate questions in that original language.
+    
+    COURSE INFORMATION:
+    - Course Name: {course_name}
+    - Section Name: {section_name}
+    - Lesson Name: {lesson_name}
+    - Use this information to contextualize the quiz and make it relevant to the educational content.
+    - Include this information in the JSON response structure.
+        
+    Text:
+    {segment_text}
+        
+    Format your response as JSON with the following structure:
+    {{
+        "course_info": {{
+            "course_name": "{course_name}",
+            "section_name": "{section_name}",
+            "lesson_name": "{lesson_name}"
+        }},
+        "segments": [
+            {{
+                "topic_name": "Unique and Specific Topic Name",
+                "key_concepts": ["concept1", "concept2", "concept3"],
+                "summary": "Brief summary of this segment.",
+                "quiz_questions": [
+                    {{
+                        "question": "Question text?",
+                        "options": [
+                            {{
+                                "text": "Option A",
+                                "correct": false
+                            }},
+                            {{
+                                "text": "Option B",
+                                "correct": true
+                            }},
+                            {{
+                                "text": "Option C",
+                                "correct": false
+                            }}
+                        ]
+                    }}
+                ]
+            }}
+        ]
+    }}
+    IMPORTANT: Each segment must have a DISTINCT topic name that clearly differentiates it from others.
+    - **Do NOT repeat** key concepts across multiple segments unless absolutely necessary.
+    - **Ensure the quiz questions challenge the reader** and **are not easily guessable**.
+    - **Tailor the content to fit within the context of the specified course, section, and lesson.**
+    """
+
+    try:
+        response = llm.invoke(prompt)
+        response_text = response.content
+
+        json_match = re.search(r'\{[\s\S]*\}', response_text)
+        if json_match:
+            return json.loads(json_match.group(0))
+        else:
+            return json.loads(response_text)
+    except Exception as e:
+        print(f"Error in Gemini analysis: {e}")
+        return {
+            "course_info": {
+                "course_name": course_name,
+                "section_name": section_name,
+                "lesson_name": lesson_name
+            },
+            "segments": [
+                {
+                    "topic_name": "Analysis Error",
+                    "key_concepts": ["Could not process text"],
+                    "summary": "An error occurred during text analysis.",
+                    "quiz_questions": []
+                }
+            ]
+        }
+
+def format_quiz_for_display(results):
+    output = []
+    
+    if "course_info" in results:
+        course_info = results["course_info"]
+        output.append(f"{'='*40}")
+        output.append(f"COURSE: {course_info.get('course_name', 'N/A')}")
+        output.append(f"SECTION: {course_info.get('section_name', 'N/A')}")
+        output.append(f"LESSON: {course_info.get('lesson_name', 'N/A')}")
+        output.append(f"{'='*40}\n")
+    
+    segments = results.get("segments", [])
+    for i, segment in enumerate(segments):
+        topic = segment["topic_name"]
+        segment_num = i + 1 
+        output.append(f"\n\n{'='*40}")
+        output.append(f"SEGMENT {segment_num}: {topic}")
+        output.append(f"{'='*40}\n")
+        output.append("KEY CONCEPTS:")
+        for concept in segment["key_concepts"]:
+            output.append(f"• {concept}")
+        output.append("\nSUMMARY:")
+        output.append(segment["summary"])
+        output.append("\nQUIZ QUESTIONS:")
+        for i, q in enumerate(segment["quiz_questions"]):
+            output.append(f"\n{i+1}. {q['question']}")
+            for j, option in enumerate(q['options']):
+                letter = chr(97 + j).upper()
+                correct_marker = " ✓" if option["correct"] else ""
+                output.append(f"   {letter}. {option['text']}{correct_marker}")
+    return "\n".join(output)
+
+
+def save_quiz_json(results):
+    json_filename = "generated_quiz.json"
+    with open(json_filename, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2)
+    return json_filename
+
+def process_text(transcript_text, google_api_key, course_name="", section_name="", lesson_name=""):
+    if not transcript_text:
+        return "No text to analyze", None, None
+    text_parts = split_text_by_tokens(transcript_text)
+
+    all_results = {
+        "course_info": {
+            "course_name": course_name,
+            "section_name": section_name,
+            "lesson_name": lesson_name
+        },
+        "segments": []
+    }
+    segment_counter = 1
+
+    for part in text_parts:
+        analysis = analyze_segment_with_gemini(part, google_api_key, course_name, section_name, lesson_name)
+
+        if "segments" in analysis:
+            for segment in analysis["segments"]:
+                segment["segment_number"] = segment_counter
+                all_results["segments"].append(segment)
+                segment_counter += 1
+
+    formatted_quiz = format_quiz_for_display(all_results)
+
+    quiz_filename = "generated_quiz.txt"
+    with open(quiz_filename, "w", encoding="utf-8") as f:
+        f.write(formatted_quiz)
+        
+    json_filename = save_quiz_json(all_results)
+
+    return formatted_quiz, quiz_filename, json_filename