Update app.py

app.py

@@ -894,10 +894,12 @@ def export_to_docx(episodes: List[Dict], genre: str, title: str = "") -> bytes:
     doc.save(bytes_io)
     bytes_io.seek(0)
     return bytes_io.getvalue()
+
+    
 def generate_random_webnovel_theme(genre: str, language: str) -> str:
     """Generate random web novel theme using novel_themes.json and LLM"""
     try:
-        # Load novel_themes.json with error handling
+        # Load novel_themes.json with better error handling
         json_path = Path("novel_themes.json")
         if not json_path.exists():
             logger.warning("novel_themes.json not found, using fallback")
@@ -906,93 +908,165 @@ def generate_random_webnovel_theme(genre: str, language: str) -> str:
         try:
             with open(json_path, 'r', encoding='utf-8') as f:
                 content = f.read()
-                # Try to fix common JSON errors
-                content = content.replace("'", '"')  # Replace single quotes with double quotes
-                content = re.sub(r',\s*}', '}', content)  # Remove trailing commas before }
-                content = re.sub(r',\s*]', ']', content)  # Remove trailing commas before ]
+                
+                # Remove comments from JSON (/* */ style)
+                content = re.sub(r'/\*.*?\*/', '', content, flags=re.DOTALL)
+                # Remove single line comments (// style)
+                content = re.sub(r'//.*$', '', content, flags=re.MULTILINE)
+                # Remove trailing commas before } or ]
+                content = re.sub(r',\s*([}\]])', r'\1', content)
+                
+                # Handle all variations of placeholder patterns
+                content = re.sub(r'\.\.\.\s*\(기존.*?\)\s*\.\.\.', '[]', content)
+                content = re.sub(r'\.\.\.\(기존.*?\)\.\.\.', '[]', content)
+                content = re.sub(r'\{ \.\.\. \(기존 그대로\) \}', '{}', content)
+                content = re.sub(r'\{\s*\.\.\.\s*\(기존 그대로\)\s*\}', '{}', content)
+                content = re.sub(r'\{ \.\.\. \}', '{}', content)
+                content = re.sub(r'\{\s*\.\.\.\s*\}', '{}', content)
+                content = re.sub(r'\[ \.\.\. \]', '[]', content)
+                content = re.sub(r'\[\s*\.\.\.\s*\]', '[]', content)
+                
+                # Handle ellipsis in strings
+                content = re.sub(r'"[^"]*\.\.\.[^"]*"', '""', content)
+                
+                # Debug: save cleaned JSON for inspection
+                with open('novel_themes_cleaned.json', 'w', encoding='utf-8') as debug_file:
+                    debug_file.write(content)
+                
+                # Parse JSON
                 themes_data = json.loads(content)
+                logger.info("Successfully parsed novel_themes.json")
+                
         except json.JSONDecodeError as e:
-            logger.error(f"JSON parsing error: {e}")
-            logger.error(f"Error at position: {e.pos if hasattr(e, 'pos') else 'unknown'}")
-            # If JSON parsing fails, use LLM with genre-specific prompt
+            logger.error(f"JSON parsing error at line {e.lineno}, column {e.colno}: {e.msg}")
+            if hasattr(e, 'pos'):
+                error_context = content[max(0, e.pos-100):e.pos+100]
+                logger.error(f"Context around error: ...{error_context}...")
+            # Save problematic content for debugging
+            with open('novel_themes_error.json', 'w', encoding='utf-8') as error_file:
+                error_file.write(content)
             return generate_theme_with_llm_only(genre, language)
         
-        # Map genres to theme data
+        # Map genres to theme data - updated mapping
         genre_mapping = {
-            "로맨스": ["romance_fantasy_villainess", "villainess_wants_to_be_lazy", "chaebol_family_intrigue"],
-            "로판": ["romance_fantasy_villainess", "BL_novel_transmigration", "regression_childcare"],
-            "판타지": ["system_constellation_hunter", "tower_ascension_challenger", "necromancer_solo_leveling"],
-            "현판": ["system_constellation_hunter", "chaebol_family_intrigue", "post_apocalypse_survival"],
-            "무협": ["regression_revenge_pro", "necromancer_solo_leveling"],
-            "미스터리": ["post_apocalypse_survival", "tower_ascension_challenger"],
-            "라이트노벨": ["BL_novel_transmigration", "villainess_wants_to_be_lazy"]
+            "로맨스": ["romance_fantasy_villainess", "villainess_wants_to_be_lazy", "office_romance_rivals", "chaebol_family_intrigue"],
+            "로판": ["romance_fantasy_villainess", "BL_novel_transmigration", "regression_childcare", "saeguk_court_intrigue"],
+            "판타지": ["system_constellation_hunter", "tower_ascension_challenger", "necromancer_solo_leveling", "ai_dungeon_masters"],
+            "현판": ["system_constellation_hunter", "chaebol_family_intrigue", "post_apocalypse_survival", "esports_king_prodigy", "vr_streamer_ranker"],
+            "무협": ["regression_revenge_pro", "necromancer_solo_leveling", "exorcist_k_cult"],
+            "���스터리": ["post_apocalypse_survival", "tower_ascension_challenger", "survival_reality_show"],
+            "라이트노벨": ["BL_novel_transmigration", "villainess_wants_to_be_lazy", "vr_streamer_ranker", "healing_cafe_fantasy", "idol_regression_superstar"]
         }
         
         # Get relevant core genres for selected genre
         relevant_genres = genre_mapping.get(genre, ["regression_revenge_pro"])
-        selected_genre_key = random.choice(relevant_genres)
         
-        # Get genre data
-        core_genre = themes_data["core_genres"].get(selected_genre_key, {})
+        # Filter out genres that might not exist in the JSON
+        available_genres = []
+        core_genres = themes_data.get("core_genres", {})
+        
+        # Debug log available genres
+        logger.debug(f"Available core genres: {list(core_genres.keys())}")
+        
+        for genre_key in relevant_genres:
+            if genre_key in core_genres:
+                available_genres.append(genre_key)
+        
+        if not available_genres:
+            logger.warning(f"No matching genres found for {genre}, available: {list(core_genres.keys())[:5]}...")
+            # Try to use any available genre
+            available_genres = list(core_genres.keys())[:3]
+            
+        selected_genre_key = random.choice(available_genres)
+        logger.debug(f"Selected genre key: {selected_genre_key}")
+        
+        # Get genre data safely
+        core_genre = core_genres.get(selected_genre_key, {})
         compatible_elements = core_genre.get("compatible_elements", {})
         
-        # Select random elements
+        # Select random elements with fallbacks
         character_keys = compatible_elements.get("characters", [])
+        if not character_keys:
+            # Get any available characters
+            all_characters = list(themes_data.get("characters", {}).keys())
+            character_keys = all_characters[:4] if all_characters else ["betrayed_protagonist"]
+            
         selected_character_key = random.choice(character_keys) if character_keys else "betrayed_protagonist"
         
-        # Get character variations
-        character_data = themes_data["characters"].get(selected_character_key, {})
+        # Get character data safely
+        characters_data = themes_data.get("characters", {})
+        character_data = characters_data.get(selected_character_key, {})
         character_variations = character_data.get("variations", [])
-        character_desc = random.choice(character_variations) if character_variations else ""
-        character_traits = character_data.get("traits", [])
         
-        # Get settings
+        # Filter out empty or placeholder variations
+        valid_variations = [v for v in character_variations if v and not v.startswith("...") and len(v) > 10]
+        character_desc = random.choice(valid_variations) if valid_variations else "주인공은 특별한 운명을 타고났다."
+        
+        character_traits = character_data.get("traits", ["결단력", "성장형", "매력적"])
+        
+        # Get settings safely
         settings = compatible_elements.get("settings", [])
-        selected_setting = random.choice(settings) if settings else ""
+        if not settings:
+            # Try to get from general settings
+            all_settings_categories = themes_data.get("settings", {})
+            for category_name, category_settings in all_settings_categories.items():
+                if isinstance(category_settings, list):
+                    valid_settings = [s for s in category_settings if s and not s.startswith("...") and len(s) > 5]
+                    settings.extend(valid_settings)
         
-        # Get all available settings for more variety
-        all_settings = themes_data.get("settings", {})
-        setting_details = []
-        for setting_list in all_settings.values():
-            setting_details.extend(setting_list)
-        specific_setting = random.choice(setting_details) if setting_details else selected_setting
+        selected_setting = random.choice(settings) if settings else "현대 도시"
         
-        # Get mechanics
-        mechanics_keys = list(themes_data.get("core_mechanics", {}).keys())
-        selected_mechanic = random.choice(mechanics_keys) if mechanics_keys else ""
-        mechanic_data = themes_data["core_mechanics"].get(selected_mechanic, {})
-        plot_points = mechanic_data.get("plot_points", [])
-        reader_questions = mechanic_data.get("reader_questions", [])
+        # Get mechanics safely
+        mechanics_data = themes_data.get("core_mechanics", {})
+        mechanics_keys = [k for k in mechanics_data.keys() if k]
+        selected_mechanic = random.choice(mechanics_keys) if mechanics_keys else "regression_loop_mastery"
         
-        # Get hooks
-        hook_types = list(themes_data.get("episode_hooks", {}).keys())
+        mechanic_info = mechanics_data.get(selected_mechanic, {})
+        plot_points = mechanic_info.get("plot_points", [])
+        reader_questions = mechanic_info.get("reader_questions", [])
+        
+        # Filter valid plot points and questions
+        valid_plot_points = [p for p in plot_points if p and not p.startswith("...") and len(p) > 10]
+        valid_questions = [q for q in reader_questions if q and not q.startswith("...") and len(q) > 10]
+        
+        # Get hooks safely
+        hooks_data = themes_data.get("episode_hooks", {})
+        hook_types = list(hooks_data.keys())
         selected_hook_type = random.choice(hook_types) if hook_types else "introduction"
-        hooks = themes_data["episode_hooks"].get(selected_hook_type, [])
-        selected_hook = random.choice(hooks) if hooks else ""
         
-        # Get items/artifacts
-        item_categories = list(themes_data.get("key_items_and_artifacts", {}).keys())
-        if item_categories and genre in ["판타지", "현판", "무협"]:
-            selected_category = random.choice(item_categories)
-            items = themes_data["key_items_and_artifacts"].get(selected_category, [])
-            selected_item = random.choice(items) if items else ""
-        else:
-            selected_item = ""
+        hooks = hooks_data.get(selected_hook_type, [])
+        valid_hooks = [h for h in hooks if h and not h.startswith("...") and len(h) > 10]
+        selected_hook = random.choice(valid_hooks) if valid_hooks else "운명적인 만남이 시작되었다."
+        
+        # Get items/artifacts for certain genres
+        selected_item = ""
+        if genre in ["판타지", "현판", "무협"]:
+            items_data = themes_data.get("key_items_and_artifacts", {})
+            item_categories = list(items_data.keys())
+            if item_categories:
+                selected_category = random.choice(item_categories)
+                items = items_data.get(selected_category, [])
+                valid_items = [i for i in items if i and not i.startswith("...") and len(i) > 10]
+                selected_item = random.choice(valid_items) if valid_items else ""
         
-        # Get plot twists
-        twist_categories = list(themes_data.get("plot_twists_and_cliches", {}).keys())
+        # Get plot twists safely
+        twists_data = themes_data.get("plot_twists_and_cliches", {})
+        twist_categories = list(twists_data.keys())
+        selected_twist = ""
         if twist_categories:
             selected_twist_cat = random.choice(twist_categories)
-            twists = themes_data["plot_twists_and_cliches"].get(selected_twist_cat, [])
-            selected_twist = random.choice(twists) if twists else ""
-        else:
-            selected_twist = ""
+            twists = twists_data.get(selected_twist_cat, [])
+            valid_twists = [t for t in twists if t and not t.startswith("...") and len(t) > 10]
+            selected_twist = random.choice(valid_twists) if valid_twists else ""
         
         # Check for fusion genres
         fusion_genres = themes_data.get("fusion_genres", {})
-        fusion_options = list(fusion_genres.values())
+        fusion_options = [v for v in fusion_genres.values() if v and not v.startswith("...") and len(v) > 10]
         selected_fusion = random.choice(fusion_options) if fusion_options and random.random() > 0.7 else ""
         
+        # Log selected elements for debugging
+        logger.debug(f"Selected elements - Genre: {selected_genre_key}, Character: {selected_character_key}, Mechanic: {selected_mechanic}")
+        
         # Now use LLM to create a coherent theme from these elements
         system = WebNovelSystem()
         
@@ -1001,11 +1075,11 @@ def generate_random_webnovel_theme(genre: str, language: str) -> str:
             prompt = f"""다음 요소들을 활용하여 {genre} 장르의 매력적인 웹소설을 기획하세요:
 
 【선택된 요소들】
-- 핵심 장르: {selected_genre_key}
+- 핵심 장르: {selected_genre_key.replace('_', ' ')}
 - 캐릭터: {character_desc}
 - 캐릭터 특성: {', '.join(character_traits[:3])}
-- 배경: {specific_setting}
-- 핵심 메커니즘: {selected_mechanic}
+- 배경: {selected_setting}
+- 핵심 메커니즘: {selected_mechanic.replace('_', ' ')}
 {"- 아이템: " + selected_item if selected_item else ""}
 {"- 반전 요소: " + selected_twist if selected_twist else ""}
 {"- 퓨전 설정: " + selected_fusion if selected_fusion else ""}
@@ -1014,7 +1088,7 @@ def generate_random_webnovel_theme(genre: str, language: str) -> str:
 {selected_hook}
 
 【독자를 사로잡을 질문들】
-{chr(10).join(reader_questions[:2]) if reader_questions else ""}
+{chr(10).join(valid_questions[:2]) if valid_questions else "독자의 호기심을 자극하는 질문들"}
 
 다음 형식으로 정확히 작성하세요:
 
@@ -1036,11 +1110,11 @@ def generate_random_webnovel_theme(genre: str, language: str) -> str:
             prompt = f"""Create an engaging web novel for {genre} genre using these elements:
 
 【Selected Elements】
-- Core genre: {selected_genre_key}
+- Core genre: {selected_genre_key.replace('_', ' ')}
 - Character: {character_desc}
 - Character traits: {', '.join(character_traits[:3])}
-- Setting: {specific_setting}
-- Core mechanism: {selected_mechanic}
+- Setting: {selected_setting}
+- Core mechanism: {selected_mechanic.replace('_', ' ')}
 {"- Item: " + selected_item if selected_item else ""}
 {"- Twist: " + selected_twist if selected_twist else ""}
 {"- Fusion: " + selected_fusion if selected_fusion else ""}
@@ -1048,6 +1122,9 @@ def generate_random_webnovel_theme(genre: str, language: str) -> str:
 【Reference Hook】
 {selected_hook}
 
+【Questions to captivate readers】
+{chr(10).join(valid_questions[:2]) if valid_questions else "Questions that spark reader curiosity"}
+
 Format exactly as follows:
 
 📖 **Title:**
@@ -1068,10 +1145,11 @@ Format exactly as follows:
         messages = [{"role": "user", "content": prompt}]
         generated_theme = system.call_llm_sync(messages, "writer", language)
         
+        logger.info("Successfully generated theme using JSON elements")
         return generated_theme
         
     except Exception as e:
-        logger.error(f"Error generating theme from JSON: {e}")
+        logger.error(f"Error generating theme from JSON: {e}", exc_info=True)
         return generate_fallback_theme(genre, language)
 
 def generate_fallback_theme(genre: str, language: str) -> str: