Upload app.py

app.py

@@ -6,6 +6,8 @@ import torch
 import os
 from typing import List, Dict, Any
 import time
+import requests
+import re
 
 # Configure device
 device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -17,7 +19,7 @@ class RAGtimBot:
         self.knowledge_base = []
         self.embeddings = []
         self.initialize_models()
-        self.load_knowledge_base()
+        self.load_markdown_knowledge_base()
         
     def initialize_models(self):
         """Initialize the embedding model"""
@@ -33,84 +35,35 @@ class RAGtimBot:
             print(f"❌ Error loading embedding model: {e}")
             raise e
     
-    def load_knowledge_base(self):
-        """Load and process the knowledge base"""
-        print("Loading knowledge base...")
+    def load_markdown_knowledge_base(self):
+        """Load knowledge base from markdown files"""
+        print("Loading knowledge base from markdown files...")
         
-        # Comprehensive knowledge base about Raktim Mondol
-        self.knowledge_base = [
-            {
-                "id": "about_1",
-                "content": "Raktim Mondol is a PhD candidate in Computer Science & Engineering at UNSW Sydney. He is a researcher, data scientist, bioinformatician, biostatistician, LLM engineer, and father. His research focuses on deep learning-based prognosis and explainability for breast cancer. He is located at the School of Computer Science and Engineering, Building K17, UNSW Sydney, NSW 2052. Contact: [email protected], Phone: +61 412 936 237.",
-                "metadata": {"type": "about", "priority": 10}
-            },
-            {
-                "id": "education_1", 
-                "content": "Raktim Mondol is pursuing a PhD in Computer Science & Engineering at UNSW Sydney (2021-2025 Expected). His thesis is on 'Deep Learning Based Prognosis and Explainability for Breast Cancer'. He completed his MS by Research in Computer Science & Bioinformatics at RMIT University (2017-2019) with High Distinction. His master's thesis was 'Deep learning in classifying cancer subtypes, extracting relevant genes and identifying novel mutations'.",
-                "metadata": {"type": "education", "priority": 9}
-            },
-            {
-                "id": "research_llm",
-                "content": "Raktim's research focuses on Large Language Models (LLMs) including training, fine-tuning, and evaluating LLMs using parameter-efficient techniques like LoRA and QLoRA, with applications in retrieval-augmented generation, summarisation, and multi-hop reasoning. He works on Agentic AI & Multi-Agent Systems, designing autonomous, tool-using agents for reasoning, planning, and collaboration using frameworks like the Agent Development Kit.",
-                "metadata": {"type": "research", "priority": 9}
-            },
-            {
-                "id": "research_rag",
-                "content": "His expertise includes Retrieval-Augmented Generation (RAG), building hybrid search and generation pipelines integrating semantic and keyword-based retrieval using technologies like FAISS, BM25, ChromaDB, Weaviate, and Milvus for vector search and retrieval systems.",
-                "metadata": {"type": "research", "priority": 9}
-            },
-            {
-                "id": "skills_ai",
-                "content": "Raktim has expertise in Generative AI & LLM Toolkits including Hugging Face Transformers, LoRA/QLoRA (PEFT), LangChain, OpenAI API/Gemini Pro, GPTQ/GGUF, Prompt Engineering, Agent Development Kit, and RAG Pipelines. He is skilled in Multimodal & CV + NLP including CLIP/BLIP/LLaVA, Segment Anything (SAM), Visual Question Answering, and Multimodal Transformers.",
-                "metadata": {"type": "skills", "priority": 7}
-            },
-            {
-                "id": "skills_programming",
-                "content": "Programming languages: Python, R, SQL, LaTeX. Deep Learning Frameworks: PyTorch, TensorFlow. Cloud Computing: AWS, GCP, Galaxy. Development tools: Git, Jupyter Notebook, RStudio, Spyder. Statistical Analysis: Stata, SPSS, SAS, NCSS.",
-                "metadata": {"type": "skills", "priority": 7}
-            },
-            {
-                "id": "experience_current",
-                "content": "Raktim Mondol has been working as a Casual Academic at UNSW since July 2021, conducting laboratory and tutorial classes for Computer Vision, Neural Networks and Deep Learning, and Artificial Intelligence courses. He provides guidance to students and assists in course material development.",
-                "metadata": {"type": "experience", "priority": 8}
-            },
-            {
-                "id": "experience_rmit",
-                "content": "Previously, he was a Teaching Assistant at RMIT University (July 2017 - Oct 2019), conducting laboratory classes for Electronics, Software Engineering Design, Engineering Computing, and Introduction to Embedded Systems.",
-                "metadata": {"type": "experience", "priority": 8}
-            },
-            {
-                "id": "experience_lecturer",
-                "content": "He worked as a full-time Lecturer at World University of Bangladesh (Sep 2013 - Dec 2016), teaching Electrical Circuit I & II, Engineering Materials, Electronics I & II, Digital Logic Design, and supervising student projects and thesis.",
-                "metadata": {"type": "experience", "priority": 8}
-            },
-            {
-                "id": "publication_biofusion",
-                "content": "BioFusionNet: Deep Learning-Based Survival Risk Stratification in ER+ Breast Cancer Through Multifeature and Multimodal Data Fusion published in IEEE Journal of Biomedical and Health Informatics (2024). This work demonstrates novel multimodal fusion architecture combining histopathology, genomics, and clinical data with attention-based feature selection for interpretability.",
-                "metadata": {"type": "publications", "priority": 8}
-            },
-            {
-                "id": "publication_hist2rna",
-                "content": "hist2RNA: An Efficient Deep Learning Architecture to Predict Gene Expression from Breast Cancer Histopathology Images published in Cancers journal (2023). This enables gene expression profiling without expensive molecular assays, making personalized medicine more accessible.",
-                "metadata": {"type": "publications", "priority": 8}
-            },
-            {
-                "id": "publication_afexnet",
-                "content": "AFExNet: An Adversarial Autoencoder for Differentiating Breast Cancer Sub-types and Extracting Biologically Relevant Genes published in IEEE/ACM Transactions on Computational Biology and Bioinformatics (2021). Provides insights into cancer biology while achieving high classification accuracy.",
-                "metadata": {"type": "publications", "priority": 8}
-            },
-            {
-                "id": "statistics_expertise",
-                "content": "Raktim demonstrates exceptional proficiency in advanced statistical methods including survival analysis with weighted Cox proportional hazards models, multivariate regression analysis, hypothesis testing, correlation analysis with multiple-testing control, and comprehensive biostatistical applications. His BioFusionNet work achieved mean concordance index of 0.77 and time-dependent AUC of 0.84.",
-                "metadata": {"type": "statistics", "priority": 9}
-            },
-            {
-                "id": "awards",
-                "content": "Awards include: Doctoral Research Scholarship from UNSW Sydney (2021), Masters by Research with High Distinction from RMIT University (2019), RMIT Research Scholarships (2017), B.Sc. with High Distinction from BRAC University (2013), Vice Chancellor Award from BRAC University (2013), and Dean Awards (2010-2011).",
-                "metadata": {"type": "awards", "priority": 6}
-            }
+        # Reset knowledge base
+        self.knowledge_base = []
+        
+        # Load all markdown files
+        markdown_files = [
+            'about.md',
+            'research_details.md', 
+            'publications_detailed.md',
+            'skills_expertise.md',
+            'experience_detailed.md',
+            'statistics.md'
         ]
         
+        for filename in markdown_files:
+            try:
+                if os.path.exists(filename):
+                    with open(filename, 'r', encoding='utf-8') as f:
+                        content = f.read()
+                    self.process_markdown_file(content, filename)
+                    print(f"✅ Loaded {filename}")
+                else:
+                    print(f"⚠️ File not found: {filename}")
+            except Exception as e:
+                print(f"❌ Error loading {filename}: {e}")
+        
         # Generate embeddings for knowledge base
         print("Generating embeddings for knowledge base...")
         self.embeddings = []
@@ -127,6 +80,66 @@ class RAGtimBot:
         
         print(f"✅ Knowledge base loaded with {len(self.knowledge_base)} documents")
     
+    def process_markdown_file(self, content: str, filename: str):
+        """Process a markdown file and extract sections"""
+        # Determine file type and priority
+        file_type_map = {
+            'about.md': ('about', 10),
+            'research_details.md': ('research', 9),
+            'publications_detailed.md': ('publications', 8),
+            'skills_expertise.md': ('skills', 7),
+            'experience_detailed.md': ('experience', 8),
+            'statistics.md': ('statistics', 9)
+        }
+        
+        file_type, priority = file_type_map.get(filename, ('general', 5))
+        
+        # Split content into sections
+        sections = self.split_markdown_into_sections(content)
+        
+        for section in sections:
+            if len(section['content'].strip()) > 100:  # Only process substantial content
+                doc = {
+                    "id": f"{filename}_{section['title']}_{len(self.knowledge_base)}",
+                    "content": section['content'],
+                    "metadata": {
+                        "type": file_type,
+                        "priority": priority,
+                        "section": section['title'],
+                        "source": filename
+                    }
+                }
+                self.knowledge_base.append(doc)
+    
+    def split_markdown_into_sections(self, content: str) -> List[Dict[str, str]]:
+        """Split markdown content into sections based on headers"""
+        sections = []
+        lines = content.split('\n')
+        current_section = {'title': 'Introduction', 'content': ''}
+        
+        for line in lines:
+            # Check if line is a header
+            if line.startswith('#'):
+                # Save previous section if it has content
+                if current_section['content'].strip():
+                    sections.append(current_section.copy())
+                
+                # Start new section
+                header_level = len(line) - len(line.lstrip('#'))
+                title = line.lstrip('#').strip()
+                current_section = {
+                    'title': title,
+                    'content': line + '\n'
+                }
+            else:
+                current_section['content'] += line + '\n'
+        
+        # Add the last section
+        if current_section['content'].strip():
+            sections.append(current_section)
+        
+        return sections
+    
     def cosine_similarity(self, a, b):
         """Calculate cosine similarity between two vectors"""
         return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
@@ -150,8 +163,8 @@ class RAGtimBot:
                     "index": i
                 })
             
-            # Sort by similarity and return top_k
-            similarities.sort(key=lambda x: x["score"], reverse=True)
+            # Sort by similarity and priority
+            similarities.sort(key=lambda x: (x["score"], x["metadata"]["priority"]), reverse=True)
             return similarities[:top_k]
             
         except Exception as e:
@@ -168,12 +181,16 @@ class RAGtimBot:
             content_lower = doc["content"].lower()
             score = sum(content_lower.count(term) for term in query_terms)
             
+            # Add priority boost
+            priority_boost = doc["metadata"]["priority"] / 10
+            final_score = score + priority_boost
+            
             if score > 0:
                 results.append({
                     "id": doc["id"],
                     "content": doc["content"],
                     "metadata": doc["metadata"],
-                    "score": score,
+                    "score": final_score,
                     "index": i
                 })
         
@@ -181,7 +198,7 @@ class RAGtimBot:
         return results[:top_k]
 
 # Initialize the bot
-print("Initializing RAGtim Bot...")
+print("Initializing RAGtim Bot with markdown knowledge base...")
 bot = RAGtimBot()
 
 def search_only_api(query, top_k=5):
@@ -192,7 +209,8 @@ def search_only_api(query, top_k=5):
             "results": results,
             "query": query,
             "top_k": top_k,
-            "search_type": "semantic"
+            "search_type": "semantic",
+            "total_documents": len(bot.knowledge_base)
         }
     except Exception as e:
         print(f"Error in search API: {e}")
@@ -200,29 +218,63 @@ def search_only_api(query, top_k=5):
 
 def get_stats_api():
     """API endpoint for knowledge base statistics"""
+    # Calculate document distribution by type
+    doc_types = {}
+    sections_by_file = {}
+    
+    for doc in bot.knowledge_base:
+        doc_type = doc["metadata"]["type"]
+        source_file = doc["metadata"]["source"]
+        
+        doc_types[doc_type] = doc_types.get(doc_type, 0) + 1
+        sections_by_file[source_file] = sections_by_file.get(source_file, 0) + 1
+    
     return {
         "total_documents": len(bot.knowledge_base),
+        "document_types": doc_types,
+        "sections_by_file": sections_by_file,
         "model_name": "sentence-transformers/all-MiniLM-L6-v2",
         "embedding_dimension": 384,
-        "search_capabilities": ["Semantic Search", "GPU Accelerated", "Transformer Embeddings"],
-        "backend_type": "Hugging Face Space"
+        "search_capabilities": ["Semantic Search", "GPU Accelerated", "Transformer Embeddings", "Markdown Knowledge Base"],
+        "backend_type": "Hugging Face Space",
+        "knowledge_sources": list(sections_by_file.keys())
     }
 
 def chat_interface(message, history):
-    """Main chat interface function - now just for demo purposes"""
+    """Chat interface with markdown knowledge base"""
     if not message.strip():
-        return "Please ask me something about Raktim Mondol!"
+        return "Please ask me something about Raktim Mondol! I have comprehensive information loaded from his complete portfolio markdown files."
     
     try:
         # Search knowledge base
-        search_results = bot.search_knowledge_base(message, top_k=5)
+        search_results = bot.search_knowledge_base(message, top_k=6)
         
-        # Simple response for demo (in hybrid mode, DeepSeek will handle this)
         if search_results:
+            # Build comprehensive response
+            response_parts = []
+            response_parts.append(f"Based on my markdown knowledge base (found {len(search_results)} relevant sections):\n")
+            
+            # Use the best match as primary response
             best_match = search_results[0]
-            return f"Based on the search results (similarity: {best_match['score']:.2f}):\n\n{best_match['content']}\n\n[Note: In hybrid mode, DeepSeek LLM will generate more natural responses using this context]"
+            response_parts.append(f"**Primary Answer** (Relevance: {best_match['score']:.2f}):")
+            response_parts.append(f"Source: {best_match['metadata']['source']} - {best_match['metadata']['section']}")
+            response_parts.append(f"{best_match['content']}\n")
+            
+            # Add additional context if available
+            if len(search_results) > 1:
+                response_parts.append("**Additional Context:**")
+                for i, result in enumerate(search_results[1:3], 1):  # Show up to 2 additional results
+                    section_info = f"{result['metadata']['source']} - {result['metadata']['section']}"
+                    response_parts.append(f"{i}. {section_info} (Relevance: {result['score']:.2f})")
+                    # Add a brief excerpt
+                    excerpt = result['content'][:200] + "..." if len(result['content']) > 200 else result['content']
+                    response_parts.append(f"   {excerpt}\n")
+            
+            response_parts.append("\n[Note: This response is generated from your complete markdown knowledge base. In hybrid mode, DeepSeek LLM would generate more natural responses using this context.]")
+            
+            return "\n".join(response_parts)
         else:
-            return "I don't have specific information about that topic. Could you please ask something else about Raktim Mondol?"
+            return "I don't have specific information about that topic in my markdown knowledge base. Could you please ask something else about Raktim Mondol?"
         
     except Exception as e:
         print(f"Error in chat interface: {e}")
@@ -246,39 +298,53 @@ css = """
 # Create the main chat interface
 iface = gr.ChatInterface(
     fn=chat_interface,
-    title="🤖 RAGtim Bot - Hybrid Search Engine",
-    description="""
-    **Hybrid RAG System**: This Hugging Face Space provides GPU-accelerated semantic search that can be combined with external LLMs like DeepSeek for response generation.
+    title="🤖 RAGtim Bot - Markdown Knowledge Base",
+    description=f"""
+    **Complete Markdown Knowledge Base**: This Hugging Face Space loads all markdown files from Raktim Mondol's portfolio with **{len(bot.knowledge_base)} knowledge sections**.
+    
+    **Loaded Markdown Files:**
+    - 📄 **about.md** - Personal information, contact details, professional summary
+    - 🔬 **research_details.md** - Detailed research projects, methodologies, current work
+    - 📚 **publications_detailed.md** - Complete publication details, technical contributions
+    - 💻 **skills_expertise.md** - Comprehensive technical skills, tools, frameworks
+    - 💼 **experience_detailed.md** - Professional experience, teaching, research roles
+    - 📊 **statistics.md** - Statistical methods, biostatistics expertise, methodologies
     
     **Search Capabilities:**
     - 🔍 Semantic similarity search using transformers
-    - 🚀 GPU-accelerated embeddings
-    - 📊 Relevance scoring and ranking
-    - 🎯 Context-aware retrieval
+    - 🚀 GPU-accelerated embeddings with priority ranking
+    - 📊 Relevance scoring across all markdown content
+    - 🎯 Section-level granular search within each file
     
     **API Endpoints:**
-    - `/api/search` - Search-only functionality
-    - `/api/stats` - Knowledge base statistics
+    - `/api/search` - Search across complete markdown knowledge base
+    - `/api/stats` - Detailed statistics about loaded content
     
-    **Ask me about Raktim Mondol:**
-    - 🔬 Research in LLMs, RAG, and AI for healthcare
-    - 📚 Publications and academic work
-    - 💻 Technical skills and programming expertise
-    - 🎓 Education and academic background
-    - 👨‍🏫 Teaching and professional experience
-    - 📊 Statistical methods and biostatistics
+    **Ask me anything about Raktim Mondol:**
+    - Research projects, methodologies, and innovations
+    - Publications with technical details and impact
+    - Technical skills, programming expertise, and tools
+    - Educational background and academic achievements
+    - Professional experience and teaching roles
+    - Statistical methods and biostatistics applications
+    - Awards, recognition, and professional development
+    - Contact information and collaboration opportunities
     
-    **Note**: This demo shows search results. In hybrid mode, these results are passed to DeepSeek LLM for natural response generation.
+    **Note**: This demo shows search results from the complete markdown knowledge base. In hybrid mode, these results are passed to DeepSeek LLM for natural response generation.
     """,
     examples=[
         "What is Raktim's research about?",
-        "Tell me about his publications",
-        "What programming languages does he know?",
-        "What is his educational background?",
-        "How can I contact Raktim?",
-        "What is BioFusionNet?",
-        "Tell me about his LLM research",
-        "What statistical methods does he use?"
+        "Tell me about BioFusionNet in detail",
+        "What are his LLM and RAG expertise?",
+        "Describe his statistical methods and biostatistics work",
+        "What programming languages and frameworks does he use?",
+        "Tell me about his educational background",
+        "What is his current position at UNSW?",
+        "How can I contact Raktim for collaboration?",
+        "What awards and recognition has he received?",
+        "Explain his multimodal AI research",
+        "What is hist2RNA and its impact?",
+        "Tell me about his teaching experience"
     ],
     css=css,
     theme=gr.themes.Soft(
@@ -287,17 +353,17 @@ iface = gr.ChatInterface(
         neutral_hue="slate"
     ),
     chatbot=gr.Chatbot(
-        height=500,
+        height=600,
         show_label=False,
         container=True,
         bubble_full_width=False
     ),
     textbox=gr.Textbox(
-        placeholder="Ask me anything about Raktim Mondol...",
+        placeholder="Ask me anything about Raktim Mondol's research, skills, experience, publications...",
         container=False,
         scale=7
     ),
-    submit_btn="Search",
+    submit_btn="Search Knowledge Base",
     retry_btn="🔄 Retry",
     undo_btn="↩️ Undo",
     clear_btn="🗑️ Clear"
@@ -307,31 +373,32 @@ iface = gr.ChatInterface(
 search_api = gr.Interface(
     fn=search_only_api,
     inputs=[
-        gr.Textbox(label="Search Query", placeholder="Enter your search query..."),
-        gr.Slider(minimum=1, maximum=10, value=5, step=1, label="Top K Results")
+        gr.Textbox(label="Search Query", placeholder="Enter your search query about Raktim Mondol..."),
+        gr.Slider(minimum=1, maximum=15, value=5, step=1, label="Top K Results")
     ],
-    outputs=gr.JSON(label="Search Results"),
-    title="🔍 Search API",
-    description="Direct access to the semantic search functionality"
+    outputs=gr.JSON(label="Markdown Knowledge Base Search Results"),
+    title="🔍 Markdown Knowledge Base Search API",
+    description="Direct access to semantic search across all loaded markdown files"
 )
 
 stats_api = gr.Interface(
     fn=get_stats_api,
     inputs=[],
-    outputs=gr.JSON(label="Knowledge Base Statistics"),
-    title="📊 Stats API",
-    description="Knowledge base statistics and capabilities"
+    outputs=gr.JSON(label="Markdown Knowledge Base Statistics"),
+    title="📊 Knowledge Base Stats",
+    description="Detailed statistics about the loaded markdown knowledge base"
 )
 
 # Combine interfaces
 demo = gr.TabbedInterface(
     [iface, search_api, stats_api],
-    ["💬 Chat Demo", "🔍 Search API", "📊 Stats API"],
-    title="🤖 RAGtim Bot - Hybrid Search System"
+    ["💬 Markdown Chat", "🔍 Search API", "📊 Stats API"],
+    title="🤖 RAGtim Bot - Complete Markdown Knowledge Base"
 )
 
 if __name__ == "__main__":
-    print("🚀 Launching RAGtim Bot Hybrid Search System...")
+    print("🚀 Launching RAGtim Bot with Markdown Knowledge Base...")
+    print(f"📚 Loaded {len(bot.knowledge_base)} sections from markdown files")
     demo.launch(
         server_name="0.0.0.0",
         server_port=7860,