Update app.py

app.py

@@ -173,13 +173,25 @@ class PDFRAGSystem:
             # 텍스트를 청크로 분할
             chunks = self.text_splitter.split_text(pdf_data["full_text"])
             
+            if not chunks:
+                print("Warning: No chunks created from PDF")
+                return {"success": False, "error": "No text content found in PDF"}
+            
+            print(f"Created {len(chunks)} chunks from PDF")
+            
             # 청크 저장
             self.document_chunks[doc_id] = chunks
             
-            # 임베딩 생성
+            # 임베딩 생성 (선택적)
             if self.embedder:
-                embeddings = self.embedder.encode(chunks)
-                self.embeddings_store[doc_id] = embeddings
+                try:
+                    print("Generating embeddings...")
+                    embeddings = self.embedder.encode(chunks)
+                    self.embeddings_store[doc_id] = embeddings
+                    print(f"Generated {len(embeddings)} embeddings")
+                except Exception as e:
+                    print(f"Warning: Failed to generate embeddings: {e}")
+                    # 임베딩 실패해도 계속 진행
             
             # 문서 정보 저장
             self.documents[doc_id] = {
@@ -188,6 +200,9 @@ class PDFRAGSystem:
                 "upload_time": datetime.now().isoformat()
             }
             
+            # 디버그: 첫 번째 청크 출력
+            print(f"First chunk preview: {chunks[0][:200]}...")
+            
             return {
                 "success": True,
                 "doc_id": doc_id,
@@ -197,6 +212,7 @@ class PDFRAGSystem:
             }
             
         except Exception as e:
+            print(f"Error processing PDF: {e}")
             return {"success": False, "error": str(e)}
     
     def search_relevant_chunks(self, query: str, doc_ids: List[str], top_k: int = 3) -> List[Dict]:
@@ -205,54 +221,75 @@ class PDFRAGSystem:
         
         print(f"Searching chunks for query: '{query[:50]}...' in {len(doc_ids)} documents")
         
-        if self.embedder and self.embeddings_store:
-            # 임베딩 기반 검색
-            query_embedding = self.embedder.encode([query])[0]
+        # 먼저 문서가 있는지 확인
+        for doc_id in doc_ids:
+            if doc_id not in self.document_chunks:
+                print(f"Warning: Document {doc_id} not found in chunks")
+                continue
+                
+            chunks = self.document_chunks[doc_id]
+            print(f"Document {doc_id} has {len(chunks)} chunks")
             
-            for doc_id in doc_ids:
-                if doc_id in self.embeddings_store and doc_id in self.document_chunks:
+            # 임베딩 기반 검색 시도
+            if self.embedder and doc_id in self.embeddings_store:
+                try:
+                    query_embedding = self.embedder.encode([query])[0]
                     doc_embeddings = self.embeddings_store[doc_id]
-                    chunks = self.document_chunks[doc_id]
                     
-                    # 코사인 유사도 계산
+                    # 코사인 유사도 계산 (안전하게)
                     similarities = []
-                    for emb in doc_embeddings:
-                        sim = np.dot(query_embedding, emb) / (np.linalg.norm(query_embedding) * np.linalg.norm(emb))
-                        similarities.append(sim)
-                    
-                    # 상위 청크 선택 - 임계값 낮춤
-                    top_indices = np.argsort(similarities)[-top_k:][::-1]
+                    for i, emb in enumerate(doc_embeddings):
+                        try:
+                            query_norm = np.linalg.norm(query_embedding)
+                            emb_norm = np.linalg.norm(emb)
+                            
+                            if query_norm > 0 and emb_norm > 0:
+                                sim = np.dot(query_embedding, emb) / (query_norm * emb_norm)
+                                similarities.append(sim)
+                            else:
+                                similarities.append(0.0)
+                        except Exception as e:
+                            print(f"Error calculating similarity for chunk {i}: {e}")
+                            similarities.append(0.0)
                     
-                    for idx in top_indices:
-                        if similarities[idx] > 0.1:  # 임계값을 0.2에서 0.1로 낮춤
-                            all_relevant_chunks.append({
-                                "content": chunks[idx],
-                                "doc_name": self.documents[doc_id]["metadata"]["file_name"],
-                                "similarity": similarities[idx]
-                            })
-                            print(f"Found chunk with similarity: {similarities[idx]:.3f}")
-        else:
-            # 키워드 기반 검색
-            print("Using keyword-based search (embedder not available)")
-            query_keywords = set(query.lower().split())
+                    # 상위 청크 선택
+                    if similarities:
+                        top_indices = np.argsort(similarities)[-min(top_k, len(similarities)):][::-1]
+                        
+                        for idx in top_indices:
+                            if idx < len(chunks):  # 인덱스 범위 확인
+                                all_relevant_chunks.append({
+                                    "content": chunks[idx],
+                                    "doc_name": self.documents[doc_id]["metadata"]["file_name"],
+                                    "similarity": similarities[idx]
+                                })
+                                print(f"Added chunk {idx} with similarity: {similarities[idx]:.3f}")
+                except Exception as e:
+                    print(f"Error in embedding search: {e}")
+                    # 임베딩 실패시 폴백
             
-            for doc_id in doc_ids:
-                if doc_id in self.document_chunks:
-                    chunks = self.document_chunks[doc_id]
-                    for i, chunk in enumerate(chunks):  # 모든 청크 검색
-                        chunk_lower = chunk.lower()
-                        score = sum(1 for keyword in query_keywords if keyword in chunk_lower)
-                        if score > 0:
-                            all_relevant_chunks.append({
-                                "content": chunk[:800],  # 더 긴 청크 사용
-                                "doc_name": self.documents[doc_id]["metadata"]["file_name"],
-                                "similarity": score / len(query_keywords) if query_keywords else 0
-                            })
+            # 임베딩이 없거나 실패한 경우 - 간단히 처음 N개 청크 반환
+            if not all_relevant_chunks:
+                print(f"Falling back to simple chunk selection for {doc_id}")
+                for i in range(min(top_k, len(chunks))):
+                    all_relevant_chunks.append({
+                        "content": chunks[i],
+                        "doc_name": self.documents[doc_id]["metadata"]["file_name"],
+                        "similarity": 1.0 - (i * 0.1)  # 순서대로 가중치
+                    })
+                    print(f"Added chunk {i} (fallback)")
         
-        # 정렬 및 반환
+        # 유사도 기준 정렬
         all_relevant_chunks.sort(key=lambda x: x.get('similarity', 0), reverse=True)
+        
+        # 상위 K개 선택
         result = all_relevant_chunks[:top_k]
         print(f"Returning {len(result)} chunks")
+        
+        # 디버그: 첫 번째 청크 내용 일부 출력
+        if result:
+            print(f"First chunk preview: {result[0]['content'][:100]}...")
+        
         return result
     
     def create_rag_prompt(self, query: str, doc_ids: List[str], top_k: int = 3) -> tuple:
@@ -262,10 +299,23 @@ class PDFRAGSystem:
         relevant_chunks = self.search_relevant_chunks(query, doc_ids, top_k)
         
         if not relevant_chunks:
-            print("No relevant chunks found")
-            return query, ""
+            print("No relevant chunks found - checking if documents exist")
+            # 문서가 있는데 청크를 못 찾은 경우, 첫 번째 청크라도 사용
+            for doc_id in doc_ids:
+                if doc_id in self.document_chunks and self.document_chunks[doc_id]:
+                    print(f"Using first chunk from {doc_id} as fallback")
+                    relevant_chunks = [{
+                        "content": self.document_chunks[doc_id][0],
+                        "doc_name": self.documents[doc_id]["metadata"]["file_name"],
+                        "similarity": 0.5
+                    }]
+                    break
+            
+            if not relevant_chunks:
+                print("No documents or chunks available")
+                return query, ""
         
-        print(f"Found {len(relevant_chunks)} relevant chunks")
+        print(f"Using {len(relevant_chunks)} chunks for context")
         
         # 컨텍스트 구성
         context_parts = []
@@ -274,15 +324,18 @@ class PDFRAGSystem:
         
         for i, chunk in enumerate(relevant_chunks, 1):
             context_parts.append(f"\n[Document Reference {i} - {chunk['doc_name']}]")
-            content = chunk['content'][:500] if len(chunk['content']) > 500 else chunk['content']
+            # 청크 크기 증가
+            content = chunk['content'][:1000] if len(chunk['content']) > 1000 else chunk['content']
             context_parts.append(content)
-            print(f"Added chunk {i} with similarity: {chunk.get('similarity', 0):.3f}")
+            print(f"Added chunk {i} ({len(content)} chars) with similarity: {chunk.get('similarity', 0):.3f}")
         
         context_parts.append("\n" + "=" * 40)
         
         context = "\n".join(context_parts)
         enhanced_query = f"{context}\n\nQuestion: {query}\n\nAnswer based on the document context provided above:"
         
+        print(f"Enhanced query length: {len(enhanced_query)} chars (original: {len(query)} chars)")
+        
         return enhanced_query, context
 
 # Initialize model and RAG system