Update app.py

app.py

@@ -29,12 +29,21 @@ def extract_pdf_text_by_page(file) -> List[str]:
             return [page.get_text("text") for page in doc]
 
 def extract_pdf_text(file) -> str:
-    """Extracts text from a PDF file using pdfminer."""
-    if isinstance(file, str):
-        with open(file, 'rb') as f:
-            return extract_text(f, laparams=LAParams())
-    else:
-        return extract_text(file, laparams=LAParams())
+    """Extracts full text from a PDF file using PyMuPDF."""
+    try:
+        # Open the PDF file
+        doc = fitz.open(stream=file.read(), filetype="pdf") if not isinstance(file, str) else fitz.open(file)
+        full_text = ""
+        for page_num, page in enumerate(doc, start=1):
+            text = page.get_text("text")
+            full_text += text + "\n"
+            print(f"Extracted text from page {page_num}: {len(text)} characters.")
+        doc.close()
+        print(f"Total extracted text length: {len(full_text)} characters.")
+        return full_text
+    except Exception as e:
+        print(f"Error extracting text from PDF: {e}")
+        return ""
 
 def check_text_presence(full_text: str, search_terms: List[str]) -> Dict[str, bool]:
     """Checks for the presence of required terms in the text."""
@@ -94,32 +103,29 @@ def check_structure(full_text: str) -> Dict[str, bool]:
     }
 
 def check_language_issues(full_text: str) -> Dict[str, Any]:
-    """Check for issues with capitalization, hyphenation, punctuation, spacing, etc."""
-    language_tool = language_tool_python.LanguageTool('en-US')
-    matches = language_tool.check(full_text)
-    word_count = len(full_text.split())
-    issues_count = len(matches)
-    issues_per_1000 = (issues_count / word_count) * 1000 if word_count else 0
-    
-    serializable_matches = [
-        {
-            "message": match.message,
-            "replacements": match.replacements,
-            "offset": match.offset,
-            "errorLength": match.errorLength,
-            "category": match.category,
-            "ruleIssueType": match.ruleIssueType,
-            "sentence": match.sentence
+    """Check for language issues using LanguageTool."""
+    try:
+        language_tool = language_tool_python.LanguageTool('en-US')
+        matches = language_tool.check(full_text)
+        issues = []
+        for match in matches:
+            issues.append({
+                "message": match.message,
+                "context": match.context.strip(),
+                "suggestions": match.replacements[:3] if match.replacements else [],
+                "category": match.category,
+                "rule_id": match.ruleId,
+                "offset": match.offset,
+                "length": match.errorLength
+            })
+        print(f"Total language issues found: {len(issues)}")
+        return {
+            "total_issues": len(issues),
+            "issues": issues
         }
-        for match in matches
-    ]
-    
-    return {
-        "issues_count": issues_count,
-        "issues_per_1000": issues_per_1000,
-        "failed": issues_per_1000 > 20,
-        "matches": serializable_matches
-    }
+    except Exception as e:
+        print(f"Error checking language issues: {e}")
+        return {"error": str(e)}
 
 def check_language(full_text: str) -> Dict[str, Any]:
     """Check language quality."""
@@ -259,97 +265,107 @@ def find_text_instances(words, text):
             instances.append(inst)
     return instances
 
-def highlight_issues_in_pdf(file, inconsistent_refs: List[Tuple[int, str, str]], language_matches: List[Dict[str, Any]]) -> bytes:
-    """Highlight inconsistent references and add notes for language issues in a single PDF."""
+def highlight_issues_in_pdf(file, language_matches: List[Dict[str, Any]]) -> bytes:
+    """
+    Highlights language issues in the PDF and returns the annotated PDF as bytes.
+    This function maps LanguageTool matches to specific words in the PDF
+    and highlights those words.
+    """
     try:
-        if isinstance(file, str):
-            doc = fitz.open(file)
-        else:
-            doc = fitz.open(stream=file.read(), filetype="pdf")
-            
-        added_notes = set()
-
-        for page_number, page in enumerate(doc, start=1):
-            words = page.get_text("words")
-            
-            if inconsistent_refs:
-                for ref_num, ref_text, ref_style in inconsistent_refs:
-                    annotation_text = f"Reference {ref_num}: Inconsistent style ({ref_style}). Should be consolidated to {ref_style}."
-                    highlight_text(page, words, ref_text, annotation_text)
-
-            if language_matches:
-                for match in language_matches:
-                    issue_text = match['sentence']
-                    error_message = f"{match['message']}\nSuggested correction: {match['replacements'][0] if match['replacements'] else 'No suggestion'}"
-                    issue_key = (issue_text, error_message)
-                    
-                    if issue_key not in added_notes:
-                        if highlight_text(page, words, issue_text, error_message):
-                            added_notes.add(issue_key)
-
-        annotated_pdf_bytes = doc.write()
+        # Open the PDF
+        doc = fitz.open(stream=file.read(), filetype="pdf") if not isinstance(file, str) else fitz.open(file)
+        print(f"Opened PDF with {len(doc)} pages.")
+
+        # Extract words with positions from each page
+        word_list = []  # List of tuples: (page_number, word, x0, y0, x1, y1)
+        for page_number in range(len(doc)):
+            page = doc[page_number]
+            words = page.get_text("words")  # List of tuples: (x0, y0, x1, y1, "word", block_no, line_no, word_no)
+            for w in words:
+                word_text = w[4]
+                # **Fix:** Insert a space before '[' to ensure "globally [2]" instead of "globally[2]"
+                if '[' in word_text:
+                    word_text = word_text.replace('[', ' [')
+                word_list.append((page_number, word_text, w[0], w[1], w[2], w[3]))
+        print(f"Total words extracted: {len(word_list)}")
+
+        # Concatenate all words to form the full text
+        concatenated_text = " ".join([w[1] for w in word_list])
+        print(f"Concatenated text length: {len(concatenated_text)} characters.")
+
+        # Iterate over each language issue
+        for idx, issue in enumerate(language_matches, start=1):
+            offset = issue["offset"]
+            length = issue["length"]
+            error_text = concatenated_text[offset:offset+length]
+            print(f"\nIssue {idx}: '{error_text}' at offset {offset} with length {length}")
+
+            # Find the words that fall within the error span
+            current_pos = 0
+            target_words = []
+            for word in word_list:
+                word_text = word[1]
+                word_length = len(word_text) + 1  # +1 for the space
+
+                if current_pos + word_length > offset and current_pos < offset + length:
+                    target_words.append(word)
+                current_pos += word_length
+
+            if not target_words:
+                print("No matching words found for this issue.")
+                continue
+
+            # Add highlight annotations to the target words
+            for target in target_words:
+                page_num, word_text, x0, y0, x1, y1 = target
+                page = doc[page_num]
+                # Define a rectangle around the word with some padding
+                rect = fitz.Rect(x0 - 1, y0 - 1, x1 + 1, y1 + 1)
+                # Add a highlight annotation
+                highlight = page.add_highlight_annot(rect)
+                highlight.set_colors(stroke=(1, 1, 0))  # Yellow color
+                highlight.update()
+                print(f"Highlighted '{word_text}' on page {page_num + 1} at position ({x0}, {y0}, {x1}, {y1})")
+
+        # Save annotated PDF to bytes
+        byte_stream = io.BytesIO()
+        doc.save(byte_stream)
+        annotated_pdf_bytes = byte_stream.getvalue()
         doc.close()
-        return annotated_pdf_bytes
 
+        # Save annotated PDF locally for verification
+        with open("annotated_temp.pdf", "wb") as f:
+            f.write(annotated_pdf_bytes)
+        print("Annotated PDF saved as 'annotated_temp.pdf' for manual verification.")
+
+        return annotated_pdf_bytes
     except Exception as e:
-        print(f"An error occurred while annotating the PDF: {str(e)}")
-        traceback.print_exc()
+        print(f"Error in highlighting PDF: {e}")
         return b""
-
 # ------------------------------
 # Main Analysis Function
 # ------------------------------
 
 def analyze_pdf(file) -> Tuple[Dict[str, Any], bytes]:
-    """
-    Analyze the uploaded PDF and return analysis results and annotated PDF bytes.
-    """
+    """Analyzes the PDF for language issues and returns results and annotated PDF."""
     try:
-        pages_text = extract_pdf_text_by_page(file)
+        # Reset file pointer before reading
+        file.seek(0)
         full_text = extract_pdf_text(file)
-        full_text = label_authors(full_text)
-
-        # Perform analyses
-        metadata = check_metadata(full_text)
-        disclosures = check_disclosures(full_text)
-        figures_and_tables = check_figures_and_tables(full_text)
-        figure_order = check_figure_order(full_text)
-        references = check_references(full_text)
-        reference_order = check_reference_order(full_text)
-        reference_style = check_reference_style(full_text)
-        structure = check_structure(full_text)
-        language = check_language(full_text)
-
-        # Compile results
-        results = {
-            "metadata": metadata,
-            "disclosures": disclosures,
-            "figures_and_tables": figures_and_tables,
-            "figure_order": figure_order,
-            "references": references,
-            "reference_order": reference_order,
-            "reference_style": reference_style,
-            "structure": structure,
-            "language": language
-        }
-
-        # Handle annotations
-        inconsistent_refs = reference_style.get("inconsistent_refs", [])
-        language_matches = language.get("language_issues", {}).get("matches", [])
-
-        if inconsistent_refs or language_matches:
-            annotated_pdf_bytes = highlight_issues_in_pdf(file, inconsistent_refs, language_matches)
-        else:
-            annotated_pdf_bytes = None
-
-        return results, annotated_pdf_bytes
-
+        if not full_text:
+            return {"error": "Failed to extract text from PDF."}, None
+
+        language_issues = check_language_issues(full_text)
+        if "error" in language_issues:
+            return language_issues, None
+
+        issues = language_issues.get("issues", [])
+        # Reset file pointer before highlighting
+        file.seek(0)
+        annotated_pdf = highlight_issues_in_pdf(file, issues) if issues else None
+        return language_issues, annotated_pdf
     except Exception as e:
-        error_message = {
-            "error": str(e),
-            "traceback": traceback.format_exc()
-        }
-        return error_message, None
+        return {"error": str(e)}, None
 
 # ------------------------------
 # Gradio Interface