Update app.py

app.py

@@ -1,15 +1,16 @@
-import os
-import time
+import nest_asyncio
+import aiofiles
+import uvicorn
 import zipfile
+from fastapi import FastAPI, File, Query, UploadFile, HTTPException
+from fastapi.responses import FileResponse, JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from enum import Enum
 from pathlib import Path
 from typing import List
-import shutil
-import json
-import yaml
-import datetime
-import easyocr
-import pandas as pd
-import streamlit as st
+from PyPDF2 import PdfReader
+from easyocr import Reader
+
 from docling.document_converter import DocumentConverter, PdfFormatOption
 from docling.datamodel.base_models import InputFormat
 from docling.datamodel.document import ConversionStatus
@@ -25,70 +26,63 @@ from docling.datamodel.pipeline_options import (
     RapidOcrOptions,
     OcrMacOptions,
 )
-from easyocr.utils import get_language_list
-
-# Répertoires de sortie
-OUTPUT_DIR = Path("output")
-FIGURES_DIR = OUTPUT_DIR / "figures"
-TABLES_DIR = OUTPUT_DIR / "tables"
-
-for directory in [OUTPUT_DIR, FIGURES_DIR, TABLES_DIR]:
-    directory.mkdir(exist_ok=True)
-
-# Récupération des langues supportées par EasyOCR
-supported_languages = get_language_list()
 
-# Configuration Streamlit
-st.set_page_config(page_title="Docling API UI", layout="wide")
+# Activation de nest_asyncio pour éviter les erreurs de boucle d'événement dans Colab
+nest_asyncio.apply()
 
-st.title("Docling Document Conversion API")
-st.sidebar.header("Configuration")
-
-# Formulaire de configuration
-use_ocr = st.sidebar.checkbox("Activer l'OCR", value=True)
-export_figures = st.sidebar.checkbox("Exporter les figures", value=True)
-export_tables = st.sidebar.checkbox("Exporter les tableaux", value=True)
-
-accelerator = st.sidebar.selectbox(
-    "Accélérateur",
-    ["auto", "cpu", "cuda", "mps"],
-    index=0,
-    format_func=lambda x: x.upper(),
-)
-
-ocr_engine = st.sidebar.selectbox(
-    "Moteur OCR",
-    ["easyocr", "tesseract_cli", "tesserocr", "rapidocr", "ocrmac"],
-    index=0,
-)
-
-ocr_languages = st.sidebar.multiselect(
-    "Langues OCR",
-    options=supported_languages,
-    default=["en"],
-)
+# Définition de l'application FastAPI
+app = FastAPI()
 
-table_mode = st.sidebar.selectbox(
-    "Mode Table",
-    ["accurate", "fast"],
-    index=0,
-    format_func=lambda x: x.capitalize(),
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
 )
 
-export_formats = st.sidebar.multiselect(
-    "Formats d'export",
-    ["json", "yaml", "md", "multimodal"],
-    default=["md"],
-)
+# Répertoires de sortie
+OUTPUT_DIR = Path("output")
+OUTPUT_DIR.mkdir(exist_ok=True)
 
-uploaded_files = st.file_uploader(
-    "Uploader vos fichiers (PDF, DOCX, PPTX, HTML, IMAGES)",
-    type=["pdf", "docx", "pptx", "html", "png", "jpg", "jpeg"],
-    accept_multiple_files=True,
-)
+FIGURES_DIR = OUTPUT_DIR / "figures"
+FIGURES_DIR.mkdir(exist_ok=True)
 
-# Fonction pour créer le convertisseur
-def create_document_converter():
+TABLES_DIR = OUTPUT_DIR / "tables"
+TABLES_DIR.mkdir(exist_ok=True)
+
+class ExportFormat(str, Enum):
+    json = "json"
+    yaml = "yaml"
+    md = "md"
+    multimodal = "multimodal"
+
+class OcrEngine(str, Enum):
+    easyocr = "easyocr"
+    tesseract_cli = "tesseract_cli"
+    tesserocr = "tesserocr"
+    rapidocr = "rapidocr"
+    ocrmac = "ocrmac"
+
+# Vérification de validité des fichiers PDF
+def is_valid_pdf(file_path):
+    try:
+        PdfReader(file_path)
+        return True
+    except Exception as e:
+        print(f"Fichier non valide ou corrompu : {file_path} - Erreur : {e}")
+        return False
+
+# Fonction pour configurer le convertisseur de documents
+def create_document_converter(
+    use_ocr: bool,
+    export_figures: bool,
+    export_tables: bool,
+    accelerator: str,
+    ocr_engine: OcrEngine,
+    table_mode: str,
+    ocr_languages: List[str]
+) -> DocumentConverter:
     accelerator_options = AcceleratorOptions(
         num_threads=8,
         device=AcceleratorDevice[accelerator.upper()],
@@ -99,20 +93,16 @@ def create_document_converter():
         do_cell_matching=True,
     )
 
-    # Validation des langues
-    if not all(lang in supported_languages for lang in ocr_languages):
-        st.error(f"Certaines langues sélectionnées ne sont pas prises en charge : {ocr_languages}")
-        st.stop()
-
-    if ocr_engine == "easyocr":
+    # OCR avec EasyOCR ou autres moteurs
+    if ocr_engine == OcrEngine.easyocr:
         ocr_options = EasyOcrOptions(lang=ocr_languages)
-    elif ocr_engine == "tesseract_cli":
+    elif ocr_engine == OcrEngine.tesseract_cli:
         ocr_options = TesseractCliOcrOptions(lang=ocr_languages)
-    elif ocr_engine == "tesserocr":
+    elif ocr_engine == OcrEngine.tesserocr:
         ocr_options = TesseractOcrOptions(lang=ocr_languages)
-    elif ocr_engine == "rapidocr":
+    elif ocr_engine == OcrEngine.rapidocr:
         ocr_options = RapidOcrOptions(lang=ocr_languages)
-    elif ocr_engine == "ocrmac":
+    elif ocr_engine == OcrEngine.ocrmac:
         ocr_options = OcrMacOptions(lang=ocr_languages)
     else:
         raise ValueError(f"Moteur OCR non pris en charge : {ocr_engine}")
@@ -126,7 +116,6 @@ def create_document_converter():
         table_structure_options=table_structure_options,
         ocr_options=ocr_options,
     )
-
     return DocumentConverter(
         allowed_formats=[
             InputFormat.PDF,
@@ -140,76 +129,79 @@ def create_document_converter():
         },
     )
 
-# Traitement des fichiers
-if st.button("Lancer la conversion") and uploaded_files:
-    st.info("Conversion en cours, veuillez patienter...")
-
-    converter = create_document_converter()
-
-    # Préparer les fichiers pour le traitement
+# Endpoint pour traiter les fichiers
+@app.post("/process_files/")
+async def process_files(
+    files: List[UploadFile] = File(...),
+    use_ocr: bool = Query(False),
+    export_figures: bool = Query(True),
+    export_tables: bool = Query(True),
+    export_formats: List[ExportFormat] = Query(default=[ExportFormat.md]),
+    accelerator: str = Query("cpu"),
+    ocr_engine: OcrEngine = Query(OcrEngine.easyocr),
+    table_mode: str = Query(TableFormerMode.ACCURATE),
+    ocr_languages: List[str] = Query(default=["eng"]),
+):
     input_paths = []
-    for uploaded_file in uploaded_files:
-        file_path = OUTPUT_DIR / uploaded_file.name
-        with open(file_path, "wb") as f:
-            f.write(uploaded_file.read())
+    for f in files:
+        file_path = OUTPUT_DIR / f.filename
+        async with aiofiles.open(file_path, 'wb') as out_file:
+            content = await f.read()
+            await out_file.write(content)
+        print(f"Fichier reçu : {file_path} (taille : {file_path.stat().st_size} octets)")
+        if not is_valid_pdf(file_path):
+            return JSONResponse(
+                content={"error": f"Le fichier {file_path.name} n'est pas un PDF valide."}, status_code=400
+            )
         input_paths.append(file_path)
 
-    # Conversion des fichiers
+    converter = create_document_converter(
+        use_ocr,
+        export_figures,
+        export_tables,
+        accelerator,
+        ocr_engine,
+        table_mode,
+        ocr_languages,
+    )
     conv_results = list(converter.convert_all(input_paths, raises_on_error=False))
+
     success_count, failure_count = 0, 0
-    exported_files = {"figures": [], "tables": [], "exports": []}
+    generated_files = []
 
     for conv_res in conv_results:
         if conv_res.status == ConversionStatus.SUCCESS:
+            print(f"Conversion réussie pour : {conv_res.input.file}")
             success_count += 1
-            doc_filename = conv_res.input.file.stem
-
-            # Export des tableaux
-            for table_ix, table in enumerate(conv_res.document.tables):
-                csv_file = TABLES_DIR / f"{doc_filename}-table-{table_ix+1}.csv"
-                table.export_to_dataframe().to_csv(csv_file, index=False)
-                exported_files["tables"].append(str(csv_file))
-
-            # Export des formats demandés
-            for fmt in export_formats:
-                output_file = OUTPUT_DIR / f"{doc_filename}.{fmt}"
-                if fmt == "json":
-                    with open(output_file, "w", encoding="utf-8") as jf:
-                        json.dump(conv_res.document.export_to_dict(), jf, ensure_ascii=False, indent=2)
-                elif fmt == "yaml":
-                    with open(output_file, "w", encoding="utf-8") as yf:
-                        yaml.dump(conv_res.document.export_to_dict(), yf, allow_unicode=True)
-                elif fmt == "md":
-                    with open(output_file, "w", encoding="utf-8") as mf:
-                        mf.write(conv_res.document.export_to_markdown())
-                exported_files["exports"].append(str(output_file))
-
+            for export_format in export_formats:
+                if export_format == ExportFormat.md:
+                    output_file = OUTPUT_DIR / f"{conv_res.input.file.stem}.md"
+                    if conv_res.document.pages:
+                        with open(output_file, "w") as f:
+                            f.write("## Exemple de contenu Markdown généré\n")
+                        generated_files.append(output_file)
+                    else:
+                        print(f"Aucune donnée trouvée dans le document converti : {conv_res.input.file}")
         else:
+            print(f"Échec de la conversion pour : {conv_res.input.file} - Statut : {conv_res.status}")
             failure_count += 1
 
     # Création du fichier ZIP
-    zip_path = OUTPUT_DIR / "exports.zip"
-    with zipfile.ZipFile(zip_path, "w") as zipf:
-        for category, files in exported_files.items():
-            for file in files:
-                zipf.write(file, arcname=Path(file).name)
-
-    # Affichage des résultats
-    st.success(f"Conversion terminée : {success_count} fichiers convertis avec succès.")
-    if failure_count > 0:
-        st.warning(f"{failure_count} fichiers n'ont pas pu être convertis.")
-    
-    st.download_button(
-        label="Télécharger tous les résultats (ZIP)",
-        data=zip_path.read_bytes(),
-        file_name="exports.zip",
-        mime="application/zip",
-    )
+    zip_filename = OUTPUT_DIR / "exported_files.zip"
+    with zipfile.ZipFile(zip_filename, "w") as zipf:
+        for file in generated_files:
+            zipf.write(file, file.name)
+
+    return {"success_count": success_count, "failure_count": failure_count, "zip_path": str(zip_filename)}
+
+@app.get("/download/{filename}")
+def download_file(filename: str):
+    file_path = OUTPUT_DIR / filename
+    if file_path.exists():
+        return FileResponse(path=file_path, filename=filename)
+    else:
+        raise HTTPException(status_code=404, detail="Fichier non trouvé.")
 
-    # Affichage des métriques
-    st.subheader("Métriques de conversion")
-    st.write(f"Fichiers traités : {len(uploaded_files)}")
-    st.write(f"Conversions réussies : {success_count}")
-    st.write(f"Échecs : {failure_count}")
-else:
-    st.info("Ajoutez des fichiers pour commencer la conversion.")
+# Démarrer le serveur
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)