Add download TDocs

app.py

@@ -1,8 +1,10 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import json
 import traceback
-from fastapi import FastAPI, BackgroundTasks
+from fastapi import FastAPI, BackgroundTasks, HTTPException
 from schemas import *
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import FileResponse
+from fastapi.responses import FileResponse, StreamingResponse
 from litellm.router import Router
 from aiolimiter import AsyncLimiter
 import pandas as pd
@@ -238,6 +240,51 @@ def get_change_request_dataframe(req: DataRequest):
     df = filtered_df.fillna("")
     return DataResponse(data=df[["TDoc", "Title", "Type", "TDoc Status", "Agenda item description", "URL"]].to_dict(orient="records"))
 
+@app.post("/download_tdocs")
+def download_tdocs(req: DownloadRequest):
+    documents = req.documents
+
+    def process_document(doc: str):
+        doc_id = doc
+        url = requests.post(
+            'https://organizedprogrammers-3gppdocfinder.hf.space/find',
+            headers={"Content-Type": "application/json"},
+            data=json.dumps({"doc_id": doc_id}),
+            verify=False
+        )
+        print(url.status_code)
+        url = url.json()['url']
+        print(url)
+        try:
+            txt = "\n".join(docx_to_txt(doc_id, url))
+        except Exception as e:
+            txt = f"Document {doc_id} text extraction failed: {e}"
+        return doc_id, txt.encode("utf-8")
+
+    def process_batch(batch):
+        results = {}
+        for doc in batch:
+            try:
+                doc_id, file_bytes = process_document(doc)
+                results[doc_id] = file_bytes
+            except Exception as e:
+                traceback.print_exception(e)
+                results[doc] = b"Erreur"
+        return results
+
+    documents_bytes = process_batch(documents)
+
+    zip_buffer = io.BytesIO()
+    with zipfile.ZipFile(zip_buffer, mode='w', compression=zipfile.ZIP_DEFLATED) as zip_file:
+        for doc_id, txt_data in documents_bytes.items():
+            zip_file.writestr(f'{doc_id}.txt', txt_data)
+
+    zip_buffer.seek(0)
+    return StreamingResponse(
+        zip_buffer,
+        media_type="application/zip"
+    )
+
 @app.post("/generate_requirements", response_model=RequirementsResponse)
 async def gen_reqs(req: RequirementsRequest, background_tasks: BackgroundTasks):
     documents = req.documents

index.html

@@ -52,7 +52,7 @@
             </div>
         </div>
 
-        <div class="flex justify-center mt-12 min-h-screen hidden" id="queryReqForm">
+        <div class="flex justify-center mt-12 min-h-[10vh] hidden" id="queryReqForm">
             <div class="w-full max-w-md">
                 <div class="grid grid-cols-1 gap-4">
                     <textarea placeholder="Enter your problem description here ..." class="w-full mx-auto px-4 py-2 border rounded" id="problemDescription"></textarea>
@@ -69,7 +69,7 @@
         </center>
 
         <!-- Tableau des données -->
-        <div class="max-h-[65vh] overflow-y-auto" id="dataFrameDiv">
+        <div class="max-h-[65vh] overflow-y-auto mt-12" id="dataFrameDiv">
             <table class="table table-zebra w-full" id="dataFrame">
                 <thead class="sticky top-0 bg-base-200 z-10">
                     <tr class="bg-base-200">
@@ -90,7 +90,8 @@
             <div id="buttons">
                 <p id="reqStatus" class="mt-6 hidden">Requirements extracted</p>
                 <div class="grid grid-cols-1 md:grid-cols-3 gap-4 mb-6">
-                    <button class="btn mt-6" id="getReqs">Get Requirements</button>
+                    <button class="btn mt-6 hidden" id="getReqs">Extract Requirements</button>
+                    <button class="btn mt-6 hidden" id="downloadZip">Download TDocs</button>
                     <button class="btn mt-6 hidden" id="searchReq">Query requirements</button>
                     <button class="btn mt-6 hidden" id="categorizeReq">Categorize requirements</button>
                 </div>
@@ -101,6 +102,34 @@
     <script>
         let requirements;
 
+        function downloadTDocs(){
+            const data = tableToGenBody({"TDoc": "doc"});
+            const dataSet = [...new Set(data.map(item => item.doc))];
+            console.log(dataSet);
+            let body = {"documents": dataSet, "meeting": document.getElementById('meetingSelect').value};
+            if (document.getElementById('agendaItem').value != "" | document.getElementById('agendaItem').value != "Tous"){
+                body['agenda_item'] = document.getElementById('agendaItem').value;
+            }
+            fetch('/download_tdocs', {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify(body)})
+            .then(resp => resp.blob())
+            .then(blob => {
+                const url = window.URL.createObjectURL(blob);
+                const a = document.createElement("a");
+                a.href = url;
+                let dl_name = `${document.getElementById('meetingSelect').value}`;
+                if (document.getElementById('agendaItem').value != "" | document.getElementById('agendaItem').value != "Tous"){dl_name = dl_name + `_${document.getElementById('agendaItem').value}`};
+                if (document.getElementById('docStatus').value != "" | document.getElementById('docStatus').value != "Tous"){dl_name = dl_name + `_${document.getElementById('docStatus').value}`};
+                if (document.getElementById('docType').value != "" | document.getElementById('docType').value != "Tous"){dl_name =  `${document.getElementById('docType').value}_${dl_name}`};
+                if (document.getElementById('queryReqForm').classList.contains('hidden')){dl_name = `requirements_${dl_name}_${url.split('/').pop()}`}
+                dl_name = dl_name + ".zip";
+                a.download = dl_name;
+                document.body.appendChild(a);
+                a.click();
+                a.remove();
+                window.URL.revokeObjectURL(url);  // libération mémoire
+            })
+        }
+
         function getDataFrame(){
             document.getElementById("loadingBar").classList.remove("hidden");
             const wg = document.getElementById('workingGroupSelect').value;
@@ -126,8 +155,10 @@
             fetch("/get_dataframe", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({"working_group": wg, "meeting": meeting})})
             .then(resp => resp.json())
             .then(data => {
-                document.getElementById("filters").classList.remove("hidden")
+                document.getElementById("filters").classList.remove("hidden");
                 document.getElementById("loadingBar").classList.add("hidden");
+                document.getElementById("downloadZip").classList.remove("hidden");
+                document.getElementById("getReqs").classList.remove("hidden");
                 const dataframeBody = dataFrame.querySelector("tbody");
                 dataframeBody.innerHTML = "";
                 const setType = new Set();
@@ -216,7 +247,7 @@
         }
 
         function generateRequirements(){
-            const bodyreq = tableToGenBody();
+            const bodyreq = tableToGenBody({"TDoc": "document", "URL": "url"});
             document.getElementById("progressText").classList.remove('hidden');
             document.getElementById("progressText").innerHTML = "Generating requirements, please wait, it may take a while ...";
             document.getElementById("loadingBar").classList.remove("hidden");
@@ -224,19 +255,18 @@
             fetch("/generate_requirements", {method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify({"documents": bodyreq})})
             .then(resp => resp.json())
             .then(data => {
-                requirements = [];
-                data.requirements.forEach(obj => {
-                    obj.requirements.forEach(req => {
-                        requirements.push({"document": obj.document, "context": obj.context, "requirement": req})
-                    })
-                })
-
                 document.getElementById("loadingBar").classList.add("hidden");
                 document.getElementById("progressText").classList.add("hidden");
                 document.getElementById("reqStatus").classList.remove("hidden");
                 document.getElementById("getReqs").classList.add("hidden");
                 document.getElementById("searchReq").classList.remove("hidden");
                 document.getElementById("categorizeReq").classList.remove("hidden");
+                requirements = [];
+                data.requirements.forEach(obj => {
+                    obj.requirements.forEach(req => {
+                        requirements.push({"document": obj.document, "context": obj.context, "requirement": req})
+                    })
+                })
             })
         }
 
@@ -247,6 +277,12 @@
                 const dataFrame = document.getElementById("dataFrameDiv");
                 const dataFrameHead = dataFrame.querySelector("thead");
                 const dataFrameBody = dataFrame.querySelector("tbody");
+                document.getElementById("buttons").classList.remove("hidden");
+                document.getElementById("searchReq").classList.add("hidden");
+                document.getElementById("categorizeReq").classList.add("hidden");
+                document.getElementById("getReqs").classList.add("hidden");
+                document.getElementById("reqStatus").classList.add("hidden");
+                document.getElementById("downloadZip").classList.remove("hidden");
 
                 dataFrame.classList.remove("hidden");
 
@@ -270,9 +306,9 @@
             })
         }
 
-        function tableToGenBody() {
+        function tableToGenBody(columnsMap) {
             // columnsMap : { "NomHeaderDansTable": "nom_voulu", ... }
-            let columnsMap = {"TDoc": "document", "URL": "url"};
+            const dataFrame = document.getElementById("dataFrame");
             const headers = Array.from(dataFrame.querySelectorAll('thead th')).map(th => th.innerText.trim());
             
             // Indices des colonnes à extraire
@@ -302,6 +338,7 @@
         document.getElementById("workingGroupSelect").addEventListener('change', getMeetings)
         document.getElementById('getTDocs').addEventListener('click', getDataFrame)
         document.getElementById("getReqs").addEventListener("click", generateRequirements);
+        document.getElementById("downloadZip").addEventListener('click', downloadTDocs)
         document.getElementById("queryReq").addEventListener("click", queryRequirements)
         document.getElementById('searchReq').addEventListener('click', ()=>{
             document.getElementById('dataFrameForm').classList.add('hidden');

schemas.py

@@ -16,10 +16,11 @@ class DataResponse(BaseModel):
 
 # --------------------------------------
 
+class DocInfo(BaseModel):
+    document: str
+    url: str
+
 class RequirementsRequest(BaseModel):
-    class DocInfo(BaseModel):
-        document: str
-        url: str
     documents: List[DocInfo]
 
 class DocRequirements(BaseModel):
@@ -41,4 +42,9 @@ class ReqSearchRequest(BaseModel):
     requirements: List[SingleRequirement]
 
 class ReqSearchResponse(BaseModel):
-    requirements: List[SingleRequirement]
+    requirements: List[SingleRequirement]
+
+# --------------------------------------
+
+class DownloadRequest(BaseModel):
+    documents: List[str]