Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -13,17 +13,18 @@ from flask import Flask, request, render_template, Response, stream_with_context
|
|
13 |
from werkzeug.utils import secure_filename
|
14 |
|
15 |
# Ensure gevent is imported and monkey patched if needed for other libraries
|
16 |
-
# that might not be gevent-friendly. For built-in libs and requests (with Gunicorn gevent worker),
|
17 |
-
# this is often handled by Gunicorn.
|
18 |
# from gevent import monkey
|
19 |
# monkey.patch_all() # Apply this early if you suspect issues with other libs
|
20 |
|
21 |
-
import requests
|
|
|
|
|
22 |
import pdfplumber
|
23 |
from pdf2image import convert_from_path, convert_from_bytes
|
24 |
import pytesseract
|
25 |
from PIL import Image
|
26 |
-
from huggingface_hub import HfApi, create_repo
|
|
|
27 |
|
28 |
# --- Flask App Initialization ---
|
29 |
app = Flask(__name__)
|
@@ -42,13 +43,11 @@ hf_api = HfApi()
|
|
42 |
# --- Helper to yield messages for streaming ---
|
43 |
def yield_message(type, data):
|
44 |
"""Helper to format messages as JSON strings for streaming."""
|
45 |
-
# Add a newline so client can easily split messages
|
46 |
return json.dumps({"type": type, **data}) + "\n"
|
47 |
|
48 |
# --- PDF Processing Helper Functions (Adapted for Streaming) ---
|
49 |
|
50 |
def check_poppler():
|
51 |
-
# (Same as before)
|
52 |
try:
|
53 |
result = subprocess.run(["pdftoppm", "-v"], capture_output=True, text=True, check=False)
|
54 |
version_info_log = result.stderr.strip() if result.stderr else result.stdout.strip()
|
@@ -65,37 +64,44 @@ def check_poppler():
|
|
65 |
return False
|
66 |
|
67 |
def ensure_hf_dataset():
|
68 |
-
# (Same as before, but logs info useful for streaming if an error occurs)
|
69 |
if not HF_TOKEN:
|
70 |
msg = "HF_TOKEN is not set. Cannot ensure Hugging Face dataset. Image uploads will fail."
|
71 |
logger.warning(msg)
|
72 |
return "Error: " + msg
|
73 |
try:
|
|
|
|
|
|
|
74 |
repo_id_obj = create_repo(repo_id=HF_DATASET_REPO_NAME, token=HF_TOKEN, repo_type="dataset", exist_ok=True)
|
75 |
logger.info(f"Dataset repo ensured: {repo_id_obj.repo_id}")
|
76 |
return repo_id_obj.repo_id
|
77 |
-
except
|
78 |
-
if e.response.status_code == 409:
|
79 |
-
logger.info(f"Dataset repo '{HF_DATASET_REPO_NAME}' already exists.")
|
80 |
# Attempt to construct the full repo_id (namespace/repo_name)
|
81 |
try:
|
82 |
-
user_info = hf_api.whoami(token=HF_TOKEN)
|
83 |
namespace = user_info.get('name') if user_info else None
|
84 |
if namespace:
|
85 |
return f"{namespace}/{HF_DATASET_REPO_NAME}"
|
|
|
|
|
|
|
86 |
except Exception as whoami_e:
|
87 |
-
logger.error(f"Could not determine namespace for existing repo via whoami: {whoami_e}")
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
|
|
|
|
|
|
|
|
93 |
return f"Error: Failed to access or create dataset '{HF_DATASET_REPO_NAME}': {str(e)}"
|
94 |
|
95 |
|
96 |
def upload_image_to_hf_stream(image_pil, filename_base, page_num_for_log=""):
|
97 |
-
# (Adapted to potentially yield status during this sub-process if it were longer)
|
98 |
-
# For now, it's synchronous but part of the larger stream.
|
99 |
repo_id_or_error = ensure_hf_dataset()
|
100 |
if isinstance(repo_id_or_error, str) and repo_id_or_error.startswith("Error"):
|
101 |
return repo_id_or_error
|
@@ -117,7 +123,7 @@ def upload_image_to_hf_stream(image_pil, filename_base, page_num_for_log=""):
|
|
117 |
)
|
118 |
logger.info(f"Successfully uploaded image: {file_url}")
|
119 |
return file_url
|
120 |
-
except Exception as e:
|
121 |
logger.error(f"Image upload error for {filename_base}{page_num_for_log}: {str(e)}", exc_info=True)
|
122 |
return f"Error uploading image {filename_base}{page_num_for_log}: {str(e)}"
|
123 |
finally:
|
@@ -127,11 +133,7 @@ def upload_image_to_hf_stream(image_pil, filename_base, page_num_for_log=""):
|
|
127 |
|
128 |
|
129 |
def format_page_text_to_markdown_chunk(page_text_content):
|
130 |
-
"""Formats a single page's text content into a markdown chunk.
|
131 |
-
More complex formatting logic can be applied here page by page.
|
132 |
-
"""
|
133 |
chunk_md = ""
|
134 |
-
# Normalize newlines: multiple consecutive newlines become a single blank line (two \n chars)
|
135 |
page_text_content = re.sub(r'\n\s*\n+', '\n\n', page_text_content.strip())
|
136 |
lines = page_text_content.split('\n')
|
137 |
is_in_list = False
|
@@ -160,16 +162,10 @@ def format_page_text_to_markdown_chunk(page_text_content):
|
|
160 |
# --- Main PDF Processing Logic (Generator Function for Streaming) ---
|
161 |
|
162 |
def generate_pdf_conversion_stream(pdf_input_source_path_or_url):
|
163 |
-
"""
|
164 |
-
Processes the PDF incrementally and yields status messages and markdown content.
|
165 |
-
`pdf_input_source_path_or_url` is a local file path or a URL string.
|
166 |
-
"""
|
167 |
try:
|
168 |
-
# Initial Markdown Title
|
169 |
yield yield_message("markdown_replace", {"content": "# Extracted PDF Content\n\n"})
|
170 |
-
time.sleep(0.01)
|
171 |
|
172 |
-
# 1. Text and Table Extraction (Page by Page)
|
173 |
yield yield_message("status", {"message": "Opening PDF for text extraction..."})
|
174 |
time.sleep(0.01)
|
175 |
|
@@ -177,22 +173,26 @@ def generate_pdf_conversion_stream(pdf_input_source_path_or_url):
|
|
177 |
pdf_input_source_path_or_url.startswith(('http://', 'https://'))
|
178 |
|
179 |
pdf_handle_for_text = None
|
180 |
-
pdf_bytes_for_images = None
|
181 |
|
182 |
if source_is_url:
|
183 |
try:
|
184 |
-
response = requests.get(pdf_input_source_path_or_url, stream=
|
185 |
response.raise_for_status()
|
186 |
-
pdf_bytes_for_images = response.content
|
187 |
-
pdf_handle_for_text = io.BytesIO(pdf_bytes_for_images)
|
188 |
yield yield_message("status", {"message": f"PDF downloaded from URL ({len(pdf_bytes_for_images)/1024:.2f} KB)."})
|
189 |
time.sleep(0.01)
|
190 |
-
except
|
191 |
-
logger.error(f"URL fetch error for PDF processing: {str(e)}", exc_info=True)
|
192 |
-
yield yield_message("error", {"message": f"Error fetching PDF from URL: {str(e)}"})
|
193 |
-
return
|
194 |
-
|
195 |
-
|
|
|
|
|
|
|
|
|
196 |
|
197 |
total_text_pages = 0
|
198 |
try:
|
@@ -203,7 +203,7 @@ def generate_pdf_conversion_stream(pdf_input_source_path_or_url):
|
|
203 |
|
204 |
for i, page in enumerate(pdf.pages):
|
205 |
yield yield_message("status", {"message": f"Extracting text from page {i+1}/{total_text_pages}..."})
|
206 |
-
time.sleep(0.01)
|
207 |
|
208 |
page_text = page.extract_text(layout=True, x_density=1, y_density=1) or ""
|
209 |
|
@@ -211,10 +211,11 @@ def generate_pdf_conversion_stream(pdf_input_source_path_or_url):
|
|
211 |
tables = page.extract_tables()
|
212 |
if tables:
|
213 |
for table_idx, table_data in enumerate(tables):
|
214 |
-
if table_data:
|
215 |
yield yield_message("status", {"message": f" Processing table {table_idx+1} on page {i+1}..."})
|
216 |
-
|
217 |
-
|
|
|
218 |
body = [" | ".join(str(cell) if cell is not None else "" for cell in row) for row in table_data[1:]]
|
219 |
table_md_lines = header + separator + body
|
220 |
page_tables_md += f"**Table (Page {i+1}):**\n" + "\n".join(table_md_lines) + "\n\n"
|
@@ -224,11 +225,11 @@ def generate_pdf_conversion_stream(pdf_input_source_path_or_url):
|
|
224 |
yield yield_message("markdown_chunk", {"content": formatted_page_text_md})
|
225 |
if page_tables_md:
|
226 |
yield yield_message("markdown_chunk", {"content": page_tables_md})
|
227 |
-
time.sleep(0.01)
|
228 |
except Exception as e:
|
229 |
logger.error(f"Error during PDF text/table extraction: {str(e)}", exc_info=True)
|
230 |
yield yield_message("error", {"message": f"Error during text extraction: {str(e)}"})
|
231 |
-
#
|
232 |
|
233 |
# 2. Image Extraction and OCR
|
234 |
if not check_poppler():
|
@@ -242,52 +243,95 @@ def generate_pdf_conversion_stream(pdf_input_source_path_or_url):
|
|
242 |
time.sleep(0.01)
|
243 |
extracted_pil_images = []
|
244 |
try:
|
|
|
245 |
if source_is_url and pdf_bytes_for_images:
|
246 |
-
|
247 |
-
|
248 |
-
elif not source_is_url:
|
249 |
-
|
|
|
250 |
|
251 |
-
|
252 |
-
|
253 |
-
|
254 |
-
# TODO: Implement more granular image extraction if pdf2image supports it,
|
255 |
-
# or if you integrate a library that can extract embedded images directly.
|
256 |
-
# For now, convert_from_path/bytes often gives full pages as images.
|
257 |
-
|
258 |
-
for i, img_pil in enumerate(extracted_pil_images):
|
259 |
-
page_num_for_log = f"page_{i+1}" # Assuming one image per page from convert_from_path
|
260 |
-
yield yield_message("status", {"message": f"Processing image {i+1}/{len(extracted_pil_images)} (OCR & Upload)..."})
|
261 |
-
time.sleep(0.01)
|
262 |
-
|
263 |
-
ocr_text = ""
|
264 |
try:
|
265 |
-
|
266 |
-
if
|
267 |
-
|
268 |
-
|
269 |
-
|
270 |
-
|
271 |
-
|
272 |
-
|
273 |
-
|
274 |
-
|
275 |
-
|
276 |
-
|
277 |
-
|
278 |
-
|
279 |
-
|
280 |
-
|
281 |
-
|
282 |
-
|
283 |
-
|
284 |
-
|
285 |
-
|
286 |
-
|
287 |
-
|
288 |
-
|
289 |
-
|
290 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
291 |
logger.error(f"Error during image extraction/processing: {str(e)}", exc_info=True)
|
292 |
yield yield_message("error", {"message": f"Error during image extraction: {str(e)}"})
|
293 |
|
@@ -309,12 +353,16 @@ def process_pdf_stream():
|
|
309 |
pdf_file = request.files.get('pdf_file')
|
310 |
pdf_url = request.form.get('pdf_url', '').strip()
|
311 |
|
312 |
-
|
313 |
-
|
|
|
|
|
|
|
|
|
314 |
|
315 |
def stream_processor():
|
316 |
-
nonlocal
|
317 |
-
|
318 |
|
319 |
try:
|
320 |
if pdf_file and pdf_file.filename:
|
@@ -323,13 +371,13 @@ def process_pdf_stream():
|
|
323 |
return
|
324 |
|
325 |
filename = secure_filename(pdf_file.filename)
|
326 |
-
# Save to a temporary file (ensure UPLOAD_FOLDER is writable by app user)
|
327 |
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
|
328 |
-
fd,
|
329 |
os.close(fd)
|
330 |
-
pdf_file.save(
|
331 |
-
|
332 |
-
|
|
|
333 |
yield yield_message("status", {"message": f"Processing uploaded PDF: {filename}"})
|
334 |
time.sleep(0.01)
|
335 |
|
@@ -338,7 +386,6 @@ def process_pdf_stream():
|
|
338 |
if not (unquoted_url.startswith('http://') or unquoted_url.startswith('https://')):
|
339 |
yield yield_message("error", {"message": "Invalid URL scheme. Must be http or https."})
|
340 |
return
|
341 |
-
# Consider a light check for .pdf extension, but content-type is more reliable
|
342 |
|
343 |
pdf_input_source_for_generator = unquoted_url
|
344 |
yield yield_message("status", {"message": f"Preparing to process PDF from URL: {unquoted_url}"})
|
@@ -347,33 +394,32 @@ def process_pdf_stream():
|
|
347 |
yield yield_message("error", {"message": "No PDF file uploaded and no PDF URL provided."})
|
348 |
return
|
349 |
|
350 |
-
# Yield from the main generator
|
351 |
for message_part in generate_pdf_conversion_stream(pdf_input_source_for_generator):
|
352 |
yield message_part
|
353 |
-
# time.sleep(0.01) # Allow gevent to switch context, important for streaming
|
354 |
|
355 |
except Exception as e:
|
356 |
logger.error(f"Error setting up stream or in initial validation: {str(e)}", exc_info=True)
|
357 |
yield yield_message("error", {"message": f"Setup error: {str(e)}"})
|
|
|
|
|
|
|
358 |
finally:
|
359 |
-
if
|
360 |
try:
|
361 |
-
os.remove(
|
362 |
-
logger.info(f"Cleaned up temporary PDF: {
|
363 |
-
|
|
|
|
|
364 |
except OSError as ose:
|
365 |
-
logger.error(f"Error removing temporary PDF {
|
366 |
-
yield yield_message("error", {"message": f"Could not clean temp file: {ose}"})
|
367 |
|
368 |
-
# Using stream_with_context for proper handling of request context within the generator
|
369 |
return Response(stream_with_context(stream_processor()), mimetype='application/x-ndjson')
|
370 |
|
371 |
|
372 |
# --- Main Execution ---
|
373 |
if __name__ == '__main__':
|
374 |
-
if not check_poppler():
|
375 |
logger.warning("Poppler utilities might not be installed correctly. PDF processing might fail.")
|
376 |
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
|
377 |
-
# For local dev, Flask's built-in server is fine. Gunicorn handles production.
|
378 |
-
# The 'threaded=True' or using gevent server locally can also help test streaming.
|
379 |
app.run(host='0.0.0.0', port=int(os.getenv("PORT", 7860)), debug=True, threaded=True)
|
|
|
13 |
from werkzeug.utils import secure_filename
|
14 |
|
15 |
# Ensure gevent is imported and monkey patched if needed for other libraries
|
|
|
|
|
16 |
# from gevent import monkey
|
17 |
# monkey.patch_all() # Apply this early if you suspect issues with other libs
|
18 |
|
19 |
+
import requests # For requests.exceptions.HTTPError
|
20 |
+
from requests.exceptions import HTTPError as RequestsHTTPError # Specific import for clarity
|
21 |
+
|
22 |
import pdfplumber
|
23 |
from pdf2image import convert_from_path, convert_from_bytes
|
24 |
import pytesseract
|
25 |
from PIL import Image
|
26 |
+
from huggingface_hub import HfApi, create_repo
|
27 |
+
# from huggingface_hub.utils import HfHubHTTPError # This was the incorrect one
|
28 |
|
29 |
# --- Flask App Initialization ---
|
30 |
app = Flask(__name__)
|
|
|
43 |
# --- Helper to yield messages for streaming ---
|
44 |
def yield_message(type, data):
|
45 |
"""Helper to format messages as JSON strings for streaming."""
|
|
|
46 |
return json.dumps({"type": type, **data}) + "\n"
|
47 |
|
48 |
# --- PDF Processing Helper Functions (Adapted for Streaming) ---
|
49 |
|
50 |
def check_poppler():
|
|
|
51 |
try:
|
52 |
result = subprocess.run(["pdftoppm", "-v"], capture_output=True, text=True, check=False)
|
53 |
version_info_log = result.stderr.strip() if result.stderr else result.stdout.strip()
|
|
|
64 |
return False
|
65 |
|
66 |
def ensure_hf_dataset():
|
|
|
67 |
if not HF_TOKEN:
|
68 |
msg = "HF_TOKEN is not set. Cannot ensure Hugging Face dataset. Image uploads will fail."
|
69 |
logger.warning(msg)
|
70 |
return "Error: " + msg
|
71 |
try:
|
72 |
+
# create_repo can raise huggingface_hub.utils.RepositoryNotFoundError,
|
73 |
+
# huggingface_hub.utils.HfHubHTTPError (which inherits from requests.HTTPError for some cases),
|
74 |
+
# or other requests.exceptions
|
75 |
repo_id_obj = create_repo(repo_id=HF_DATASET_REPO_NAME, token=HF_TOKEN, repo_type="dataset", exist_ok=True)
|
76 |
logger.info(f"Dataset repo ensured: {repo_id_obj.repo_id}")
|
77 |
return repo_id_obj.repo_id
|
78 |
+
except RequestsHTTPError as e: # Catch HTTP errors from requests library directly
|
79 |
+
if e.response is not None and e.response.status_code == 409: # Conflict, repo already exists
|
80 |
+
logger.info(f"Dataset repo '{HF_DATASET_REPO_NAME}' already exists (HTTP 409).")
|
81 |
# Attempt to construct the full repo_id (namespace/repo_name)
|
82 |
try:
|
83 |
+
user_info = hf_api.whoami(token=HF_TOKEN) # This call could also fail
|
84 |
namespace = user_info.get('name') if user_info else None
|
85 |
if namespace:
|
86 |
return f"{namespace}/{HF_DATASET_REPO_NAME}"
|
87 |
+
else: # Fallback if namespace cannot be determined
|
88 |
+
logger.warning(f"Could not determine namespace for existing repo '{HF_DATASET_REPO_NAME}'. Using generic ID.")
|
89 |
+
return HF_DATASET_REPO_NAME # Or f"{YOUR_DEFAULT_USERNAME_IF_KNOWN}/{HF_DATASET_REPO_NAME}"
|
90 |
except Exception as whoami_e:
|
91 |
+
logger.error(f"Could not determine namespace for existing repo via whoami due to: {whoami_e}. Using generic ID.")
|
92 |
+
return HF_DATASET_REPO_NAME # Fallback
|
93 |
+
else: # Other HTTP errors
|
94 |
+
status_code = e.response.status_code if e.response is not None else "Unknown"
|
95 |
+
logger.error(f"Hugging Face dataset HTTP error (Status: {status_code}): {str(e)}")
|
96 |
+
return f"Error: Failed to access or create dataset '{HF_DATASET_REPO_NAME}' due to HTTP error: {str(e)}"
|
97 |
+
except Exception as e: # Catch other non-HTTP exceptions from huggingface_hub or general errors
|
98 |
+
# This could be Hf একাধিক RepoExistsError if exist_ok=False, or other utility errors.
|
99 |
+
# For exist_ok=True, a 409 is the more likely signal for existing repo.
|
100 |
+
logger.error(f"Hugging Face dataset general error: {str(e)}", exc_info=True)
|
101 |
return f"Error: Failed to access or create dataset '{HF_DATASET_REPO_NAME}': {str(e)}"
|
102 |
|
103 |
|
104 |
def upload_image_to_hf_stream(image_pil, filename_base, page_num_for_log=""):
|
|
|
|
|
105 |
repo_id_or_error = ensure_hf_dataset()
|
106 |
if isinstance(repo_id_or_error, str) and repo_id_or_error.startswith("Error"):
|
107 |
return repo_id_or_error
|
|
|
123 |
)
|
124 |
logger.info(f"Successfully uploaded image: {file_url}")
|
125 |
return file_url
|
126 |
+
except Exception as e: # Catch broadly here; specific HF errors could be caught if needed
|
127 |
logger.error(f"Image upload error for {filename_base}{page_num_for_log}: {str(e)}", exc_info=True)
|
128 |
return f"Error uploading image {filename_base}{page_num_for_log}: {str(e)}"
|
129 |
finally:
|
|
|
133 |
|
134 |
|
135 |
def format_page_text_to_markdown_chunk(page_text_content):
|
|
|
|
|
|
|
136 |
chunk_md = ""
|
|
|
137 |
page_text_content = re.sub(r'\n\s*\n+', '\n\n', page_text_content.strip())
|
138 |
lines = page_text_content.split('\n')
|
139 |
is_in_list = False
|
|
|
162 |
# --- Main PDF Processing Logic (Generator Function for Streaming) ---
|
163 |
|
164 |
def generate_pdf_conversion_stream(pdf_input_source_path_or_url):
|
|
|
|
|
|
|
|
|
165 |
try:
|
|
|
166 |
yield yield_message("markdown_replace", {"content": "# Extracted PDF Content\n\n"})
|
167 |
+
time.sleep(0.01)
|
168 |
|
|
|
169 |
yield yield_message("status", {"message": "Opening PDF for text extraction..."})
|
170 |
time.sleep(0.01)
|
171 |
|
|
|
173 |
pdf_input_source_path_or_url.startswith(('http://', 'https://'))
|
174 |
|
175 |
pdf_handle_for_text = None
|
176 |
+
pdf_bytes_for_images = None
|
177 |
|
178 |
if source_is_url:
|
179 |
try:
|
180 |
+
response = requests.get(pdf_input_source_path_or_url, stream=False, timeout=60) # stream=False to get content
|
181 |
response.raise_for_status()
|
182 |
+
pdf_bytes_for_images = response.content
|
183 |
+
pdf_handle_for_text = io.BytesIO(pdf_bytes_for_images)
|
184 |
yield yield_message("status", {"message": f"PDF downloaded from URL ({len(pdf_bytes_for_images)/1024:.2f} KB)."})
|
185 |
time.sleep(0.01)
|
186 |
+
except RequestsHTTPError as e: # Catch HTTP errors specifically
|
187 |
+
logger.error(f"URL fetch HTTP error for PDF processing: {str(e)} (Status: {e.response.status_code if e.response else 'N/A'})", exc_info=True)
|
188 |
+
yield yield_message("error", {"message": f"Error fetching PDF from URL (HTTP {e.response.status_code if e.response else 'N/A'}): {e.response.reason if e.response else str(e)}"})
|
189 |
+
return
|
190 |
+
except requests.RequestException as e: # Catch other network errors
|
191 |
+
logger.error(f"URL fetch network error for PDF processing: {str(e)}", exc_info=True)
|
192 |
+
yield yield_message("error", {"message": f"Network error fetching PDF from URL: {str(e)}"})
|
193 |
+
return
|
194 |
+
else:
|
195 |
+
pdf_handle_for_text = pdf_input_source_path_or_url
|
196 |
|
197 |
total_text_pages = 0
|
198 |
try:
|
|
|
203 |
|
204 |
for i, page in enumerate(pdf.pages):
|
205 |
yield yield_message("status", {"message": f"Extracting text from page {i+1}/{total_text_pages}..."})
|
206 |
+
time.sleep(0.01)
|
207 |
|
208 |
page_text = page.extract_text(layout=True, x_density=1, y_density=1) or ""
|
209 |
|
|
|
211 |
tables = page.extract_tables()
|
212 |
if tables:
|
213 |
for table_idx, table_data in enumerate(tables):
|
214 |
+
if table_data and len(table_data) > 0 and len(table_data[0]) > 0 : # Check table has rows and columns
|
215 |
yield yield_message("status", {"message": f" Processing table {table_idx+1} on page {i+1}..."})
|
216 |
+
header_cells = table_data[0]
|
217 |
+
header = [" | ".join(str(cell) if cell is not None else "" for cell in header_cells)]
|
218 |
+
separator = [" | ".join(["---"] * len(header_cells))]
|
219 |
body = [" | ".join(str(cell) if cell is not None else "" for cell in row) for row in table_data[1:]]
|
220 |
table_md_lines = header + separator + body
|
221 |
page_tables_md += f"**Table (Page {i+1}):**\n" + "\n".join(table_md_lines) + "\n\n"
|
|
|
225 |
yield yield_message("markdown_chunk", {"content": formatted_page_text_md})
|
226 |
if page_tables_md:
|
227 |
yield yield_message("markdown_chunk", {"content": page_tables_md})
|
228 |
+
time.sleep(0.01)
|
229 |
except Exception as e:
|
230 |
logger.error(f"Error during PDF text/table extraction: {str(e)}", exc_info=True)
|
231 |
yield yield_message("error", {"message": f"Error during text extraction: {str(e)}"})
|
232 |
+
# Decide if to return or continue to image extraction. Let's try to continue.
|
233 |
|
234 |
# 2. Image Extraction and OCR
|
235 |
if not check_poppler():
|
|
|
243 |
time.sleep(0.01)
|
244 |
extracted_pil_images = []
|
245 |
try:
|
246 |
+
image_source_for_convert = None
|
247 |
if source_is_url and pdf_bytes_for_images:
|
248 |
+
image_source_for_convert = pdf_bytes_for_images
|
249 |
+
logger.info("Using downloaded bytes for image conversion.")
|
250 |
+
elif not source_is_url:
|
251 |
+
image_source_for_convert = pdf_input_source_path_or_url # Local file path
|
252 |
+
logger.info("Using local file path for image conversion.")
|
253 |
|
254 |
+
if image_source_for_convert:
|
255 |
+
# Attempt to get page count for more granular image processing if pdf2image is the bottleneck
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
256 |
try:
|
257 |
+
pdf_info = None
|
258 |
+
if isinstance(image_source_for_convert, bytes):
|
259 |
+
pdf_info = pdf2image.pdfinfo_from_bytes(image_source_for_convert, userpw=None, poppler_path=None)
|
260 |
+
else: # path
|
261 |
+
pdf_info = pdf2image.pdfinfo_from_path(image_source_for_convert, userpw=None, poppler_path=None)
|
262 |
+
|
263 |
+
num_image_pages = pdf_info.get("Pages", 0)
|
264 |
+
yield yield_message("status", {"message": f"PDF has {num_image_pages} page(s) for potential image extraction."})
|
265 |
+
|
266 |
+
# Process images page by page (or small batches) to save memory
|
267 |
+
batch_size = 1 # Process one page at a time for images
|
268 |
+
for page_idx_start in range(1, num_image_pages + 1, batch_size):
|
269 |
+
page_idx_end = min(page_idx_start + batch_size - 1, num_image_pages)
|
270 |
+
yield yield_message("status", {"message": f"Extracting images from page(s) {page_idx_start}-{page_idx_end}..."})
|
271 |
+
time.sleep(0.01)
|
272 |
+
|
273 |
+
page_images_pil = []
|
274 |
+
if isinstance(image_source_for_convert, bytes):
|
275 |
+
page_images_pil = convert_from_bytes(image_source_for_convert, dpi=150, first_page=page_idx_start, last_page=page_idx_end)
|
276 |
+
else: # path
|
277 |
+
page_images_pil = convert_from_path(image_source_for_convert, dpi=150, first_page=page_idx_start, last_page=page_idx_end)
|
278 |
+
|
279 |
+
extracted_pil_images.extend(page_images_pil) # Add to overall list for sequential numbering later
|
280 |
+
|
281 |
+
# Process this batch of images immediately
|
282 |
+
for img_pil in page_images_pil:
|
283 |
+
current_image_index = len(extracted_pil_images) # Current overall index
|
284 |
+
page_num_for_log = f"page_{page_idx_start + page_images_pil.index(img_pil)}"
|
285 |
+
yield yield_message("status", {"message": f"Processing image {current_image_index} (from PDF page {page_num_for_log}) (OCR & Upload)..."})
|
286 |
+
time.sleep(0.01)
|
287 |
+
|
288 |
+
ocr_text = ""
|
289 |
+
try:
|
290 |
+
ocr_text = pytesseract.image_to_string(img_pil).strip()
|
291 |
+
if ocr_text: yield yield_message("status", {"message": f" OCR successful for image {current_image_index}."})
|
292 |
+
except Exception as ocr_e:
|
293 |
+
logger.error(f"OCR error for image {current_image_index}: {str(ocr_e)}")
|
294 |
+
ocr_text = f"OCR failed: {str(ocr_e)}"
|
295 |
+
|
296 |
+
image_md_chunk = ""
|
297 |
+
if HF_TOKEN:
|
298 |
+
image_url_or_error = upload_image_to_hf_stream(img_pil, "pdf_image", page_num_for_log)
|
299 |
+
if isinstance(image_url_or_error, str) and not image_url_or_error.startswith("Error"):
|
300 |
+
image_md_chunk += f"\n"
|
301 |
+
yield yield_message("status", {"message": f" Image {current_image_index} uploaded."})
|
302 |
+
else:
|
303 |
+
image_md_chunk += f"**Image {current_image_index} (Upload Error):** {str(image_url_or_error)}\n\n"
|
304 |
+
yield yield_message("error", {"message": f"Failed to upload image {current_image_index}: {str(image_url_or_error)}"})
|
305 |
+
else:
|
306 |
+
image_md_chunk += f"**Image {current_image_index} (not uploaded due to missing HF_TOKEN)**\n"
|
307 |
+
|
308 |
+
if ocr_text:
|
309 |
+
image_md_chunk += f"**Image {current_image_index} OCR Text:**\n```\n{ocr_text}\n```\n\n"
|
310 |
+
|
311 |
+
yield yield_message("image_md", {"content": image_md_chunk})
|
312 |
+
time.sleep(0.01)
|
313 |
+
except Exception as e_img_info:
|
314 |
+
logger.error(f"Could not get PDF info for image batching or during batched conversion: {e_img_info}", exc_info=True)
|
315 |
+
yield yield_message("error", {"message": f"Error preparing for image extraction: {e_img_info}. Trying bulk."})
|
316 |
+
# Fallback to bulk conversion if pdfinfo or batching fails (original behavior)
|
317 |
+
if isinstance(image_source_for_convert, bytes):
|
318 |
+
extracted_pil_images = convert_from_bytes(image_source_for_convert, dpi=150)
|
319 |
+
else: # path
|
320 |
+
extracted_pil_images = convert_from_path(image_source_for_convert, dpi=150)
|
321 |
+
# Process these bulk images (copy-paste the loop from above, adjust indexing)
|
322 |
+
for i, img_pil in enumerate(extracted_pil_images):
|
323 |
+
page_num_for_log = f"bulk_image_{i+1}"
|
324 |
+
yield yield_message("status", {"message": f"Processing image {i+1}/{len(extracted_pil_images)} (OCR & Upload)..."}) # ... (rest of loop) ...
|
325 |
+
# (omitted rest of duplicated loop for brevity, but it would be the same as the inner loop above)
|
326 |
+
ocr_text = pytesseract.image_to_string(img_pil).strip() # Simplified for brevity
|
327 |
+
image_md_chunk = f"![Image {i+1} Fallback]\n**OCR:** {ocr_text}\n\n"
|
328 |
+
yield yield_message("image_md", {"content": image_md_chunk})
|
329 |
+
time.sleep(0.01)
|
330 |
+
|
331 |
+
else: # No valid source for image conversion
|
332 |
+
yield yield_message("status", {"message": "No valid source (URL download failed or no file path) for image extraction."})
|
333 |
+
|
334 |
+
except Exception as e: # Catch errors from the image extraction block
|
335 |
logger.error(f"Error during image extraction/processing: {str(e)}", exc_info=True)
|
336 |
yield yield_message("error", {"message": f"Error during image extraction: {str(e)}"})
|
337 |
|
|
|
353 |
pdf_file = request.files.get('pdf_file')
|
354 |
pdf_url = request.form.get('pdf_url', '').strip()
|
355 |
|
356 |
+
# Use a list to hold temp_pdf_path so it can be modified in the inner function
|
357 |
+
# and accessed in finally. Or pass it around.
|
358 |
+
# For simplicity, we'll rely on the generator's finally block if it's created within.
|
359 |
+
# Here, temp_pdf_path is primarily for the *uploaded* file before passing its path.
|
360 |
+
|
361 |
+
outer_temp_pdf_path = None # For uploaded file cleanup
|
362 |
|
363 |
def stream_processor():
|
364 |
+
nonlocal outer_temp_pdf_path # Make it accessible in this inner function for cleanup
|
365 |
+
pdf_input_source_for_generator = None
|
366 |
|
367 |
try:
|
368 |
if pdf_file and pdf_file.filename:
|
|
|
371 |
return
|
372 |
|
373 |
filename = secure_filename(pdf_file.filename)
|
|
|
374 |
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
|
375 |
+
fd, temp_path = tempfile.mkstemp(suffix=".pdf", prefix="upload_", dir=app.config['UPLOAD_FOLDER'])
|
376 |
os.close(fd)
|
377 |
+
pdf_file.save(temp_path)
|
378 |
+
outer_temp_pdf_path = temp_path # Store for cleanup
|
379 |
+
logger.info(f"Uploaded PDF saved to temporary path: {outer_temp_pdf_path}")
|
380 |
+
pdf_input_source_for_generator = outer_temp_pdf_path
|
381 |
yield yield_message("status", {"message": f"Processing uploaded PDF: {filename}"})
|
382 |
time.sleep(0.01)
|
383 |
|
|
|
386 |
if not (unquoted_url.startswith('http://') or unquoted_url.startswith('https://')):
|
387 |
yield yield_message("error", {"message": "Invalid URL scheme. Must be http or https."})
|
388 |
return
|
|
|
389 |
|
390 |
pdf_input_source_for_generator = unquoted_url
|
391 |
yield yield_message("status", {"message": f"Preparing to process PDF from URL: {unquoted_url}"})
|
|
|
394 |
yield yield_message("error", {"message": "No PDF file uploaded and no PDF URL provided."})
|
395 |
return
|
396 |
|
|
|
397 |
for message_part in generate_pdf_conversion_stream(pdf_input_source_for_generator):
|
398 |
yield message_part
|
|
|
399 |
|
400 |
except Exception as e:
|
401 |
logger.error(f"Error setting up stream or in initial validation: {str(e)}", exc_info=True)
|
402 |
yield yield_message("error", {"message": f"Setup error: {str(e)}"})
|
403 |
+
# The 'finally' block for cleaning outer_temp_pdf_path will be outside this generator,
|
404 |
+
# in the main route function after the Response is fully generated.
|
405 |
+
# However, with stream_with_context, the 'finally' here is better.
|
406 |
finally:
|
407 |
+
if outer_temp_pdf_path and os.path.exists(outer_temp_pdf_path):
|
408 |
try:
|
409 |
+
os.remove(outer_temp_pdf_path)
|
410 |
+
logger.info(f"Cleaned up temporary PDF: {outer_temp_pdf_path}")
|
411 |
+
# Yielding from finally inside a generator that's part of a streamed response can be tricky.
|
412 |
+
# It's better if status messages about cleanup are logged or handled differently.
|
413 |
+
# For this case, logging is sufficient.
|
414 |
except OSError as ose:
|
415 |
+
logger.error(f"Error removing temporary PDF {outer_temp_pdf_path}: {ose}")
|
|
|
416 |
|
|
|
417 |
return Response(stream_with_context(stream_processor()), mimetype='application/x-ndjson')
|
418 |
|
419 |
|
420 |
# --- Main Execution ---
|
421 |
if __name__ == '__main__':
|
422 |
+
if not check_poppler():
|
423 |
logger.warning("Poppler utilities might not be installed correctly. PDF processing might fail.")
|
424 |
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
|
|
|
|
|
425 |
app.run(host='0.0.0.0', port=int(os.getenv("PORT", 7860)), debug=True, threaded=True)
|