Update app.py

app.py

@@ -14,6 +14,7 @@ from typing import List, Dict, Any, Optional
 from ultralytics import YOLO
 import piexif
 import zipfile
+import subprocess
 
 os.environ["YOLO_CONFIG_DIR"] = "/tmp/Ultralytics"
 logging.basicConfig(filename="app.log", level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -46,22 +47,57 @@ def zip_all_outputs(report_path: str, video_path: str, chart_path: str, map_path
     zip_path = os.path.join(OUTPUT_DIR, f"drone_analysis_outputs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip")
     try:
         with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-            if os.path.exists(report_path):
+            # Add report file
+            if report_path and os.path.exists(report_path):
                 zipf.write(report_path, os.path.basename(report_path))
-            if os.path.exists(video_path):
+                log_entries.append(f"Added report to ZIP: {report_path}")
+            else:
+                log_entries.append(f"Warning: Report file not found for ZIP: {report_path}")
+
+            # Add video file
+            if video_path and os.path.exists(video_path):
                 zipf.write(video_path, os.path.join("outputs", os.path.basename(video_path)))
-            if os.path.exists(chart_path):
+                log_entries.append(f"Added video to ZIP: {video_path}")
+            else:
+                log_entries.append(f"Warning: Video file not found for ZIP: {video_path}")
+
+            # Add chart file
+            if chart_path and os.path.exists(chart_path):
                 zipf.write(chart_path, os.path.join("outputs", os.path.basename(chart_path)))
-            if os.path.exists(map_path):
+                log_entries.append(f"Added chart to ZIP: {chart_path}")
+            else:
+                log_entries.append(f"Warning: Chart file not found for ZIP: {chart_path}")
+
+            # Add map file
+            if map_path and os.path.exists(map_path):
                 zipf.write(map_path, os.path.join("outputs", os.path.basename(map_path)))
+                log_entries.append(f"Added map to ZIP: {map_path}")
+            else:
+                log_entries.append(f"Warning: Map file not found for ZIP: {map_path}")
+
+            # Add detected issue images
             for file in detected_issues:
                 if os.path.exists(file):
                     zipf.write(file, os.path.join("captured_frames", os.path.basename(file)))
+                    log_entries.append(f"Added image to ZIP: {file}")
+                else:
+                    log_entries.append(f"Warning: Image file not found for ZIP: {file}")
+
+            # Add flight logs
             for root, _, files in os.walk(FLIGHT_LOG_DIR):
                 for file in files:
                     file_path = os.path.join(root, file)
-                    zipf.write(file_path, os.path.join("flight_logs", file))
+                    if os.path.exists(file_path):
+                        zipf.write(file_path, os.path.join("flight_logs", file))
+                        log_entries.append(f"Added flight log to ZIP: {file_path}")
+                    else:
+                        log_entries.append(f"Warning: Flight log not found for ZIP: {file_path}")
+
         log_entries.append(f"Created ZIP: {zip_path}")
+        if os.path.exists(zip_path):
+            log_entries.append(f"Confirmed ZIP file exists: {zip_path}")
+        else:
+            log_entries.append(f"Error: ZIP file not created: {zip_path}")
         return zip_path
     except Exception as e:
         log_entries.append(f"Error: Failed to create ZIP: {str(e)}")
@@ -163,10 +199,11 @@ def generate_report(
     io_times: List[float]
 ) -> str:
     # Generating LaTeX-based PDF report
-    log_entries.append("Generating LaTeX report...")
-    report_path = os.path.join(OUTPUT_DIR, f"drone_analysis_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf")
-    tex_path = os.path.join(OUTPUT_DIR, f"drone_analysis_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.tex")
+    log_entries.append("Generating report...")
     timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    report_path = os.path.join(OUTPUT_DIR, f"drone_analysis_report_{timestamp}.pdf")
+    tex_path = os.path.join(OUTPUT_DIR, f"drone_analysis_report_{timestamp}.tex")
+    md_path = os.path.join(OUTPUT_DIR, f"drone_analysis_report_{timestamp}.md")  # Fallback Markdown path
 
     # LaTeX document content
     report_content = [
@@ -218,7 +255,7 @@ def generate_report(
         r"\item \textbf{Flight Pattern}: Single lap, ROW centered",
         r"\item \textbf{Geotagging}: Enabled",
         r"\item \textbf{Satellite Lock}: 12 satellites",
-        r"\item \textbf{Terrain Follow Mode}: Enabled",
+        r"\item \textbf terrain Follow Mode}: Enabled",
         r"\end{itemize}",
         r"\section{Quality Check Results}",
         r"\begin{itemize}",
@@ -312,7 +349,6 @@ def generate_report(
         rf"Issue Locations Map: \texttt{{outputs/map\_<{timestamp}>.png}}\par"
     ])
 
-    # Including chart and map images
     if chart_path and os.path.exists(chart_path):
         report_content.extend([
             r"\begin{figure}[h]",
@@ -360,6 +396,140 @@ def generate_report(
         r"\end{document}"
     ])
 
+    # Fallback Markdown report content
+    md_report_content = [
+        "# NHAI Drone Survey Analysis Report",
+        "",
+        "## Project Details",
+        "- Project Name: NH-44 Delhi-Hyderabad Section (Package XYZ)",
+        "- Highway Section: Km 100 to Km 150",
+        "- State: Telangana",
+        "- Region: South",
+        f"- Survey Date: {datetime.now().strftime('%Y-%m-%d')}",
+        "- Drone Service Provider: ABC Drone Services Pvt. Ltd.",
+        "- Technology Service Provider: XYZ AI Analytics Ltd.",
+        f"- Work Order Reference: Data Lake WO-{datetime.now().strftime('%Y-%m-%d')}-XYZ",
+        "- Report Prepared By: Nagasurendra, Data Analyst",
+        f"- Report Date: {datetime.now().strftime('%Y-%m-%d')}",
+        "",
+        "## 1. Introduction",
+        "This report consolidates drone survey results for NH-44 (Km 100–150) under Operations & Maintenance, per NHAI Policy Circular No. 18.98/2024, detecting potholes and cracks using YOLOv8 for Monthly Progress Report integration.",
+        "",
+        "## 2. Drone Survey Metadata",
+        "- Drone Speed: 5 m/s",
+        "- Drone Height: 60 m",
+        "- Camera Sensor: RGB, 12 MP",
+        "- Recording Type: JPEG, 90° nadir",
+        "- Image Overlap: 85%",
+        "- Flight Pattern: Single lap, ROW centered",
+        "- Geotagging: Enabled",
+        "- Satellite Lock: 12 satellites",
+        "- Terrain Follow Mode: Enabled",
+        "",
+        "## 3. Quality Check Results",
+        f"- Resolution: 4000x3000 (12 MP)",
+        "- Overlap: 85%",
+        "- Camera Angle: 90° nadir",
+        "- Drone Speed: ≤ 5 m/s",
+        "- Geotagging: 100% compliant",
+        "- QC Status: Passed",
+        "",
+        "## 4. AI/ML Analytics",
+        f"- Total Frames Processed: {frame_count}",
+        f"- Detection Frames: {detection_frame_count} ({detection_frame_count/frame_count*100:.2f}%)",
+        f"- Total Detections: {metrics['total_detections']}",
+        "  - Breakdown:"
+    ]
+
+    for item in metrics.get("items", []):
+        percentage = (item["count"] / metrics["total_detections"] * 100) if metrics["total_detections"] > 0 else 0
+        md_report_content.append(f"    - {item['type']}: {item['count']} ({percentage:.2f}%)")
+    md_report_content.extend([
+        f"- Processing Time: {total_time:.2f} seconds",
+        f"- Average Frame Time: {sum(frame_times)/len(frame_times):.2f} ms" if frame_times else "- Average Frame Time: N/A",
+        f"- Average Resize Time: {sum(resize_times)/len(resize_times):.2f} ms" if resize_times else "- Average Resize Time: N/A",
+        f"- Average Inference Time: {sum(inference_times)/len(inference_times):.2f} ms" if inference_times else "- Average Inference Time: N/A",
+        f"- Average I/O Time: {sum(io_times)/len(io_times):.2f} ms" if io_times else "- Average I/O Time: N/A",
+        f"- Timestamp: {metrics.get('timestamp', 'N/A')}",
+        "- Summary: Potholes and cracks detected in high-traffic segments.",
+        "",
+        "## 5. Output File Structure",
+        "- ZIP file contains:",
+        f"  - `drone_analysis_report_{timestamp}.md`: This report (fallback due to PDF generation failure)",
+        "  - `outputs/processed_output.mp4`: Processed video with annotations",
+        f"  - `outputs/chart_{timestamp}.png`: Detection trend chart",
+        f"  - `outputs/map_{timestamp}.png`: Issue locations map",
+        "  - `captured_frames/detected_<frame>.jpg`: Geotagged images for detected issues",
+        "  - `flight_logs/flight_log_<frame>.csv`: Flight logs matching image frames",
+        "- Note: Images and logs share frame numbers (e.g., `detected_000001.jpg` corresponds to `flight_log_000001.csv`).",
+        "",
+        "## 6. Geotagged Images",
+        f"- Total Images: {len(detected_issues)}",
+        f"- Storage: Data Lake `/project_xyz/images/{datetime.now().strftime('%Y-%m-%d')}`",
+        "",
+        "| Frame | Issue Type | GPS (Lat, Lon) | Timestamp | Confidence | Image Path |",
+        "|-------|------------|----------------|-----------|------------|------------|"
+    ])
+
+    for detection in all_detections[:100]:
+        md_report_content.append(
+            f"| {detection['frame']:06d} | {detection['label']} | ({detection['gps'][0]:.6f}, {detection['gps'][1]:.6f}) | {detection['timestamp']} | {detection['conf']:.2f} | captured_frames/{os.path.basename(detection['path'])} |"
+        )
+
+    md_report_content.extend([
+        "",
+        "## 7. Flight Logs",
+        f"- Total Logs: {len(detected_issues)}",
+        f"- Storage: Data Lake `/project_xyz/flight_logs/{datetime.now().strftime('%Y-%m-%d')}`",
+        "",
+        "| Frame | Timestamp | Latitude | Longitude | Speed (m/s) | Satellites | Altitude (m) | Log Path |",
+        "|-------|-----------|----------|-----------|-------------|------------|--------------|----------|"
+    ])
+
+    for detection in all_detections[:100]:
+        log_path = f"flight_logs/flight_log_{detection['frame']:06d}.csv"
+        md_report_content.append(
+            f"| {detection['frame']:06d} | {detection['timestamp']} | {detection['gps'][0]:.6f} | {detection['gps'][1]:.6f} | 5.0 | 12 | 60 | {log_path} |"
+        )
+
+    md_report_content.extend([
+        "",
+        "## 8. Processed Video",
+        f"- Path: outputs/processed_output.mp4",
+        f"- Frames: {output_frames}",
+        f"- FPS: {output_fps:.2f}",
+        f"- Duration: {output_duration:.2f} seconds",
+        "",
+        "## 9. Visualizations",
+        f"- Detection Trend Chart: outputs/chart_{timestamp}.png",
+        f"- Issue Locations Map: outputs/map_{timestamp}.png",
+        "",
+        "## 10. Processing Timestamps",
+        f"- Total Processing Time: {total_time:.2f} seconds",
+        "- Log Entries (Last 10):"
+    ])
+
+    for entry in log_entries[-10:]:
+        md_report_content.append(f"  - {entry}")
+
+    md_report_content.extend([
+        "",
+        "## 11. Stakeholder Validation",
+        "- AE/IE Comments: [Pending]",
+        "- PD/RO Comments: [Pending]",
+        "",
+        "## 12. Recommendations",
+        "- Repair potholes in high-traffic segments.",
+        "- Seal cracks to prevent degradation.",
+        "- Schedule follow-up survey.",
+        "",
+        "## 13. Data Lake References",
+        f"- Images: `/project_xyz/images/{datetime.now().strftime('%Y-%m-%d')}`",
+        f"- Flight Logs: `/project_xyz/flight_logs/{datetime.now().strftime('%Y-%m-%d')}`",
+        f"- Video: `/project_xyz/videos/processed_output_{datetime.now().strftime('%Y%m%d')}.mp4`",
+        f"- DAMS Dashboard: `/project_xyz/dams/{datetime.now().strftime('%Y-%m-%d')}`"
+    ])
+
     try:
         # Writing LaTeX file
         with open(tex_path, 'w') as f:
@@ -367,7 +537,6 @@ def generate_report(
         log_entries.append(f"LaTeX file saved: {tex_path}")
 
         # Compiling LaTeX to PDF using latexmk
-        import subprocess
         result = subprocess.run(
             ["latexmk", "-pdf", "-interaction=nonstopmode", tex_path],
             cwd=OUTPUT_DIR,
@@ -378,76 +547,30 @@ def generate_report(
             log_entries.append(f"PDF report generated: {report_path}")
             if os.path.exists(report_path):
                 log_entries.append(f"Confirmed report file exists: {report_path}")
+                return report_path
             else:
                 log_entries.append(f"Error: PDF report not found at {report_path}")
-                return ""
         else:
             log_entries.append(f"Error: LaTeX compilation failed: {result.stderr}")
-            return ""
-        return report_path
     except Exception as e:
-        log_entries.append(f"Error: Failed to generate report: {str(e)}")
-        return ""
+        log_entries.append(f"Error: Failed to generate PDF report: {str(e)}")
 
-def zip_all_outputs(report_path: str, video_path: str, chart_path: str, map_path: str) -> str:
-    zip_path = os.path.join(OUTPUT_DIR, f"drone_analysis_outputs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip")
+    # Fallback: Generate Markdown report if PDF fails
+    log_entries.append("Falling back to Markdown report due to PDF generation failure")
     try:
-        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-            # Add report file
-            if report_path and os.path.exists(report_path):
-                zipf.write(report_path, os.path.basename(report_path))
-                log_entries.append(f"Added report to ZIP: {report_path}")
-            else:
-                log_entries.append(f"Warning: Report file not found for ZIP: {report_path}")
-
-            # Add video file
-            if video_path and os.path.exists(video_path):
-                zipf.write(video_path, os.path.join("outputs", os.path.basename(video_path)))
-                log_entries.append(f"Added video to ZIP: {video_path}")
-            else:
-                log_entries.append(f"Warning: Video file not found for ZIP: {video_path}")
-
-            # Add chart file
-            if chart_path and os.path.exists(chart_path):
-                zipf.write(chart_path, os.path.join("outputs", os.path.basename(chart_path)))
-                log_entries.append(f"Added chart to ZIP: {chart_path}")
-            else:
-                log_entries.append(f"Warning: Chart file not found for ZIP: {chart_path}")
-
-            # Add map file
-            if map_path and os.path.exists(map_path):
-                zipf.write(map_path, os.path.join("outputs", os.path.basename(map_path)))
-                log_entries.append(f"Added map to ZIP: {map_path}")
-            else:
-                log_entries.append(f"Warning: Map file not found for ZIP: {map_path}")
-
-            # Add detected issue images
-            for file in detected_issues:
-                if os.path.exists(file):
-                    zipf.write(file, os.path.join("captured_frames", os.path.basename(file)))
-                    log_entries.append(f"Added image to ZIP: {file}")
-                else:
-                    log_entries.append(f"Warning: Image file not found for ZIP: {file}")
-
-            # Add flight logs
-            for root, _, files in os.walk(FLIGHT_LOG_DIR):
-                for file in files:
-                    file_path = os.path.join(root, file)
-                    if os.path.exists(file_path):
-                        zipf.write(file_path, os.path.join("flight_logs", file))
-                        log_entries.append(f"Added flight log to ZIP: {file_path}")
-                    else:
-                        log_entries.append(f"Warning: Flight log not found for ZIP: {file_path}")
-
-        log_entries.append(f"Created ZIP: {zip_path}")
-        if os.path.exists(zip_path):
-            log_entries.append(f"Confirmed ZIP file exists: {zip_path}")
+        with open(md_path, 'w') as f:
+            f.write("\n".join(md_report_content))
+        log_entries.append(f"Markdown report saved: {md_path}")
+        if os.path.exists(md_path):
+            log_entries.append(f"Confirmed Markdown report file exists: {md_path}")
+            return md_path
         else:
-            log_entries.append(f"Error: ZIP file not created: {zip_path}")
-        return zip_path
+            log_entries.append(f"Error: Markdown report not found at {md_path}")
+            return ""
     except Exception as e:
-        log_entries.append(f"Error: Failed to create ZIP: {str(e)}")
+        log_entries.append(f"Error: Failed to save Markdown report: {str(e)}")
         return ""
+
 def process_video(video, resize_width=4000, resize_height=3000, frame_skip=5):
     global frame_count, last_metrics, detected_counts, detected_issues, gps_coordinates, log_entries
     frame_count = 0