Spaces:
Running
Running
| # diagnostics.py | |
| import logging | |
| import time | |
| import os | |
| from pathlib import Path | |
| import pandas as pd | |
| import streamlit as st | |
| # Configure logging | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', | |
| handlers=[ | |
| logging.FileHandler("metadata_manager.log"), | |
| logging.StreamHandler() | |
| ] | |
| ) | |
| def diagnose_parquet_files(directory_path): | |
| """Diagnostic tool to verify parquet files are readable and valid""" | |
| logger = logging.getLogger("ParquetDiagnostic") | |
| logger.info(f"Starting parquet file diagnostics in {directory_path}") | |
| dir_path = Path(directory_path) | |
| if not dir_path.exists(): | |
| logger.error(f"Directory does not exist: {dir_path}") | |
| return False | |
| all_files = list(dir_path.glob("*.parquet")) | |
| logger.info(f"Found {len(all_files)} parquet files") | |
| if not all_files: | |
| logger.warning("No parquet files found") | |
| return False | |
| success_count = 0 | |
| issue_count = 0 | |
| total_rows = 0 | |
| for file_path in all_files: | |
| logger.info(f"Diagnosing file: {file_path}") | |
| file_size_mb = os.path.getsize(file_path) / (1024 * 1024) | |
| logger.info(f"File size: {file_size_mb:.2f} MB") | |
| try: | |
| # Try to read the file metadata without loading the data | |
| import pyarrow.parquet as pq | |
| parquet_file = pq.ParquetFile(file_path) | |
| # Log schema information | |
| schema = parquet_file.schema.to_arrow_schema() | |
| logger.info(f"Schema: {schema}") | |
| # Log file metadata | |
| metadata = parquet_file.metadata | |
| num_rows = metadata.num_rows | |
| num_columns = len(schema.names) | |
| logger.info(f"Rows: {num_rows}, Columns: {num_columns}") | |
| # Try to read a small sample to verify data can be loaded | |
| sample_df = pd.read_parquet(file_path, engine='pyarrow') | |
| actual_rows = len(sample_df) | |
| logger.info(f"Successfully read {actual_rows} rows") | |
| total_rows += actual_rows | |
| success_count += 1 | |
| except Exception as e: | |
| logger.error(f"Failed to read file {file_path}: {str(e)}", exc_info=True) | |
| issue_count += 1 | |
| # Try alternate engines if primary fails | |
| try: | |
| logger.info("Attempting to read with fastparquet engine") | |
| sample_df = pd.read_parquet(file_path, engine='fastparquet') | |
| logger.info(f"fastparquet succeeded, read {len(sample_df)} rows") | |
| except Exception as e2: | |
| logger.error(f"fastparquet also failed: {str(e2)}") | |
| logger.info(f"Diagnostics complete: {success_count} files OK, {issue_count} files with issues") | |
| logger.info(f"Total rows across all files: {total_rows}") | |
| return success_count > 0 |