Implement comprehensive environment variable setup and directory management for Streamlit app

- Added extensive environment variable configurations to prevent root filesystem access and ensure proper functioning of ML libraries.
- Implemented a robust directory creation process with error handling and fallback mechanisms for various library paths.
- Enhanced logging to provide detailed feedback on directory creation and environment variable settings, improving overall stability and user experience.

src/streamlit_app.py

@@ -13,6 +13,118 @@ import difflib
 import re
 import time
 
+# Set environment variables IMMEDIATELY to prevent root filesystem access
+# This must happen before any other imports or operations
+
+# Get a writable temp directory first
+try:
+    TEMP_DIR = os.path.join(tempfile.gettempdir(), "docling_temp")
+    os.makedirs(TEMP_DIR, exist_ok=True)
+except Exception:
+    try:
+        TEMP_DIR = "/tmp/docling_temp"
+        os.makedirs(TEMP_DIR, exist_ok=True)
+    except Exception:
+        TEMP_DIR = os.getcwd()
+
+# Set all environment variables that libraries might use
+os.environ.update({
+    # Streamlit configuration
+    'STREAMLIT_SERVER_FILE_WATCHER_TYPE': 'none',
+    'STREAMLIT_SERVER_HEADLESS': 'true',
+    'STREAMLIT_BROWSER_GATHER_USAGE_STATS': 'false',
+    'STREAMLIT_SERVER_ENABLE_CORS': 'false',
+    'STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION': 'false',
+    
+    # EasyOCR configuration
+    'EASYOCR_MODULE_PATH': os.path.join(TEMP_DIR, 'easyocr_models'),
+    'HOME': TEMP_DIR,
+    'USERPROFILE': TEMP_DIR,
+    'XDG_CACHE_HOME': os.path.join(TEMP_DIR, 'cache'),
+    'XDG_CONFIG_HOME': os.path.join(TEMP_DIR, 'config'),
+    'XDG_DATA_HOME': os.path.join(TEMP_DIR, 'data'),
+    
+    # Hugging Face Hub configuration
+    'HF_HOME': os.path.join(TEMP_DIR, 'huggingface'),
+    'HF_CACHE_HOME': os.path.join(TEMP_DIR, 'huggingface_cache'),
+    'TRANSFORMERS_CACHE': os.path.join(TEMP_DIR, 'transformers_cache'),
+    'HF_DATASETS_CACHE': os.path.join(TEMP_DIR, 'datasets_cache'),
+    
+    # Other ML libraries
+    'TORCH_HOME': os.path.join(TEMP_DIR, 'torch'),
+    'TENSORFLOW_HOME': os.path.join(TEMP_DIR, 'tensorflow'),
+    'KERAS_HOME': os.path.join(TEMP_DIR, 'keras'),
+    'MLFLOW_TRACKING_URI': f'file:{os.path.join(TEMP_DIR, "mlruns")}',
+    
+    # Additional cache directories
+    'CACHE_DIR': os.path.join(TEMP_DIR, 'cache'),
+    'MODEL_CACHE_DIR': os.path.join(TEMP_DIR, 'models'),
+    
+    # Additional environment variables to prevent root access
+    'PYTHONPATH': TEMP_DIR,
+    'TMPDIR': TEMP_DIR,
+    'TEMP': TEMP_DIR,
+    'TMP': TEMP_DIR,
+    'CACHE': os.path.join(TEMP_DIR, 'cache'),
+    'MODELS': os.path.join(TEMP_DIR, 'models'),
+    'DATA': os.path.join(TEMP_DIR, 'data'),
+    'CONFIG': os.path.join(TEMP_DIR, 'config'),
+    
+    # Specific cache overrides
+    'HF_HUB_CACHE': os.path.join(TEMP_DIR, 'huggingface_cache'),
+    'HF_DATASETS_CACHE': os.path.join(TEMP_DIR, 'datasets_cache'),
+    'TRANSFORMERS_CACHE': os.path.join(TEMP_DIR, 'transformers_cache'),
+    'DIFFUSERS_CACHE': os.path.join(TEMP_DIR, 'diffusers_cache'),
+    'ACCELERATE_CACHE': os.path.join(TEMP_DIR, 'accelerate_cache'),
+})
+
+# Create all necessary directories
+directories_to_create = [
+    os.environ['EASYOCR_MODULE_PATH'],
+    os.environ['XDG_CACHE_HOME'],
+    os.environ['XDG_CONFIG_HOME'],
+    os.environ['XDG_DATA_HOME'],
+    os.environ['HF_HOME'],
+    os.environ['HF_CACHE_HOME'],
+    os.environ['TRANSFORMERS_CACHE'],
+    os.environ['HF_DATASETS_CACHE'],
+    os.environ['TORCH_HOME'],
+    os.environ['TENSORFLOW_HOME'],
+    os.environ['KERAS_HOME'],
+    os.environ['CACHE_DIR'],
+    os.environ['MODEL_CACHE_DIR'],
+    os.environ['CACHE'],
+    os.environ['MODELS'],
+    os.environ['DATA'],
+    os.environ['CONFIG'],
+    os.environ['HF_HUB_CACHE'],
+    os.environ['DIFFUSERS_CACHE'],
+    os.environ['ACCELERATE_CACHE'],
+]
+
+# Monkey patch os.makedirs to prevent root directory access
+original_makedirs = os.makedirs
+
+def safe_makedirs(name, mode=0o777, exist_ok=False):
+    """Safe version of makedirs that prevents root directory access."""
+    # Check if trying to create directory in root filesystem
+    if name.startswith('/') and not name.startswith('/tmp') and not name.startswith('/app'):
+        # Redirect to temp directory
+        basename = os.path.basename(name)
+        safe_name = os.path.join(TEMP_DIR, basename)
+        print(f"Redirecting root directory creation from {name} to {safe_name}")
+        return original_makedirs(safe_name, mode, exist_ok)
+    return original_makedirs(name, mode, exist_ok)
+
+# Apply the monkey patch
+os.makedirs = safe_makedirs
+
+for directory in directories_to_create:
+    try:
+        os.makedirs(directory, exist_ok=True)
+    except Exception as e:
+        print(f"Warning: Could not create directory {directory}: {e}")
+
 # Configure logging early to avoid issues
 logging.basicConfig(
     level=logging.INFO,
@@ -29,125 +141,12 @@ AZURE_OPENAI_KEY = os.getenv("AZURE_OPENAI_KEY")
 AZURE_OPENAI_VERSION = os.getenv("AZURE_OPENAI_VERSION")
 AZURE_OPENAI_DEPLOYMENT = os.getenv("AZURE_OPENAI_DEPLOYMENT")
 
-# Use system temp directory instead of local directory to avoid permission issues
-TEMP_DIR = os.path.join(tempfile.gettempdir(), "docling_temp")
-try:
-    os.makedirs(TEMP_DIR, exist_ok=True)
-    logging.info(f"Using temp directory: {TEMP_DIR}")
-except PermissionError as e:
-    logging.warning(f"Permission error creating temp dir {TEMP_DIR}: {e}")
-    # Fallback to a subdirectory of the current working directory if temp dir fails
-    TEMP_DIR = os.path.join(os.getcwd(), "temp_files")
-    try:
-        os.makedirs(TEMP_DIR, exist_ok=True)
-        logging.info(f"Using fallback temp directory: {TEMP_DIR}")
-    except PermissionError as e2:
-        logging.warning(f"Permission error creating fallback temp dir {TEMP_DIR}: {e2}")
-        # Last resort: use a directory that should be writable
-        TEMP_DIR = "/tmp/docling_temp"
-        try:
-            os.makedirs(TEMP_DIR, exist_ok=True)
-            logging.info(f"Using last resort temp directory: {TEMP_DIR}")
-        except Exception as e3:
-            logging.error(f"Failed to create any temp directory: {e3}")
-            # Use current directory as absolute last resort
-            TEMP_DIR = "."
-            logging.warning(f"Using current directory as temp: {TEMP_DIR}")
-
-# Configure Streamlit to use writable directories
-os.environ['STREAMLIT_SERVER_FILE_WATCHER_TYPE'] = 'none'
-os.environ['STREAMLIT_SERVER_HEADLESS'] = 'true'
-os.environ['STREAMLIT_BROWSER_GATHER_USAGE_STATS'] = 'false'
-
-# Additional environment variables for Hugging Face deployment
-os.environ['STREAMLIT_SERVER_ENABLE_CORS'] = 'false'
-os.environ['STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION'] = 'false'
-
-# Configure EasyOCR to use writable directory for model storage
-os.environ['EASYOCR_MODULE_PATH'] = os.path.join(TEMP_DIR, 'easyocr_models')
-try:
-    os.makedirs(os.environ['EASYOCR_MODULE_PATH'], exist_ok=True)
-    logging.info(f"EasyOCR model directory: {os.environ['EASYOCR_MODULE_PATH']}")
-except Exception as e:
-    logging.warning(f"Could not create EasyOCR model directory: {e}")
-    # Fallback to /tmp if the temp directory fails
-    os.environ['EASYOCR_MODULE_PATH'] = '/tmp/easyocr_models'
-    try:
-        os.makedirs(os.environ['EASYOCR_MODULE_PATH'], exist_ok=True)
-        logging.info(f"Using fallback EasyOCR model directory: {os.environ['EASYOCR_MODULE_PATH']}")
-    except Exception as e2:
-        logging.error(f"Failed to create EasyOCR model directory: {e2}")
-        # Last resort: use current directory
-        os.environ['EASYOCR_MODULE_PATH'] = os.path.join(os.getcwd(), 'easyocr_models')
-        os.makedirs(os.environ['EASYOCR_MODULE_PATH'], exist_ok=True)
-        logging.warning(f"Using current directory for EasyOCR models: {os.environ['EASYOCR_MODULE_PATH']}")
-
-# Additional EasyOCR environment variables to prevent root directory access
-os.environ['HOME'] = TEMP_DIR  # Set HOME to temp directory
-os.environ['USERPROFILE'] = TEMP_DIR  # For Windows compatibility
-os.environ['XDG_CACHE_HOME'] = os.path.join(TEMP_DIR, 'cache')
-os.environ['XDG_CONFIG_HOME'] = os.path.join(TEMP_DIR, 'config')
-os.environ['XDG_DATA_HOME'] = os.path.join(TEMP_DIR, 'data')
-
-# Create additional directories that EasyOCR might need
-for env_var in ['XDG_CACHE_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME']:
-    try:
-        os.makedirs(os.environ[env_var], exist_ok=True)
-        logging.info(f"Created directory for {env_var}: {os.environ[env_var]}")
-    except Exception as e:
-        logging.warning(f"Could not create directory for {env_var}: {e}")
-
-# Configure Hugging Face Hub to use writable directories
-os.environ['HF_HOME'] = os.path.join(TEMP_DIR, 'huggingface')
-os.environ['HF_CACHE_HOME'] = os.path.join(TEMP_DIR, 'huggingface_cache')
-os.environ['TRANSFORMERS_CACHE'] = os.path.join(TEMP_DIR, 'transformers_cache')
-os.environ['HF_DATASETS_CACHE'] = os.path.join(TEMP_DIR, 'datasets_cache')
-
-# Create Hugging Face directories
-hf_dirs = ['HF_HOME', 'HF_CACHE_HOME', 'TRANSFORMERS_CACHE', 'HF_DATASETS_CACHE']
-for env_var in hf_dirs:
-    try:
-        os.makedirs(os.environ[env_var], exist_ok=True)
-        logging.info(f"Created Hugging Face directory for {env_var}: {os.environ[env_var]}")
-    except Exception as e:
-        logging.warning(f"Could not create Hugging Face directory for {env_var}: {e}")
-        # Fallback to /tmp if the temp directory fails
-        fallback_path = os.path.join('/tmp', env_var.lower())
-        os.environ[env_var] = fallback_path
-        try:
-            os.makedirs(fallback_path, exist_ok=True)
-            logging.info(f"Using fallback Hugging Face directory for {env_var}: {fallback_path}")
-        except Exception as e2:
-            logging.error(f"Failed to create fallback Hugging Face directory for {env_var}: {e2}")
-
-# Additional environment variables for other libraries that might access root directories
-os.environ['TORCH_HOME'] = os.path.join(TEMP_DIR, 'torch')
-os.environ['TENSORFLOW_HOME'] = os.path.join(TEMP_DIR, 'tensorflow')
-os.environ['KERAS_HOME'] = os.path.join(TEMP_DIR, 'keras')
-os.environ['MLFLOW_TRACKING_URI'] = 'file:' + os.path.join(TEMP_DIR, 'mlruns')
-
-# Create additional library directories
-lib_dirs = ['TORCH_HOME', 'TENSORFLOW_HOME', 'KERAS_HOME']
-for env_var in lib_dirs:
-    try:
-        os.makedirs(os.environ[env_var], exist_ok=True)
-        logging.info(f"Created library directory for {env_var}: {os.environ[env_var]}")
-    except Exception as e:
-        logging.warning(f"Could not create library directory for {env_var}: {e}")
-        # Fallback to /tmp
-        fallback_path = os.path.join('/tmp', env_var.lower())
-        os.environ[env_var] = fallback_path
-        try:
-            os.makedirs(fallback_path, exist_ok=True)
-            logging.info(f"Using fallback library directory for {env_var}: {fallback_path}")
-        except Exception as e2:
-            logging.error(f"Failed to create fallback library directory for {env_var}: {e2}")
-
 # Log startup information
 logging.info("=" * 50)
 logging.info("Docling Streamlit App Starting")
 logging.info(f"Temp directory: {TEMP_DIR}")
 logging.info(f"EasyOCR model directory: {os.environ.get('EASYOCR_MODULE_PATH', 'NOT_SET')}")
+logging.info(f"Hugging Face cache: {os.environ.get('HF_CACHE_HOME', 'NOT_SET')}")
 logging.info(f"Current working directory: {os.getcwd()}")
 logging.info(f"Python version: {sys.version}")
 logging.info("=" * 50)