app/download_embedding.py

import os
from huggingface_hub import login
from sentence_transformers import SentenceTransformer

hf_token = os.getenv('HF_TOKEN')
if hf_token:
    login(token=hf_token)

cache_dir = '/app/hf_cache'
model_name = 'sentence-transformers/all-MiniLM-L6-v2'

expected_cache_path = os.path.join(cache_dir, 'sentence-transformers', model_name.replace('/', '--'))

if not os.path.exists(expected_cache_path):
    print(f'Downloading embedding model: {model_name}...')
    embedding_model = SentenceTransformer(model_name, cache_folder=cache_dir)
else:
    print(f'Embedding model ({model_name}) already cached. Skipping download.')

print('Embedding model download check complete.')