Spaces:

yonnel
/

karl-movie-vector-backend

Sleeping

App Files Files Community

yonnel commited on Jun 12

Commit

14e32e0

1 Parent(s): 0236fb6

Add admin router and vector update functionality; enhance environment configuration

Browse files

Files changed (10) hide show

.env.example +14 -1
app/main.py +23 -1
app/routers/__init__.py +3 -0
app/routers/admin.py +319 -0
app/services/__init__.py +3 -0
app/services/embedding_service.py +139 -0
app/services/tmdb_service.py +88 -0
app/services/vector_storage.py +153 -0
app/services/vector_updater.py +170 -0
requirements.txt +7 -9

.env.example CHANGED Viewed

@@ -14,4 +14,17 @@ ENV=dev
 LOG_LEVEL=INFO
 # Remove adult content from TMDB results
-FILTER_ADULT_CONTENT=true  # Set to true to filter out adult content

 LOG_LEVEL=INFO
 # Remove adult content from TMDB results
+FILTER_ADULT_CONTENT=true  # Set to true to filter out adult content
+# Hugging Face configuration
+HF_TOKEN=your_hf_token_here
+HF_DATASET_REPO=your-username/karl-movie-vectors
+# Vector update configuration
+AUTO_UPDATE_VECTORS=true
+UPDATE_INTERVAL_HOURS=24
+BATCH_SIZE=100
+MAX_MOVIES_LIMIT=10000
+# Admin configuration
+ADMIN_TOKEN=your_admin_token_here

app/main.py CHANGED Viewed

@@ -187,6 +187,15 @@ def compute_barycenter(liked_indices: List[int], coords: np.ndarray):
 # FastAPI app setup
 app = FastAPI(title="Karl-Movie Vector Backend", version="1.0.0")
 # CORS configuration
 DEV_ORIGINS = [
     "http://localhost:5173",
@@ -210,7 +219,20 @@ app.add_middleware(
 @app.on_event("startup")
 async def startup_event():
     """Load data on startup"""
-    load_data()
 @app.get("/health")
 async def health_check():

 # FastAPI app setup
 app = FastAPI(title="Karl-Movie Vector Backend", version="1.0.0")
+# Ajouter l'import du router admin
+try:
+    from .routers import admin
+except ImportError:
+    from app.routers import admin
+# Ajouter le router admin
+app.include_router(admin.router)
 # CORS configuration
 DEV_ORIGINS = [
     "http://localhost:5173",
 @app.on_event("startup")
 async def startup_event():
     """Load data on startup"""
+    global vectors, id_map, faiss_index, movie_metadata
+    vectors, id_map, faiss_index, movie_metadata = load_data()
+    # Vérifier et mettre à jour les vecteurs si nécessaire au démarrage
+    if os.getenv('AUTO_UPDATE_VECTORS', 'false').lower() == 'true':
+        # Lancer en arrière-plan sans attendre
+        import asyncio
+        try:
+            from .services.vector_updater import VectorUpdater
+        except ImportError:
+            from app.services.vector_updater import VectorUpdater
+        vector_updater = VectorUpdater()
+        asyncio.create_task(vector_updater.update_vectors_if_needed())
 @app.get("/health")
 async def health_check():

app/routers/__init__.py ADDED Viewed

	@@ -0,0 +1,3 @@

+"""
+Routers package for Karl Movie Vector Backend
+"""

app/routers/admin.py ADDED Viewed

	@@ -0,0 +1,319 @@

+from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks
+from fastapi.responses import HTMLResponse
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+import os
+import logging
+from datetime import datetime
+from typing import Optional
+# Import avec gestion des erreurs pour les imports relatifs
+try:
+    from ..services.vector_updater import VectorUpdater
+except ImportError:
+    from app.services.vector_updater import VectorUpdater
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/admin", tags=["admin"])
+security = HTTPBearer()
+# Instance globale du updater
+vector_updater = VectorUpdater()
+def verify_admin_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    """Vérification du token admin"""
+    admin_token = os.getenv('ADMIN_TOKEN')
+    if not admin_token or credentials.credentials != admin_token:
+        raise HTTPException(status_code=403, detail="Invalid admin token")
+    return credentials.credentials
+@router.get("/", response_class=HTMLResponse)
+async def admin_dashboard():
+    """Interface web d'administration"""
+    html_content = """
+    <!DOCTYPE html>
+    <html lang="fr">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Karl Movie Vector - Admin</title>
+        <style>
+            body { font-family: Arial, sans-serif; margin: 40px; background: #f5f5f5; }
+            .container { max-width: 1000px; margin: 0 auto; background: white; padding: 30px; border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }
+            h1 { color: #333; text-align: center; margin-bottom: 30px; }
+            .card { background: #f8f9fa; padding: 20px; margin: 20px 0; border-radius: 5px; border-left: 4px solid #007bff; }
+            .status { padding: 10px; margin: 10px 0; border-radius: 4px; }
+            .status.success { background: #d4edda; color: #155724; border: 1px solid #c3e6cb; }
+            .status.error { background: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; }
+            .status.warning { background: #fff3cd; color: #856404; border: 1px solid #ffeaa7; }
+            .status.info { background: #d1ecf1; color: #0c5460; border: 1px solid #bee5eb; }
+            button { background: #007bff; color: white; border: none; padding: 12px 24px; border-radius: 4px; cursor: pointer; margin: 5px; font-size: 14px; }
+            button:hover { background: #0056b3; }
+            button:disabled { background: #6c757d; cursor: not-allowed; }
+            .danger { background: #dc3545; }
+            .danger:hover { background: #c82333; }
+            .loading { display: none; color: #007bff; margin: 10px 0; }
+            .log { background: #f8f9fa; border: 1px solid #dee2e6; padding: 15px; margin: 15px 0; border-radius: 4px; font-family: monospace; font-size: 12px; max-height: 400px; overflow-y: auto; white-space: pre-line; }
+            .config-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
+            .config-item { background: white; padding: 10px; border-radius: 4px; border: 1px solid #dee2e6; }
+        </style>
+    </head>
+    <body>
+        <div class="container">
+            <h1>🎬 Karl Movie Vector - Administration</h1>
+            <div class="card">
+                <h3>📊 Statut du Système</h3>
+                <div id="status-info"></div>
+                <button onclick="loadStatus()">🔄 Actualiser le Statut</button>
+            </div>
+            <div class="card">
+                <h3>🚀 Mise à jour des Vecteurs</h3>
+                <p>Déclenchez manuellement la mise à jour des vecteurs de films depuis TMDB.</p>
+                <div>
+                    <button onclick="updateVectors()" id="updateBtn">🔄 Mettre à jour (Conditionnel)</button>
+                    <button onclick="forceUpdateVectors()" id="forceUpdateBtn" class="danger">⚡ Forcer la Mise à jour</button>
+                </div>
+                <div id="update-loading" class="loading">⏳ Mise à jour en cours...</div>
+                <div id="update-result"></div>
+            </div>
+            <div class="card">
+                <h3>📋 Logs de Mise à jour</h3>
+                <button onclick="loadLogs()">📄 Charger les Logs</button>
+                <button onclick="clearLogs()">🗑️ Effacer l'affichage</button>
+                <div id="logs" class="log">Cliquez sur "Charger les Logs" pour voir les logs...</div>
+            </div>
+        </div>
+        <script>
+            const API_BASE = '/admin';
+            let ADMIN_TOKEN = localStorage.getItem('admin_token');
+            if (!ADMIN_TOKEN) {
+                ADMIN_TOKEN = prompt('Token Admin:');
+                if (ADMIN_TOKEN) {
+                    localStorage.setItem('admin_token', ADMIN_TOKEN);
+                }
+            }
+            const headers = {
+                'Authorization': `Bearer ${ADMIN_TOKEN}`,
+                'Content-Type': 'application/json'
+            };
+            async function apiCall(endpoint, method = 'GET') {
+                try {
+                    const response = await fetch(`${API_BASE}${endpoint}`, {
+                        method,
+                        headers
+                    });
+                    if (response.status === 403) {
+                        localStorage.removeItem('admin_token');
+                        location.reload();
+                        return;
+                    }
+                    if (!response.ok) {
+                        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                    }
+                    return await response.json();
+                } catch (error) {
+                    console.error('API Error:', error);
+                    throw error;
+                }
+            }
+            async function loadStatus() {
+                try {
+                    const status = await apiCall('/status');
+                    const statusDiv = document.getElementById('status-info');
+                    let statusClass = 'info';
+                    if (status.is_updating) statusClass = 'warning';
+                    else if (!status.hf_configured) statusClass = 'error';
+                    else if (status.last_update_result && status.last_update_result.success) statusClass = 'success';
+                    let lastUpdateInfo = 'Aucune';
+                    if (status.last_update_result) {
+                        const result = status.last_update_result;
+                        if (result.success) {
+                            lastUpdateInfo = `✅ ${result.count} films (${new Date(result.timestamp).toLocaleString()})`;
+                        } else {
+                            lastUpdateInfo = `❌ Erreur: ${result.error}`;
+                        }
+                    }
+                    statusDiv.innerHTML = `
+                        <div class="status ${statusClass}">
+                            <div class="config-grid">
+                                <div class="config-item">
+                                    <strong>Statut:</strong> ${status.is_updating ? '🔄 Mise à jour en cours...' : '✅ Prêt'}
+                                </div>
+                                <div class="config-item">
+                                    <strong>HF Configuré:</strong> ${status.hf_configured ? '✅ Oui' : '❌ Non'}
+                                </div>
+                                <div class="config-item">
+                                    <strong>Auto-update:</strong> ${status.auto_update_enabled ? '✅ Activé' : '❌ Désactivé'}
+                                </div>
+                                <div class="config-item">
+                                    <strong>Intervalle:</strong> ${status.update_interval_hours}h
+                                </div>
+                                <div class="config-item">
+                                    <strong>Taille batch:</strong> ${status.batch_size}
+                                </div>
+                                <div class="config-item">
+                                    <strong>Limite films:</strong> ${status.max_movies_limit}
+                                </div>
+                                <div class="config-item">
+                                    <strong>Logs:</strong> ${status.logs_count} entrées
+                                </div>
+                                <div class="config-item">
+                                    <strong>Dernière MAJ:</strong> ${lastUpdateInfo}
+                                </div>
+                            </div>
+                        </div>
+                    `;
+                } catch (error) {
+                    document.getElementById('status-info').innerHTML = `
+                        <div class="status error">❌ Erreur: ${error.message}</div>
+                    `;
+                }
+            }
+            async function updateVectors() {
+                const btn = document.getElementById('updateBtn');
+                const loading = document.getElementById('update-loading');
+                const result = document.getElementById('update-result');
+                btn.disabled = true;
+                loading.style.display = 'block';
+                result.innerHTML = '';
+                try {
+                    const response = await apiCall('/update-vectors', 'POST');
+                    result.innerHTML = `
+                        <div class="status ${response.success ? 'success' : 'warning'}">
+                            ${response.success ? '✅' : '⚠️'} ${response.message}
+                        </div>
+                    `;
+                    // Actualiser le statut après quelques secondes
+                    if (response.success) {
+                        setTimeout(loadStatus, 2000);
+                    }
+                } catch (error) {
+                    result.innerHTML = `
+                        <div class="status error">❌ Erreur: ${error.message}</div>
+                    `;
+                } finally {
+                    btn.disabled = false;
+                    loading.style.display = 'none';
+                }
+            }
+            async function forceUpdateVectors() {
+                if (!confirm('Êtes-vous sûr de vouloir forcer la mise à jour ? Cela peut prendre plusieurs minutes et consommer des crédits API.')) {
+                    return;
+                }
+                const btn = document.getElementById('forceUpdateBtn');
+                const loading = document.getElementById('update-loading');
+                const result = document.getElementById('update-result');
+                btn.disabled = true;
+                loading.style.display = 'block';
+                result.innerHTML = '';
+                try {
+                    const response = await apiCall('/force-update-vectors', 'POST');
+                    result.innerHTML = `
+                        <div class="status ${response.success ? 'success' : 'error'}">
+                            ${response.success ? '✅' : '❌'} ${response.message}
+                        </div>
+                    `;
+                    // Actualiser le statut après quelques secondes
+                    if (response.success) {
+                        setTimeout(loadStatus, 2000);
+                    }
+                } catch (error) {
+                    result.innerHTML = `
+                        <div class="status error">❌ Erreur: ${error.message}</div>
+                    `;
+                } finally {
+                    btn.disabled = false;
+                    loading.style.display = 'none';
+                }
+            }
+            async function loadLogs() {
+                try {
+                    const response = await apiCall('/logs');
+                    const logsDiv = document.getElementById('logs');
+                    if (response.logs && response.logs.length > 0) {
+                        logsDiv.innerHTML = response.logs.join('\\n');
+                        logsDiv.scrollTop = logsDiv.scrollHeight;
+                    } else {
+                        logsDiv.innerHTML = 'Aucun log disponible';
+                    }
+                } catch (error) {
+                    document.getElementById('logs').innerHTML = `Erreur: ${error.message}`;
+                }
+            }
+            function clearLogs() {
+                document.getElementById('logs').innerHTML = 'Logs effacés (rechargez pour voir les nouveaux logs)';
+            }
+            // Charger le statut au démarrage
+            if (ADMIN_TOKEN) {
+                loadStatus();
+                // Auto-refresh du statut toutes les 30 secondes
+                setInterval(loadStatus, 30000);
+            }
+        </script>
+    </body>
+    </html>
+    """
+    return HTMLResponse(content=html_content)
+@router.get("/status")
+async def get_status(token: str = Depends(verify_admin_token)):
+    """Obtenir le statut du système"""
+    return vector_updater.get_update_status()
+@router.post("/update-vectors")
+async def update_vectors(background_tasks: BackgroundTasks, token: str = Depends(verify_admin_token)):
+    """Déclencher une mise à jour si nécessaire"""
+    if vector_updater.is_updating:
+        return {"success": False, "message": "Une mise à jour est déjà en cours"}
+    # Lancer la mise à jour en arrière-plan
+    background_tasks.add_task(vector_updater.update_vectors_if_needed)
+    return {"success": True, "message": "Mise à jour programmée (vérification des conditions)"}
+@router.post("/force-update-vectors")
+async def force_update_vectors(background_tasks: BackgroundTasks, token: str = Depends(verify_admin_token)):
+    """Forcer la mise à jour des vecteurs"""
+    if vector_updater.is_updating:
+        return {"success": False, "message": "Une mise à jour est déjà en cours"}
+    # Lancer la mise à jour forcée en arrière-plan
+    background_tasks.add_task(vector_updater.force_update_vectors)
+    return {"success": True, "message": "Mise à jour forcée programmée"}
+@router.get("/logs")
+async def get_logs(token: str = Depends(verify_admin_token)):
+    """Obtenir les logs de mise à jour"""
+    try:
+        logs = vector_updater.get_logs()
+        return {"logs": logs}
+    except Exception as e:
+        return {"logs": [f"Erreur de lecture des logs: {e}"]}

app/services/__init__.py ADDED Viewed

	@@ -0,0 +1,3 @@

+"""
+Services package for Karl Movie Vector Backend
+"""

app/services/embedding_service.py ADDED Viewed

	@@ -0,0 +1,139 @@

+import asyncio
+import time
+import logging
+from typing import List, Optional, Dict
+import os
+from openai import OpenAI
+logger = logging.getLogger(__name__)
+class EmbeddingService:
+    """Service pour générer des embeddings avec OpenAI"""
+    def __init__(self):
+        self.client = OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
+        self.model_name = "text-embedding-3-small"
+    def create_composite_text(self, movie_data: Dict) -> str:
+        """Create composite text for embedding from movie data"""
+        parts = []
+        # Title
+        if movie_data.get('title'):
+            parts.append(f"Title: {movie_data['title']}")
+        # Tagline
+        if movie_data.get('tagline'):
+            parts.append(f"Tagline: {movie_data['tagline']}")
+        # Overview
+        if movie_data.get('overview'):
+            parts.append(f"Overview: {movie_data['overview']}")
+        # Release date
+        if movie_data.get('release_date'):
+            parts.append(f"Release Date: {movie_data['release_date']}")
+        # Original language
+        if movie_data.get('original_language'):
+            parts.append(f"Language: {movie_data['original_language']}")
+        # Spoken languages
+        if movie_data.get('spoken_languages'):
+            languages = [lang.get('iso_639_1', '') for lang in movie_data['spoken_languages'] if lang.get('iso_639_1')]
+            if languages:
+                parts.append(f"Spoken Languages: {', '.join(languages)}")
+        # Genres
+        if movie_data.get('genres'):
+            genres = [genre['name'] for genre in movie_data['genres']]
+            parts.append(f"Genres: {', '.join(genres)}")
+        # Production companies
+        if movie_data.get('production_companies'):
+            companies = [company['name'] for company in movie_data['production_companies']]
+            if companies:
+                parts.append(f"Production Companies: {', '.join(companies)}")
+        # Production countries
+        if movie_data.get('production_countries'):
+            countries = [country['name'] for country in movie_data['production_countries']]
+            if countries:
+                parts.append(f"Production Countries: {', '.join(countries)}")
+        # Budget (only if > 0)
+        if movie_data.get('budget') and movie_data['budget'] > 0:
+            parts.append(f"Budget: ${movie_data['budget']:,}")
+        # Popularity
+        if movie_data.get('popularity'):
+            parts.append(f"Popularity: {movie_data['popularity']}")
+        # Vote average
+        if movie_data.get('vote_average'):
+            parts.append(f"Vote Average: {movie_data['vote_average']}")
+        # Vote count
+        if movie_data.get('vote_count'):
+            parts.append(f"Vote Count: {movie_data['vote_count']}")
+        # Director(s)
+        if movie_data.get('credits', {}).get('crew'):
+            directors = [person['name'] for person in movie_data['credits']['crew'] if person['job'] == 'Director']
+            if directors:
+                parts.append(f"Director: {', '.join(directors)}")
+        # Top 5 cast
+        if movie_data.get('credits', {}).get('cast'):
+            top_cast = [person['name'] for person in movie_data['credits']['cast'][:5]]
+            if top_cast:
+                parts.append(f"Cast: {', '.join(top_cast)}")
+        return " / ".join(parts)
+    def get_embeddings_batch(self, texts: List[str], max_retries: int = 3) -> Optional[List[List[float]]]:
+        """Get embeddings for a batch of texts with retry"""
+        for attempt in range(max_retries):
+            try:
+                response = self.client.embeddings.create(
+                    model=self.model_name,
+                    input=texts
+                )
+                return [embedding.embedding for embedding in response.data]
+            except Exception as e:
+                logger.error(f"OpenAI API error (attempt {attempt + 1}): {e}")
+                if attempt < max_retries - 1:
+                    time.sleep(2 ** attempt)
+        return None
+    async def generate_batch_embeddings(self, movies: List[Dict], batch_size: int = 100) -> Optional[List[List[float]]]:
+        """Generate embeddings for a batch of movies"""
+        try:
+            # Create composite texts
+            texts = []
+            for movie in movies:
+                composite_text = self.create_composite_text(movie)
+                texts.append(composite_text)
+            # Generate embeddings in smaller batches to avoid API limits
+            all_embeddings = []
+            for i in range(0, len(texts), batch_size):
+                batch_texts = texts[i:i + batch_size]
+                logger.debug(f"Generating embeddings for batch {i//batch_size + 1}")
+                batch_embeddings = self.get_embeddings_batch(batch_texts)
+                if batch_embeddings is None:
+                    logger.error(f"Failed to generate embeddings for batch starting at {i}")
+                    return None
+                all_embeddings.extend(batch_embeddings)
+                # Small delay between batches to respect rate limits
+                await asyncio.sleep(0.1)
+            return all_embeddings
+        except Exception as e:
+            logger.error(f"Error generating batch embeddings: {e}")
+            return None

app/services/tmdb_service.py ADDED Viewed

	@@ -0,0 +1,88 @@

+import requests
+import time
+import logging
+from typing import List, Optional, Dict
+import os
+logger = logging.getLogger(__name__)
+class TMDBService:
+    """Service pour interagir avec l'API TMDB"""
+    def __init__(self):
+        self.api_key = os.getenv('TMDB_API_KEY')
+        self.base_url = "https://api.themoviedb.org/3"
+    def _make_request(self, endpoint: str, params: dict = None, max_retries: int = 3) -> Optional[dict]:
+        """Make API request with retry and backoff"""
+        if params is None:
+            params = {}
+        params['api_key'] = self.api_key
+        for attempt in range(max_retries):
+            try:
+                response = requests.get(f"{self.base_url}{endpoint}", params=params, timeout=10)
+                if response.status_code == 200:
+                    return response.json()
+                elif response.status_code == 429:
+                    # Rate limited, wait longer
+                    wait_time = 2 ** attempt
+                    logger.warning(f"Rate limited, waiting {wait_time}s before retry {attempt + 1}")
+                    time.sleep(wait_time)
+                    continue
+                else:
+                    logger.error(f"TMDB API returned status {response.status_code}")
+                    return None
+            except Exception as e:
+                logger.error(f"Request failed (attempt {attempt + 1}): {e}")
+                if attempt < max_retries - 1:
+                    time.sleep(2 ** attempt)
+        return None
+    async def get_popular_movies(self, limit: int = 10000) -> List[Dict]:
+        """Get popular movies from TMDB"""
+        movies = []
+        page = 1
+        filter_adult = os.getenv('FILTER_ADULT_CONTENT', 'true').lower() == 'true'
+        while len(movies) < limit:
+            logger.info(f"Fetching popular movies page {page}")
+            data = self._make_request("/movie/popular", {"page": page})
+            if not data or not data.get('results'):
+                break
+            for movie in data.get('results', []):
+                if len(movies) >= limit:
+                    break
+                # Skip adult movies if filtering is enabled
+                if filter_adult and movie.get('adult', False):
+                    continue
+                # Get detailed movie data
+                movie_details = self.get_movie_details(movie['id'])
+                if movie_details:
+                    # Get credits
+                    credits = self.get_movie_credits(movie['id'])
+                    if credits:
+                        movie_details['credits'] = credits
+                    movies.append(movie_details)
+            # Check if we've reached the last page
+            if page >= data.get('total_pages', 0):
+                break
+            page += 1
+            time.sleep(0.25)  # Rate limiting
+        logger.info(f"Collected {len(movies)} movies from TMDB")
+        return movies[:limit]
+    def get_movie_details(self, movie_id: int) -> Optional[dict]:
+        """Get detailed movie information"""
+        return self._make_request(f"/movie/{movie_id}")
+    def get_movie_credits(self, movie_id: int) -> Optional[dict]:
+        """Get movie cast and crew"""
+        return self._make_request(f"/movie/{movie_id}/credits")

app/services/vector_storage.py ADDED Viewed

	@@ -0,0 +1,153 @@

+from datasets import Dataset, load_dataset
+from huggingface_hub import HfApi, create_repo
+import numpy as np
+import json
+import logging
+from typing import Dict, List, Tuple, Optional
+import os
+from datetime import datetime
+logger = logging.getLogger(__name__)
+class HFVectorStorage:
+    def __init__(self):
+        self.hf_token = os.getenv('HF_TOKEN')
+        self.repo_name = os.getenv('HF_DATASET_REPO')
+        self.api = HfApi(token=self.hf_token)
+        # Créer le repo s'il n'existe pas
+        if self.hf_token and self.repo_name:
+            try:
+                create_repo(
+                    repo_id=self.repo_name,
+                    repo_type="dataset",
+                    token=self.hf_token,
+                    private=True,
+                    exist_ok=True
+                )
+            except Exception as e:
+                logger.warning(f"Repo creation warning: {e}")
+    def save_vectors(self, embeddings: np.ndarray, movies_data: List[Dict],
+                    id_map: Dict, metadata: Dict) -> bool:
+        """Sauvegarde les vecteurs sur HF Dataset Hub"""
+        try:
+            if not self.hf_token or not self.repo_name:
+                logger.error("HF_TOKEN or HF_DATASET_REPO not configured")
+                return False
+            # Préparer les données pour le dataset
+            dataset_dict = {
+                'movie_id': [movie['id'] for movie in movies_data],
+                'title': [movie['title'] for movie in movies_data],
+                'overview': [movie.get('overview', '') for movie in movies_data],
+                'genres': [movie.get('genres', []) for movie in movies_data],
+                'release_date': [movie.get('release_date', '') for movie in movies_data],
+                'embedding': embeddings.tolist(),
+                'tmdb_data': [json.dumps(movie) for movie in movies_data]
+            }
+            # Créer le dataset
+            dataset = Dataset.from_dict(dataset_dict)
+            # Upload vers HF Hub
+            dataset.push_to_hub(
+                self.repo_name,
+                token=self.hf_token,
+                commit_message=f"Update vectors - {datetime.now().isoformat()}"
+            )
+            # Sauvegarder les métadonnées
+            metadata_with_timestamp = {
+                **metadata,
+                'last_updated': datetime.now().isoformat(),
+                'total_movies': len(movies_data)
+            }
+            with open('temp_metadata.json', 'w') as f:
+                json.dump(metadata_with_timestamp, f, indent=2)
+            self.api.upload_file(
+                path_or_fileobj='temp_metadata.json',
+                path_in_repo='metadata.json',
+                repo_id=self.repo_name,
+                repo_type='dataset',
+                token=self.hf_token,
+                commit_message=f"Update metadata - {datetime.now().isoformat()}"
+            )
+            # Nettoyer le fichier temporaire
+            if os.path.exists('temp_metadata.json'):
+                os.remove('temp_metadata.json')
+            logger.info(f"Successfully saved {len(movies_data)} movie vectors to HF Hub")
+            return True
+        except Exception as e:
+            logger.error(f"Error saving vectors to HF Hub: {e}")
+            return False
+    def load_vectors(self) -> Optional[Tuple[np.ndarray, List[Dict], Dict, Dict]]:
+        """Charge les vecteurs depuis HF Dataset Hub"""
+        try:
+            if not self.hf_token or not self.repo_name:
+                logger.error("HF_TOKEN or HF_DATASET_REPO not configured")
+                return None
+            # Charger le dataset
+            dataset = load_dataset(self.repo_name, token=self.hf_token)['train']
+            # Extraire les données
+            embeddings = np.array(dataset['embedding'])
+            movies_data = []
+            id_map = {}
+            for i, movie_id in enumerate(dataset['movie_id']):
+                movie_data = json.loads(dataset['tmdb_data'][i])
+                movies_data.append(movie_data)
+                id_map[movie_id] = i
+            # Charger les métadonnées
+            try:
+                metadata_file = self.api.hf_hub_download(
+                    repo_id=self.repo_name,
+                    filename='metadata.json',
+                    repo_type='dataset',
+                    token=self.hf_token
+                )
+                with open(metadata_file, 'r') as f:
+                    metadata = json.load(f)
+            except:
+                metadata = {'last_updated': None}
+            logger.info(f"Successfully loaded {len(movies_data)} movie vectors from HF Hub")
+            return embeddings, movies_data, id_map, metadata
+        except Exception as e:
+            logger.error(f"Error loading vectors from HF Hub: {e}")
+            return None
+    def check_update_needed(self) -> bool:
+        """Vérifie si une mise à jour est nécessaire"""
+        try:
+            update_interval = int(os.getenv('UPDATE_INTERVAL_HOURS', 24))
+            # Charger les métadonnées actuelles
+            result = self.load_vectors()
+            if not result:
+                return True
+            _, _, _, metadata = result
+            if not metadata.get('last_updated'):
+                return True
+            last_update = datetime.fromisoformat(metadata['last_updated'])
+            hours_since_update = (datetime.now() - last_update).total_seconds() / 3600
+            return hours_since_update >= update_interval
+        except Exception as e:
+            logger.error(f"Error checking update status: {e}")
+            return True

app/services/vector_updater.py ADDED Viewed

	@@ -0,0 +1,170 @@

+import asyncio
+import logging
+from datetime import datetime
+from typing import Optional, List
+import os
+import numpy as np
+from .vector_storage import HFVectorStorage
+from .tmdb_service import TMDBService
+from .embedding_service import EmbeddingService
+logger = logging.getLogger(__name__)
+class VectorUpdater:
+    def __init__(self):
+        self.storage = HFVectorStorage()
+        self.tmdb_service = TMDBService()
+        self.embedding_service = EmbeddingService()
+        self.is_updating = False
+        self.last_update_result = None
+        self.update_logs = []
+    def add_log(self, message: str, level: str = "INFO"):
+        """Ajouter un log avec timestamp"""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        log_entry = f"[{timestamp}] {level}: {message}"
+        self.update_logs.append(log_entry)
+        # Garder seulement les 100 derniers logs
+        if len(self.update_logs) > 100:
+            self.update_logs = self.update_logs[-100:]
+        # Log également dans le système de logging
+        log_level = getattr(logging, level.upper(), logging.INFO)
+        logger.log(log_level, message)
+    async def update_vectors_if_needed(self) -> bool:
+        """Met à jour les vecteurs si nécessaire"""
+        if self.is_updating:
+            self.add_log("Update already in progress, skipping...", "WARNING")
+            return False
+        if not self.storage.check_update_needed():
+            self.add_log("Vectors are up to date, no update needed", "INFO")
+            return False
+        if not os.getenv('AUTO_UPDATE_VECTORS', 'false').lower() == 'true':
+            self.add_log("Auto update disabled", "INFO")
+            return False
+        self.add_log("Auto update conditions met, starting update...", "INFO")
+        return await self.force_update_vectors()
+    async def force_update_vectors(self) -> bool:
+        """Force la mise à jour des vecteurs"""
+        if self.is_updating:
+            self.add_log("Update already in progress", "WARNING")
+            return False
+        self.is_updating = True
+        self.add_log("Starting vector update process...", "INFO")
+        try:
+            # Paramètres de configuration
+            batch_size = int(os.getenv('BATCH_SIZE', 100))
+            max_movies = int(os.getenv('MAX_MOVIES_LIMIT', 10000))
+            # Récupérer les films populaires depuis TMDB
+            self.add_log("Fetching movies from TMDB...", "INFO")
+            movies = await self.tmdb_service.get_popular_movies(limit=max_movies)
+            if not movies:
+                self.add_log("No movies fetched from TMDB", "ERROR")
+                self.last_update_result = {"success": False, "error": "No movies fetched"}
+                return False
+            self.add_log(f"Processing {len(movies)} movies in batches of {batch_size}", "INFO")
+            all_embeddings = []
+            processed_movies = []
+            id_map = {}
+            # Traiter par batches pour éviter les timeouts
+            for i in range(0, len(movies), batch_size):
+                batch = movies[i:i + batch_size]
+                batch_num = i//batch_size + 1
+                total_batches = (len(movies)-1)//batch_size + 1
+                self.add_log(f"Processing batch {batch_num}/{total_batches}", "INFO")
+                # Générer les embeddings pour le batch
+                batch_embeddings = await self.embedding_service.generate_batch_embeddings(batch)
+                if batch_embeddings is not None:
+                    all_embeddings.extend(batch_embeddings)
+                    for j, movie in enumerate(batch):
+                        processed_movies.append(movie)
+                        id_map[movie['id']] = len(processed_movies) - 1
+                else:
+                    self.add_log(f"Failed to generate embeddings for batch {batch_num}", "WARNING")
+                # Pause entre les batches pour éviter le rate limiting
+                await asyncio.sleep(1)
+            if not all_embeddings:
+                self.add_log("No embeddings generated", "ERROR")
+                self.last_update_result = {"success": False, "error": "No embeddings generated"}
+                return False
+            # Convertir en numpy array
+            embeddings_array = np.array(all_embeddings)
+            self.add_log(f"Generated {len(all_embeddings)} embeddings", "INFO")
+            # Sauvegarder sur HF Hub
+            metadata = {
+                'update_timestamp': datetime.now().isoformat(),
+                'total_movies': len(processed_movies),
+                'embedding_model': getattr(self.embedding_service, 'model_name', 'unknown'),
+                'tmdb_api_version': '3',
+                'batch_size': batch_size,
+                'max_movies_limit': max_movies
+            }
+            self.add_log("Saving vectors to HF Hub...", "INFO")
+            success = self.storage.save_vectors(
+                embeddings_array,
+                processed_movies,
+                id_map,
+                metadata
+            )
+            if success:
+                self.add_log(f"Successfully updated {len(processed_movies)} movie vectors", "INFO")
+                self.last_update_result = {
+                    "success": True,
+                    "count": len(processed_movies),
+                    "timestamp": datetime.now().isoformat()
+                }
+                return True
+            else:
+                self.add_log("Failed to save vectors to storage", "ERROR")
+                self.last_update_result = {"success": False, "error": "Failed to save to storage"}
+                return False
+        except Exception as e:
+            self.add_log(f"Error during vector update: {e}", "ERROR")
+            self.last_update_result = {"success": False, "error": str(e)}
+            return False
+        finally:
+            self.is_updating = False
+            self.add_log("Vector update process completed", "INFO")
+    def get_update_status(self) -> dict:
+        """Retourne le statut de la mise à jour"""
+        return {
+            'is_updating': self.is_updating,
+            'auto_update_enabled': os.getenv('AUTO_UPDATE_VECTORS', 'false').lower() == 'true',
+            'update_interval_hours': int(os.getenv('UPDATE_INTERVAL_HOURS', 24)),
+            'batch_size': int(os.getenv('BATCH_SIZE', 100)),
+            'max_movies_limit': int(os.getenv('MAX_MOVIES_LIMIT', 10000)),
+            'last_update_result': self.last_update_result,
+            'logs_count': len(self.update_logs),
+            'hf_configured': bool(os.getenv('HF_TOKEN') and os.getenv('HF_DATASET_REPO'))
+        }
+    def get_logs(self) -> List[str]:
+        """Retourne les logs de mise à jour"""
+        return self.update_logs.copy()

requirements.txt CHANGED Viewed

@@ -1,12 +1,10 @@
 fastapi==0.104.1
-uvicorn[standard]==0.24.0
-numpy==1.24.4
 faiss-cpu==1.7.4
-openai==1.51.0
-pydantic==2.11.5
-pydantic-settings==2.9.1
-python-multipart==0.0.6
-requests==2.31.0
-scikit-learn==1.3.2
 python-dotenv==1.0.0
-httpx==0.27.0

 fastapi==0.104.1
+uvicorn==0.24.0
+pydantic==2.5.0
+numpy==1.24.3
 faiss-cpu==1.7.4
+openai==1.3.5
 python-dotenv==1.0.0
+requests==2.31.0
+datasets>=2.14.0
+huggingface_hub>=0.17.0