feat: integrate 2 modern models for classification and update routes and examples

Dockerfile

@@ -10,6 +10,5 @@ COPY --chown=user ./requirements.txt requirements.txt
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
 RUN python -m nltk.downloader stopwords
 
-
 COPY --chown=user . /app
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -6,7 +6,7 @@ colorTo: green
 sdk: docker
 pinned: false
 license: mit
-short_description: 'Arabic text summarization and topic classification app. '
+short_description: 'Arabic text summarization and topic classification app.'
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -3,9 +3,9 @@ from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 from enum import Enum
 
-from classifier import ArabicClassifier
 from summarizer import ArabicSummarizer
 from preprocessor import ArabicPreprocessor
+from model_manager import ModelManager
 from examples import REQUEST_EXAMPLES, RESPONSE_EXAMPLES
 
 
@@ -14,56 +14,55 @@ class TaskType(str, Enum):
     SUMMARIZATION = "summarization"
 
 
+class ModelType(str, Enum):
+    TRADITIONAL_SVM = "traditional_svm"
+    MODERN_BERT = "modern_bert"
+    MODERN_LSTM = "modern_lstm"
+
+
 app = FastAPI(
     title="Arabic Text Analysis API",
-    description="API for Arabic text classification, summarization, and preprocessing",
-    version="1.0.0"
+    description="API for Arabic text classification, summarization, and preprocessing with multiple model support",
+    version="1.0.0",
 )
 
-classifier = ArabicClassifier("svm_classifier.joblib", "tfidf_vectorizer_classifier.joblib")
-summarizer = ArabicSummarizer("tfidf_vectorizer_text_summarization.joblib")
+model_manager = ModelManager(default_model="traditional_svm")
+summarizer = ArabicSummarizer("traditional_tfidf_vectorizer_summarization.joblib")
 preprocessor = ArabicPreprocessor()
 
 
 class TextInput(BaseModel):
     text: str
-    
-    model_config = {
-        "json_schema_extra": {
-            "example": REQUEST_EXAMPLES["text_input"]
-        }
-    }
+    model: Optional[ModelType] = None
+
+    model_config = {"json_schema_extra": {"example": REQUEST_EXAMPLES["text_input"]}}
 
 
 class TextInputWithSentences(BaseModel):
     text: str
     num_sentences: Optional[int] = 3
-    
+    model: Optional[ModelType] = None
+
     model_config = {
-        "json_schema_extra": {
-            "example": REQUEST_EXAMPLES["text_input_with_sentences"]
-        }
+        "json_schema_extra": {"example": REQUEST_EXAMPLES["text_input_with_sentences"]}
     }
 
 
 class BatchTextInput(BaseModel):
     texts: List[str]
-    
+    model: Optional[ModelType] = None
+
     model_config = {
-        "json_schema_extra": {
-            "example": REQUEST_EXAMPLES["batch_text_input"]
-        }
+        "json_schema_extra": {"example": REQUEST_EXAMPLES["batch_text_input"]}
     }
 
 
 class PreprocessingInput(BaseModel):
     text: str
     task_type: TaskType = TaskType.CLASSIFICATION
-    
+
     model_config = {
-        "json_schema_extra": {
-            "example": REQUEST_EXAMPLES["preprocessing_input"]
-        }
+        "json_schema_extra": {"example": REQUEST_EXAMPLES["preprocessing_input"]}
     }
 
 
@@ -86,8 +85,8 @@ class ClassificationResponse(BaseModel):
                         "description": "Numerical index of the predicted class (0=culture, 1=economy, 2=international, 3=local, 4=religion, 5=sports)"
                     }
                 }
-            }
-        }
+            },
+        },
     }
 
 
@@ -101,9 +100,7 @@ class SummarizationResponse(BaseModel):
     top_sentence_scores: Optional[List[float]]
 
     model_config = {
-        "json_schema_extra": {
-            "example": RESPONSE_EXAMPLES["summarization"]
-        }
+        "json_schema_extra": {"example": RESPONSE_EXAMPLES["summarization"]}
     }
 
 
@@ -112,9 +109,7 @@ class TextAnalysisResponse(BaseModel):
     analysis: Dict[str, Any]
 
     model_config = {
-        "json_schema_extra": {
-            "example": RESPONSE_EXAMPLES["text_analysis"]
-        }
+        "json_schema_extra": {"example": RESPONSE_EXAMPLES["text_analysis"]}
     }
 
 
@@ -125,9 +120,7 @@ class BatchClassificationResponse(BaseModel):
 
     model_config = {
         "protected_namespaces": (),
-        "json_schema_extra": {
-            "example": RESPONSE_EXAMPLES["batch_classification"]
-        }
+        "json_schema_extra": {"example": RESPONSE_EXAMPLES["batch_classification"]},
     }
 
 
@@ -137,9 +130,7 @@ class SentenceAnalysisResponse(BaseModel):
     score_statistics: Dict[str, float]
 
     model_config = {
-        "json_schema_extra": {
-            "example": RESPONSE_EXAMPLES["sentence_analysis"]
-        }
+        "json_schema_extra": {"example": RESPONSE_EXAMPLES["sentence_analysis"]}
     }
 
 
@@ -150,9 +141,7 @@ class CompleteAnalysisResponse(BaseModel):
     summarization: SummarizationResponse
 
     model_config = {
-        "json_schema_extra": {
-            "example": RESPONSE_EXAMPLES["complete_analysis"]
-        }
+        "json_schema_extra": {"example": RESPONSE_EXAMPLES["complete_analysis"]}
     }
 
 
@@ -161,9 +150,7 @@ class PreprocessingResponse(BaseModel):
     preprocessing_steps: Dict[str, Any]
 
     model_config = {
-        "json_schema_extra": {
-            "example": RESPONSE_EXAMPLES["preprocessing"]
-        }
+        "json_schema_extra": {"example": RESPONSE_EXAMPLES["preprocessing"]}
     }
 
 
@@ -171,11 +158,25 @@ class ModelInfoResponse(BaseModel):
     classifier: Dict[str, Any]
     summarizer: Dict[str, Any]
 
-    model_config = {
-        "json_schema_extra": {
-            "example": RESPONSE_EXAMPLES["model_info"]
-        }
-    }
+    model_config = {"json_schema_extra": {"example": RESPONSE_EXAMPLES["model_info"]}}
+
+
+class ModelSwitchInput(BaseModel):
+    model: ModelType
+
+    model_config = {"json_schema_extra": {"example": {"model": "modern_bert"}}}
+
+
+class ModelSwitchResponse(BaseModel):
+    success: bool
+    message: str
+    previous_model: Optional[str] = None
+    current_model: str
+
+
+class AvailableModelsResponse(BaseModel):
+    models: Dict[str, Any]
+    current_model: str
 
 
 @app.get("/")
@@ -186,7 +187,7 @@ def read_root() -> Dict[str, Any]:
         "documentation": {
             "interactive_docs": "/docs",
             "redoc": "/redoc",
-            "openapi_schema": "/openapi.json"
+            "openapi_schema": "/openapi.json",
         },
         "endpoints": {
             "classify": "POST /classify - Classify Arabic text",
@@ -196,8 +197,9 @@ def read_root() -> Dict[str, Any]:
             "preprocess": "POST /preprocess - Preprocess text with detailed steps",
             "text_analysis": "POST /text-analysis - Analyze text characteristics",
             "sentence_analysis": "POST /sentence-analysis - Detailed sentence analysis",
-            "model_info": "GET /model-info - Get model information"
-        }
+            "model_info": "GET /model-info - Get model information",
+            "available_models": "GET /models - Get all available models",
+        },
     }
 
 
@@ -205,7 +207,8 @@ def read_root() -> Dict[str, Any]:
 def classify_text(data: TextInput) -> ClassificationResponse:
     """Classify Arabic text with probability distribution and metadata."""
     try:
-        result = classifier.predict(data.text)
+        model_name = data.model.value if data.model else None
+        result = model_manager.predict(data.text, model_name)
         return result
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Classification failed: {str(e)}")
@@ -215,14 +218,19 @@ def classify_text(data: TextInput) -> ClassificationResponse:
 def classify_texts(data: BatchTextInput) -> BatchClassificationResponse:
     """Classify multiple Arabic texts in batch."""
     try:
-        results = classifier.predict_batch(data.texts)
+        model_name = data.model.value if data.model else None
+        results = model_manager.predict_batch(data.texts, model_name)
+        used_model = model_name or model_manager.default_model
+        
         return {
             "results": results,
             "total_texts": len(data.texts),
-            "model_used": classifier.model_name
+            "model_used": used_model,
         }
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Batch classification failed: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Batch classification failed: {str(e)}"
+        )
 
 
 @app.post("/summarize", response_model=SummarizationResponse)
@@ -242,25 +250,31 @@ def analyze_sentences(data: TextInput) -> SentenceAnalysisResponse:
         result = summarizer.get_sentence_analysis(data.text)
         return result
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Sentence analysis failed: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Sentence analysis failed: {str(e)}"
+        )
 
 
 @app.post("/analyze", response_model=CompleteAnalysisResponse)
 def analyze_text_complete(data: TextInputWithSentences) -> CompleteAnalysisResponse:
     """Complete analysis: classification, summarization, and text statistics."""
     try:
-        classification_result = classifier.predict(data.text)
+        model_name = data.model.value if data.model else None
+        
+        classification_result = model_manager.predict(data.text, model_name)
         summarization_result = summarizer.summarize(data.text, data.num_sentences)
         text_stats = preprocessor.analyze_text(data.text)
-        
+
         return {
             "original_text": data.text,
             "text_analysis": text_stats,
             "classification": classification_result,
-            "summarization": summarization_result
+            "summarization": summarization_result,
         }
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Complete analysis failed: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Complete analysis failed: {str(e)}"
+        )
 
 
 @app.post("/preprocess", response_model=PreprocessingResponse)
@@ -268,10 +282,7 @@ def preprocess_text(data: PreprocessingInput) -> PreprocessingResponse:
     """Preprocess text with step-by-step breakdown."""
     try:
         steps = preprocessor.get_preprocessing_steps(data.text, data.task_type.value)
-        return {
-            "task_type": data.task_type.value,
-            "preprocessing_steps": steps
-        }
+        return {"task_type": data.task_type.value, "preprocessing_steps": steps}
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Preprocessing failed: {str(e)}")
 
@@ -281,25 +292,60 @@ def analyze_text_characteristics(data: TextInput) -> TextAnalysisResponse:
     """Analyze text characteristics and statistics."""
     try:
         analysis = preprocessor.analyze_text(data.text)
-        return {
-            "text": data.text,
-            "analysis": analysis
-        }
+        return {"text": data.text, "analysis": analysis}
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Text analysis failed: {str(e)}")
 
 
 @app.get("/model-info", response_model=ModelInfoResponse)
-def get_model_info() -> ModelInfoResponse:
-    """Get information about loaded models."""
+def get_model_info(model: Optional[ModelType] = None) -> ModelInfoResponse:
+    """Get information about a specific model or the default model."""
     try:
-        classifier_info = classifier.get_model_info()
+        model_name = model.value if model else None
+        classifier_info = model_manager.get_model_info(model_name)
         return {
             "classifier": classifier_info,
             "summarizer": {
-                "vectorizer_loaded": hasattr(summarizer, 'vectorizer'),
-                "model_type": "TF-IDF based summarization"
-            }
+                "vectorizer_loaded": hasattr(summarizer, "vectorizer"),
+                "model_type": "TF-IDF based summarization",
+            },
         }
     except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Failed to get model info: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to get model info: {str(e)}"
+        )
+
+
+@app.get("/models", response_model=AvailableModelsResponse)
+def get_available_models() -> AvailableModelsResponse:
+    """Get all available classification models."""
+    try:
+        models = model_manager.get_available_models()
+        return {"models": models, "current_model": model_manager.default_model}
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Failed to get available models: {str(e)}"
+        )
+
+
+@app.get("/models/cache")
+def get_cache_status() -> Dict[str, Any]:
+    """Get information about cached models."""
+    try:
+        return model_manager.get_cache_status()
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Failed to get cache status: {str(e)}"
+        )
+
+
+@app.post("/models/cache/clear")
+def clear_model_cache(model: Optional[ModelType] = None) -> Dict[str, Any]:
+    """Clear model cache for a specific model or all models."""
+    try:
+        model_name = model.value if model else None
+        return model_manager.clear_cache(model_name)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500, detail=f"Failed to clear cache: {str(e)}"
+        )

examples.py

@@ -1,10 +1,19 @@
+
 """API request and response examples for documentation."""
 
 EXAMPLE_TEXT = "يكون سعر الفاكهة والخضراوات في موسم إنباتها أقل من غيره من المواسم. ستلجأ محلات الخضروات إلى عرض الفاكهة بأسعار مناسبة في موسمها بسبب توفر المنتجات. لا يقتصر الأمر على السعر الأقل، بل سيكون طعامك أشهى وألذ عند تناوله في موسمه. في فصل الخريف يتوفر التفاح والتين والبنجر والكمثرى."
 
+# Note: For classification endpoints, the 'model' field is optional.
+# Available models: traditional_svm, modern_bert, modern_lstm
+# If not specified, defaults to traditional_svm
+
 REQUEST_EXAMPLES = {
     "text_input": {"text": EXAMPLE_TEXT},
+    "text_input_with_model": {"text": EXAMPLE_TEXT, "model": "traditional_svm"},
+    "text_input_modern_bert": {"text": EXAMPLE_TEXT, "model": "modern_bert"},
+    "text_input_modern_lstm": {"text": EXAMPLE_TEXT, "model": "modern_lstm"},
     "text_input_with_sentences": {"text": EXAMPLE_TEXT, "num_sentences": 2},
+    "text_input_with_sentences_and_model": {"text": EXAMPLE_TEXT, "num_sentences": 2, "model": "modern_bert"},
     "batch_text_input": {
         "texts": [
             EXAMPLE_TEXT,
@@ -12,30 +21,38 @@ REQUEST_EXAMPLES = {
             "المطاعم في المدينة تقدم أطباق شهية ومتنوعة.",
         ]
     },
+    "batch_text_input_with_model": {
+        "texts": [
+            EXAMPLE_TEXT,
+            "هذا نص تجريبي آخر للتصنيف باللغة العربية.",
+            "المطاعم في المدينة تقدم أطباق شهية ومتنوعة.",
+        ],
+        "model": "modern_lstm"
+    },
     "preprocessing_input": {"text": EXAMPLE_TEXT, "task_type": "classification"},
 }
 
 RESPONSE_EXAMPLES = {
     "classification": {
-        "prediction": "culture",
-        "prediction_index": 0,
-        "confidence": 0.902,
+        "prediction": "economy",
+        "prediction_index": 1,
+        "confidence": 0.851430067618738,
         "probability_distribution": {
-            "culture": 0.902,
-            "economy": 0.001,
-            "international": 0.0,
-            "local": 0.061,
-            "religion": 0.0,
-            "sports": 0.036,
+            "culture": 0.03446028829161909,
+            "economy": 0.851430067618738,
+            "international": 0.005894582125851457,
+            "local": 0.07395284130576442,
+            "religion": 0.006103062287269104,
+            "sports": 0.028159158370758375
         },
-        "cleaned_text": "يكن سعر فاكه خضرو موسم انبات اقل غير موسم",
-        "model_used": "svm_classifier",
+        "cleaned_text": "يكون سعر فكه خضراو وسم بات اقل غير وسم تلج محل خضرو الي عرض فكه سعر نسب وسم بسب وفر نتج قصر امر علي سعر اقل سيك طعم اشه ولذ نول وسم فصل خرف وفر تفح تين نجر كمثر",
+        "model_used": "traditional_svm_classifier",
         "prediction_metadata": {
-            "max_probability": 0.902,
-            "min_probability": 0.0,
-            "entropy": 0.393,
-            "num_classes": 6,
-        },
+            "max_probability": 0.851430067618738,
+            "min_probability": 0.005894582125851457,
+            "entropy": 0.607505444397693,
+            "num_classes": 6
+        }
     },
     "summarization": {
         "summary": "ستلجأ محلات الخضروات إلى عرض الفاكهة بأسعار مناسبة في موسمها بسبب توفر المنتجات لا يقتصر الأمر على السعر الأقل، بل سيكون طعامك أشهى وألذ عند تناوله في موسمه",
@@ -45,116 +62,163 @@ RESPONSE_EXAMPLES = {
             "يكون سعر الفاكهة والخضراوات في موسم إنباتها أقل من غيره من المواسم",
             "ستلجأ محلات الخضروات إلى عرض الفاكهة بأسعار مناسبة في موسمها بسبب توفر المنتجات",
             "لا يقتصر الأمر على السعر الأقل، بل سيكون طعامك أشهى وألذ عند تناوله في موسمه",
-            "في فصل الخريف يتوفر التفاح والتين والبنجر والكمثرى",
+            "في فصل الخريف يتوفر التفاح والتين والبنجر والكمثرى"
         ],
         "selected_indices": [1, 2],
-        "sentence_scores": [2.968, 3.224, 3.234, 2.642],
-        "top_sentence_scores": [3.224, 3.234],
+        "sentence_scores": [2.9675719816913095, 3.2243029195650847, 3.2339935441416645, 2.6416207226871506],
+        "top_sentence_scores": [3.2243029195650847, 3.2339935441416645]
     },
     "text_analysis": {
         "text": EXAMPLE_TEXT,
         "analysis": {
-            "character_count": 282,
-            "word_count": 46,
+            "character_count": 278,
+            "word_count": 48,
             "sentence_count": 4,
-            "arabic_character_count": 252,
-            "arabic_character_ratio": 0.8936,
-            "average_word_length": 5.48,
-            "average_sentence_length": 11.5,
+            "arabic_character_count": 227,
+            "arabic_character_ratio": 0.8165467625899281,
+            "average_word_length": 4.8125,
+            "average_sentence_length": 12.0,
             "has_diacritics": False,
-            "punctuation_count": 3,
-        },
+            "punctuation_count": 5
+        }
     },
     "batch_classification": {
         "results": [
             {
-                "prediction": "culture",
-                "prediction_index": 0,
-                "confidence": 0.902,
+                "prediction": "economy",
+                "prediction_index": 1,
+                "confidence": 0.851430067618738,
                 "probability_distribution": {
-                    "culture": 0.902,
-                    "economy": 0.001,
-                    "international": 0.0,
-                    "local": 0.061,
-                    "religion": 0.0,
-                    "sports": 0.036,
+                    "culture": 0.03446028829161909,
+                    "economy": 0.851430067618738,
+                    "international": 0.005894582125851457,
+                    "local": 0.07395284130576442,
+                    "religion": 0.006103062287269104,
+                    "sports": 0.028159158370758375
                 },
-                "cleaned_text": "يكن سعر فاكه خضرو موسم انبات اقل غير موسم",
-                "model_used": "svm_classifier",
+                "cleaned_text": "يكون سعر فكه خضراو وسم بات اقل غير وسم تلج محل خضرو الي عرض فكه سعر نسب وسم بسب وفر نتج قصر امر علي سعر اقل سيك طعم اشه ولذ نول وسم فصل خرف وفر تفح تين نجر كمثر",
+                "model_used": "traditional_svm_classifier",
                 "prediction_metadata": {
-                    "max_probability": 0.902,
-                    "min_probability": 0.0,
-                    "entropy": 0.393,
-                    "num_classes": 6,
+                    "max_probability": 0.851430067618738,
+                    "min_probability": 0.005894582125851457,
+                    "entropy": 0.607505444397693,
+                    "num_classes": 6
+                }
+            },
+            {
+                "prediction": "economy",
+                "prediction_index": 1,
+                "confidence": 0.594920291892315,
+                "probability_distribution": {
+                    "culture": 0.2745678397668158,
+                    "economy": 0.594920291892315,
+                    "international": 0.026763844094989445,
+                    "local": 0.059508468104556384,
+                    "religion": 0.010869503926651195,
+                    "sports": 0.033370152114672154
+                },
+                "cleaned_text": "نص تجرب اخر تصن لغ عرب",
+                "model_used": "traditional_svm_classifier",
+                "prediction_metadata": {
+                    "max_probability": 0.594920291892315,
+                    "min_probability": 0.010869503926651195,
+                    "entropy": 1.2244037883788486,
+                    "num_classes": 6
+                }
+            },
+            {
+                "prediction": "local",
+                "prediction_index": 3,
+                "confidence": 0.8469012893956159,
+                "probability_distribution": {
+                    "culture": 0.03610036096593097,
+                    "economy": 0.02966970458779607,
+                    "international": 0.00998550970524301,
+                    "local": 0.8469012893956159,
+                    "religion": 0.025074648364464147,
+                    "sports": 0.05237313697694993
                 },
+                "cleaned_text": "طعم دين قدم طب شه تنع",
+                "model_used": "traditional_svm_classifier",
+                "prediction_metadata": {
+                    "max_probability": 0.8469012893956159,
+                    "min_probability": 0.00998550970524301,
+                    "entropy": 0.5892773509761756,
+                    "num_classes": 6
+                }
             }
         ],
         "total_texts": 3,
-        "model_used": "svm_classifier",
+        "model_used": "traditional_svm"
     },
     "sentence_analysis": {
         "sentences": [
             {
                 "index": 0,
                 "sentence": "يكون سعر الفاكهة والخضراوات في موسم إنباتها أقل من غيره من المواسم",
-                "score": 2.968,
-                "rank": 3,
+                "score": 2.9675719816913095,
+                "rank": 3
             },
             {
                 "index": 1,
                 "sentence": "ستلجأ محلات الخضروات إلى عرض الفاكهة بأسعار مناسبة في موسمها بسبب توفر المنتجات",
-                "score": 3.224,
-                "rank": 2,
+                "score": 3.2243029195650847,
+                "rank": 2
             },
             {
                 "index": 2,
                 "sentence": "لا يقتصر الأمر على السعر الأقل، بل سيكون طعامك أشهى وألذ عند تناوله في موسمه",
-                "score": 3.234,
-                "rank": 1,
+                "score": 3.2339935441416645,
+                "rank": 1
             },
             {
                 "index": 3,
                 "sentence": "في فصل الخريف يتوفر التفاح والتين والبنجر والكمثرى",
-                "score": 2.642,
-                "rank": 4,
-            },
+                "score": 2.6416207226871506,
+                "rank": 4
+            }
         ],
         "total_sentences": 4,
-        "score_statistics": {"mean": 3.017, "std": 0.254, "min": 2.642, "max": 3.234},
+        "score_statistics": {
+            "mean": 3.0168722920213025,
+            "std": 0.24156436215421892,
+            "min": 2.6416207226871506,
+            "max": 3.2339935441416645
+        }
     },
     "complete_analysis": {
         "original_text": EXAMPLE_TEXT,
         "text_analysis": {
-            "character_count": 282,
-            "word_count": 46,
+            "character_count": 278,
+            "word_count": 48,
             "sentence_count": 4,
-            "arabic_character_count": 252,
-            "arabic_character_ratio": 0.8936,
-            "average_word_length": 5.48,
-            "average_sentence_length": 11.5,
+            "arabic_character_count": 227,
+            "arabic_character_ratio": 0.8165467625899281,
+            "average_word_length": 4.8125,
+            "average_sentence_length": 12.0,
             "has_diacritics": False,
-            "punctuation_count": 3,
+            "punctuation_count": 5
         },
         "classification": {
-            "prediction": "culture",
-            "prediction_index": 0,
-            "confidence": 0.902,
+            "prediction": "economy",
+            "prediction_index": 1,
+            "confidence": 0.851430067618738,
             "probability_distribution": {
-                "culture": 0.902,
-                "economy": 0.001,
-                "international": 0.0,
-                "local": 0.061,
-                "religion": 0.0,
-                "sports": 0.036,
+                "culture": 0.03446028829161909,
+                "economy": 0.851430067618738,
+                "international": 0.005894582125851457,
+                "local": 0.07395284130576442,
+                "religion": 0.006103062287269104,
+                "sports": 0.028159158370758375
             },
-            "cleaned_text": "يكن سعر فاكه خضرو موسم انبات اقل غير موسم",
-            "model_used": "svm_classifier",
+            "cleaned_text": "يكون سعر فكه خضراو وسم بات اقل غير وسم تلج محل خضرو الي عرض فكه سعر نسب وسم بسب وفر نتج قصر امر علي سعر اقل سيك طعم اشه ولذ نول وسم فصل خرف وفر تفح تين نجر كمثر",
+            "model_used": "traditional_svm_classifier",
             "prediction_metadata": {
-                "max_probability": 0.902,
-                "min_probability": 0.0,
-                "entropy": 0.393,
-                "num_classes": 6,
-            },
+                "max_probability": 0.851430067618738,
+                "min_probability": 0.005894582125851457,
+                "entropy": 0.607505444397693,
+                "num_classes": 6
+            }
         },
         "summarization": {
             "summary": "ستلجأ محلات الخضروات إلى عرض الفاكهة بأسعار مناسبة في موسمها بسبب توفر المنتجات لا يقتصر الأمر على السعر الأقل، بل سيكون طعامك أشهى وألذ عند تناوله في موسمه",
@@ -164,51 +228,86 @@ RESPONSE_EXAMPLES = {
                 "يكون سعر الفاكهة والخضراوات في موسم إنباتها أقل من غيره من المواسم",
                 "ستلجأ محلات الخضروات إلى عرض الفاكهة بأسعار مناسبة في موسمها بسبب توفر المنتجات",
                 "لا يقتصر الأمر على السعر الأقل، بل سيكون طعامك أشهى وألذ عند تناوله في موسمه",
-                "في فصل الخريف يتوفر التفاح والتين والبنجر والكمثرى",
+                "في فصل الخريف يتوفر التفاح والتين والبنجر والكمثرى"
             ],
             "selected_indices": [1, 2],
-            "sentence_scores": [2.968, 3.224, 3.234, 2.642],
-            "top_sentence_scores": [3.224, 3.234],
-        },
+            "sentence_scores": [2.9675719816913095, 3.2243029195650847, 3.2339935441416645, 2.6416207226871506],
+            "top_sentence_scores": [3.2243029195650847, 3.2339935441416645]
+        }
     },
     "preprocessing": {
         "task_type": "classification",
         "preprocessing_steps": {
-            "original_text": EXAMPLE_TEXT,
-            "step_1_remove_diacritics": "يكون سعر الفاكهة والخضراوات في موسم انباتها اقل من غيره من المواسم",
-            "step_2_remove_punctuation": "يكون سعر الفاكهة والخضراوات في موسم انباتها اقل من غيره من المواسم",
-            "step_3_normalize_text": "يكون سعر الفاكهة والخضراوات في موسم انباتها اقل من غيره من المواسم",
-            "step_4_remove_stopwords": "سعر فاكهة خضراوات موسم انباتها اقل غيره مواسم",
-            "step_5_stem_words": "سعر فاكه خضرو موسم انبات اقل غير موسم",
-            "final_result": "سعر فاكه خضرو موسم انبات اقل غير موسم",
+            "original": EXAMPLE_TEXT,
+            "stripped_lowered": EXAMPLE_TEXT,
+            "normalized": "يكون سعر الفاكهه والخضراوات في موسم انباتها اقل من غيره من المواسم. ستلجا محلات الخضروات الي عرض الفاكهه باسعار مناسبه في موسمها بسبب توفر المنتجات. لا يقتصر الامر علي السعر الاقل، بل سيكون طعامك اشهي والذ عند تناوله في موسمه. في فصل الخريف يتوفر التفاح والتين والبنجر والكمثري.",
+            "diacritics_removed": "يكون سعر الفاكهه والخضراوات في موسم انباتها اقل من غيره من المواسم. ستلجا محلات الخضروات الي عرض الفاكهه باسعار مناسبه في موسمها بسبب توفر المنتجات. لا يقتصر الامر علي السعر الاقل، بل سيكون طعامك اشهي والذ عند تناوله في موسمه. في فصل الخريف يتوفر التفاح والتين والبنجر والكمثري.",
+            "punctuation_removed": "يكون سعر الفاكهه والخضراوات في موسم انباتها اقل من غيره من المواسم  ستلجا محلات الخضروات الي عرض الفاكهه باسعار مناسبه في موسمها بسبب توفر المنتجات  لا يقتصر الامر علي السعر الاقل  بل سيكون طعامك اشهي والذ عند تناوله في موسمه  في فصل الخريف يتوفر التفاح والتين والبنجر والكمثري ",
+            "repeated_chars_reduced": "يكون سعر الفاكه والخضراوات في موسم انباتها اقل من غيره من المواسم ستلجا محلات الخضروات الي عرض الفاكه باسعار مناسبه في موسمها بسب توفر المنتجات لا يقتصر الامر علي السعر الاقل بل سيكون طعامك اشهي والذ عند تناوله في موسمه في فصل الخريف يتوفر التفاح والتين والبنجر والكمثري",
+            "whitespace_normalized": "يكون سعر الفاكه والخضراوات في موسم انباتها اقل من غيره من المواسم ستلجا محلات الخضروات الي عرض الفاكه باسعار مناسبه في موسمها بسب توفر المنتجات لا يقتصر الامر علي السعر الاقل بل سيكون طعامك اشهي والذ عند تناوله في موسمه في فصل الخريف يتوفر التفاح والتين والبنجر والكمثري",
+            "final_result": "يكون سعر فكه خضراو وسم بات اقل غير وسم تلج محل خضرو الي عرض فكه سعر نسب وسم بسب وفر نتج قصر امر علي سعر اقل سيك طعم اشه ولذ نول وسم فصل خرف وفر تفح تين نجر كمثر",
             "preprocessing_summary": {
-                "original_length": 282,
-                "final_length": 47,
-                "reduction_percentage": 83.3,
-                "words_removed": 39,
-                "words_remaining": 7,
-            },
-        },
+                "original_length": 278,
+                "final_length": 165,
+                "reduction_percentage": 40.6,
+                "words_removed": 25,
+                "words_remaining": 23
+            }
+        }
     },
     "model_info": {
         "classifier": {
-            "model_name": "svm_classifier",
-            "vectorizer_loaded": True,
-            "model_loaded": True,
-            "classes": [
-                "culture",
-                "economy",
-                "international",
-                "local",
-                "religion",
-                "sports",
-            ],
+            "model_name": "traditional_svm_classifier",
+            "model_type": "SVC",
             "num_classes": 6,
-            "model_type": "SVM with TF-IDF vectorization",
+            "classes": ["culture", "economy", "international", "local", "religion", "sports"],
+            "has_predict_proba": True,
+            "has_vectorizer": True,
+            "vectorizer_type": "TfidfVectorizer",
+            "model_manager": {
+                "model_name": "traditional_svm",
+                "model_description": "Traditional SVM classifier with TF-IDF vectorization",
+                "model_config": {
+                    "type": "traditional",
+                    "classifier_path": "traditional_svm_classifier.joblib",
+                    "vectorizer_path": "traditional_tfidf_vectorizer_classifier.joblib",
+                    "description": "Traditional SVM classifier with TF-IDF vectorization"
+                },
+                "is_cached": True
+            }
         },
         "summarizer": {
             "vectorizer_loaded": True,
-            "model_type": "TF-IDF based summarization",
-        },
+            "model_type": "TF-IDF based summarization"
+        }
     },
+    "available_models": {
+        "models": {
+            "traditional_svm": {
+                "description": "Traditional SVM classifier with TF-IDF vectorization",
+                "type": "traditional",
+                "available": True,
+                "missing_files": [],
+                "is_default": True,
+                "is_cached": True
+            },
+            "modern_bert": {
+                "description": "Modern BERT-based transformer classifier",
+                "type": "modern",
+                "available": True,
+                "missing_files": [],
+                "is_default": False,
+                "is_cached": False
+            },
+            "modern_lstm": {
+                "description": "Modern LSTM-based neural network classifier",
+                "type": "modern",
+                "available": True,
+                "missing_files": [],
+                "is_default": False,
+                "is_cached": False
+            }
+        },
+        "current_model": "traditional_svm"
+    }
 }

model_manager.py

@@ -0,0 +1,179 @@
+from typing import Dict, Any
+import os
+from traditional_classifier import TraditionalClassifier
+
+try:
+    from modern_classifier import ModernClassifier
+    MODERN_MODELS_AVAILABLE = True
+except ImportError:
+    MODERN_MODELS_AVAILABLE = False
+
+
+class ModelManager:
+    """Manages different types of Arabic text classification models with per-request model selection and caching."""
+    
+    AVAILABLE_MODELS = {
+        "traditional_svm": {
+            "type": "traditional",
+            "classifier_path": "traditional_svm_classifier.joblib",
+            "vectorizer_path": "traditional_tfidf_vectorizer_classifier.joblib",
+            "description": "Traditional SVM classifier with TF-IDF vectorization"
+        },
+        
+        "modern_bert": {
+            "type": "modern",
+            "model_type": "bert",
+            "model_path": "modern_bert_classifier.safetensors",
+            "config_path": "config.json",
+            "description": "Modern BERT-based transformer classifier"
+        },
+        
+        "modern_lstm": {
+            "type": "modern", 
+            "model_type": "lstm",
+            "model_path": "modern_lstm_classifier.pth",
+            "description": "Modern LSTM-based neural network classifier"
+        }
+    }
+    
+    def __init__(self, default_model: str = "traditional_svm"):
+        self.default_model = default_model
+        self._model_cache = {}
+
+    def _get_model(self, model_name: str):
+        """Get model instance, loading from cache or creating new one."""
+        if model_name not in self.AVAILABLE_MODELS:
+            raise ValueError(f"Model '{model_name}' not available. Available models: {list(self.AVAILABLE_MODELS.keys())}")
+        
+        if model_name in self._model_cache:
+            return self._model_cache[model_name]
+        
+        model_config = self.AVAILABLE_MODELS[model_name]
+        
+        if model_config["type"] == "traditional":
+            classifier_path = model_config["classifier_path"]
+            vectorizer_path = model_config["vectorizer_path"]
+            
+            if not os.path.exists(classifier_path):
+                raise FileNotFoundError(f"Classifier file not found: {classifier_path}")
+            if not os.path.exists(vectorizer_path):
+                raise FileNotFoundError(f"Vectorizer file not found: {vectorizer_path}")
+                
+            model = TraditionalClassifier(classifier_path, vectorizer_path)
+            
+        elif model_config["type"] == "modern":
+            if not MODERN_MODELS_AVAILABLE:
+                raise ImportError("Modern models require PyTorch and transformers")
+                
+            model_path = model_config["model_path"]
+            
+            if not os.path.exists(model_path):
+                raise FileNotFoundError(f"Model file not found: {model_path}")
+                
+            config_path = model_config.get("config_path")
+            if config_path and not os.path.exists(config_path):
+                config_path = None
+                
+            model = ModernClassifier(
+                model_type=model_config["model_type"],
+                model_path=model_path,
+                config_path=config_path
+            )
+        
+        self._model_cache[model_name] = model
+        return model
+    
+    def predict(self, text: str, model_name: str = None) -> Dict[str, Any]:
+        """Predict using the specified model (or default if none specified)."""
+        if model_name is None:
+            model_name = self.default_model
+            
+        model = self._get_model(model_name)
+        result = model.predict(text)
+        
+        result["model_manager"] = {
+            "model_used": model_name,
+            "model_description": self.AVAILABLE_MODELS[model_name]["description"]
+        }
+        return result
+    
+    def predict_batch(self, texts: list, model_name: str = None) -> list:
+        """Predict batch using the specified model (or default if none specified)."""
+        if model_name is None:
+            model_name = self.default_model
+            
+        model = self._get_model(model_name)
+        results = model.predict_batch(texts)
+        
+        for result in results:
+            result["model_manager"] = {
+                "model_used": model_name,
+                "model_description": self.AVAILABLE_MODELS[model_name]["description"]
+            }
+        return results
+    
+    def get_model_info(self, model_name: str = None) -> Dict[str, Any]:
+        """Get information about a specific model (or default if none specified)."""
+        if model_name is None:
+            model_name = self.default_model
+            
+        model = self._get_model(model_name)
+        model_info = model.get_model_info()
+        model_info.update({
+            "model_manager": {
+                "model_name": model_name,
+                "model_description": self.AVAILABLE_MODELS[model_name]["description"],
+                "model_config": self.AVAILABLE_MODELS[model_name],
+                "is_cached": model_name in self._model_cache
+            }
+        })
+        return model_info
+    
+    def get_available_models(self) -> Dict[str, Any]:
+        """Get list of all available models."""
+        available = {}
+        for model_name, config in self.AVAILABLE_MODELS.items():
+            files_exist = True
+            missing_files = []
+            
+            if config["type"] == "traditional":
+                for file_key in ["classifier_path", "vectorizer_path"]:
+                    if not os.path.exists(config[file_key]):
+                        files_exist = False
+                        missing_files.append(config[file_key])
+            elif config["type"] == "modern":
+                if not os.path.exists(config["model_path"]):
+                    files_exist = False
+                    missing_files.append(config["model_path"])
+            
+            available[model_name] = {
+                "description": config["description"],
+                "type": config["type"],
+                "available": files_exist,
+                "missing_files": missing_files if not files_exist else [],
+                "is_default": model_name == self.default_model,
+                "is_cached": model_name in self._model_cache
+            }
+        
+        return available
+    
+    def clear_cache(self, model_name: str = None) -> Dict[str, Any]:
+        """Clear model cache (specific model or all models)."""
+        if model_name:
+            if model_name in self._model_cache:
+                del self._model_cache[model_name]
+                return {"message": f"Cache cleared for model: {model_name}"}
+            else:
+                return {"message": f"Model {model_name} was not cached"}
+        else:
+            cleared_count = len(self._model_cache)
+            self._model_cache.clear()
+            return {"message": f"Cache cleared for {cleared_count} models"}
+    
+    def get_cache_status(self) -> Dict[str, Any]:
+        """Get information about cached models."""
+        return {
+            "cached_models": list(self._model_cache.keys()),
+            "cache_count": len(self._model_cache),
+            "default_model": self.default_model
+        }

modern_bert_classifier.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd36d704baf314e519a6abbcde8b6239ba3867dc4a44acc9c690701b863a3a7d
+size 442511368

modern_classifier.py

@@ -0,0 +1,266 @@
+import torch
+import torch.nn as nn
+import numpy as np
+from typing import List, Dict, Any, Optional
+from preprocessor import preprocess_for_classification
+import re
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from safetensors.torch import load_file
+from transformers import AutoConfig
+
+
+class LSTMClassifier(nn.Module):
+    """LSTM-based Arabic text classifier."""
+    
+    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, num_layers=2, bidirectional=False):
+        super(LSTMClassifier, self).__init__()
+        self.embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.bidirectional = bidirectional
+        self.lstm = nn.LSTM(
+            embedding_dim,
+            hidden_dim,
+            num_layers,
+            batch_first=True,
+            dropout=0.3,
+            bidirectional=self.bidirectional
+        )
+        fc_input_dim = hidden_dim * 2 if self.bidirectional else hidden_dim
+        self.fc = nn.Linear(fc_input_dim, output_dim)
+        self.dropout = nn.Dropout(0.5)
+        
+    def forward(self, x):
+        embedded = self.embedding(x)
+        _, (hidden, _) = self.lstm(embedded)
+        if self.bidirectional:
+            forward_hidden = hidden[-2]
+            backward_hidden = hidden[-1]
+            combined = torch.cat((forward_hidden, backward_hidden), dim=1)
+            h = combined
+        else:
+            h = hidden[-1]
+        output = self.fc(self.dropout(h))
+        return output
+
+
+class ModernClassifier:
+    """Modern Arabic text classifier supporting BERT and LSTM models."""
+    
+    def __init__(self, model_type: str, model_path: str, config_path: Optional[str] = None):
+        self.model_type = model_type.lower()
+        self.model_path = model_path
+        self.config_path = config_path
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        
+        self.classes = np.array(['culture', 'economy', 'international', 'local', 'religion', 'sports'])
+        
+        if self.model_type == 'bert':
+            self._load_bert_model()
+        elif self.model_type == 'lstm':
+            self._load_lstm_model()
+        else:
+            raise ValueError(f"Unsupported model type: {model_type}")
+            
+        self.model_name = f"{model_type}_classifier"
+    
+    def _load_bert_model(self):
+        """Load BERT model from safetensors."""
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained('aubmindlab/bert-base-arabertv2')
+            state_dict = load_file(self.model_path)
+            embed_key = next(k for k in state_dict if 'embeddings.word_embeddings.weight' in k)
+            checkpoint_vocab_size = state_dict[embed_key].shape[0]
+            config = AutoConfig.from_pretrained(
+                'aubmindlab/bert-base-arabertv2',
+                num_labels=len(self.classes),
+                vocab_size=checkpoint_vocab_size
+            )
+            self.model = AutoModelForSequenceClassification.from_config(config)
+            self.model.resize_token_embeddings(checkpoint_vocab_size)
+            self.model.load_state_dict(state_dict, strict=False)
+            self.model.to(self.device)
+            self.model.eval()
+        except Exception as e:
+            raise RuntimeError(f"Error loading BERT model: {e}")
+    
+    def _load_lstm_model(self):
+        """Load LSTM model from .pth file."""
+        try:
+            checkpoint = torch.load(self.model_path, map_location=self.device)
+            state_dict = checkpoint.get('model_state_dict', checkpoint)
+            vocab_size, embedding_dim = state_dict['embedding.weight'].shape
+            _, hidden_dim = state_dict['lstm.weight_hh_l0'].shape
+            layer_nums = set(int(re.match(r'lstm\.weight_ih_l(\d+)', k).group(1))
+                             for k in state_dict if re.match(r'lstm\.weight_ih_l(\d+)$', k))
+            num_layers = len(layer_nums)
+            bidirectional = True
+            output_dim = len(self.classes)
+            self.model = LSTMClassifier(vocab_size, embedding_dim, hidden_dim,
+                                        output_dim, num_layers=num_layers,
+                                        bidirectional=bidirectional)
+            self.model.load_state_dict(state_dict, strict=False)
+            self.model.to(self.device)
+            self.model.eval()
+            self.vocab = checkpoint.get('vocab', {})
+        except Exception as e:
+            raise RuntimeError(f"Error loading LSTM model: {e}")
+    
+    def _preprocess_text_for_bert(self, text: str) -> Dict[str, torch.Tensor]:
+        """Preprocess text for BERT model."""
+        cleaned_text = preprocess_for_classification(text)
+        
+        inputs = self.tokenizer(
+            cleaned_text,
+            return_tensors='pt',
+            truncation=True,
+            padding=True,
+            max_length=512
+        )
+        
+        return {key: value.to(self.device) for key, value in inputs.items()}
+    
+    def _preprocess_text_for_lstm(self, text: str) -> torch.Tensor:
+        """Preprocess text for LSTM model."""
+        cleaned_text = preprocess_for_classification(text)
+        
+        tokens = cleaned_text.split()
+        
+        if hasattr(self, 'vocab') and self.vocab:
+            indices = [self.vocab.get(token, 0) for token in tokens]
+        else:
+            indices = [hash(token) % 10000 for token in tokens]
+        
+        max_length = 100
+        if len(indices) > max_length:
+            indices = indices[:max_length]
+        else:
+            indices.extend([0] * (max_length - len(indices)))
+        
+        return torch.tensor([indices], dtype=torch.long).to(self.device)
+    
+    def predict(self, text: str) -> Dict[str, Any]:
+        """Predict class with full probability distribution and metadata."""
+        cleaned_text = preprocess_for_classification(text)
+        
+        with torch.no_grad():
+            if self.model_type == 'bert':
+                inputs = self._preprocess_text_for_bert(text)
+                outputs = self.model(**inputs)
+                logits = outputs.logits
+            elif self.model_type == 'lstm':
+                inputs = self._preprocess_text_for_lstm(text)
+                logits = self.model(inputs)
+            
+            probabilities = torch.softmax(logits, dim=-1).cpu().numpy()[0]
+            
+            prediction_index = int(np.argmax(probabilities))
+            prediction = self.classes[prediction_index]
+            confidence = float(probabilities[prediction_index])
+        
+        prob_distribution = {}
+        for i, class_label in enumerate(self.classes):
+            prob_distribution[str(class_label)] = float(probabilities[i])
+        
+        return {
+            "prediction": str(prediction),
+            "prediction_index": prediction_index,
+            "confidence": confidence,
+            "probability_distribution": prob_distribution,
+            "cleaned_text": cleaned_text,
+            "model_used": self.model_name,
+            "prediction_metadata": {
+                "max_probability": float(np.max(probabilities)),
+                "min_probability": float(np.min(probabilities)),
+                "entropy": float(-np.sum(probabilities * np.log(probabilities + 1e-10))),
+                "num_classes": len(probabilities),
+                "model_type": self.model_type,
+                "device": str(self.device)
+            },
+        }
+    
+    def predict_batch(self, texts: List[str]) -> List[Dict[str, Any]]:
+        """Predict classes for multiple texts using true batch processing."""
+        if not texts:
+            return []
+        
+        cleaned_texts = [preprocess_for_classification(text) for text in texts]
+        
+        with torch.no_grad():
+            if self.model_type == 'bert':
+                inputs = self.tokenizer(
+                    cleaned_texts,
+                    return_tensors='pt',
+                    truncation=True,
+                    padding=True,
+                    max_length=512
+                )
+                inputs = {key: value.to(self.device) for key, value in inputs.items()}
+                outputs = self.model(**inputs)
+                logits = outputs.logits
+                
+            elif self.model_type == 'lstm':
+                batch_indices = []
+                max_length = 100
+                
+                for cleaned_text in cleaned_texts:
+                    tokens = cleaned_text.split()
+                    if hasattr(self, 'vocab') and self.vocab:
+                        indices = [self.vocab.get(token, 0) for token in tokens]
+                    else:
+                        indices = [hash(token) % 10000 for token in tokens]
+                    
+                    if len(indices) > max_length:
+                        indices = indices[:max_length]
+                    else:
+                        indices.extend([0] * (max_length - len(indices)))
+                    
+                    batch_indices.append(indices)
+                
+                batch_tensor = torch.tensor(batch_indices, dtype=torch.long).to(self.device)
+                logits = self.model(batch_tensor)
+            
+            probabilities = torch.softmax(logits, dim=-1).cpu().numpy()
+            
+            results = []
+            for i, (text, cleaned_text) in enumerate(zip(texts, cleaned_texts)):
+                probs = probabilities[i]
+                prediction_index = int(np.argmax(probs))
+                prediction = self.classes[prediction_index]
+                confidence = float(probs[prediction_index])
+                
+                prob_distribution = {}
+                for j, class_label in enumerate(self.classes):
+                    prob_distribution[str(class_label)] = float(probs[j])
+                
+                result = {
+                    "prediction": str(prediction),
+                    "prediction_index": prediction_index,
+                    "confidence": confidence,
+                    "probability_distribution": prob_distribution,
+                    "cleaned_text": cleaned_text,
+                    "model_used": self.model_name,
+                    "prediction_metadata": {
+                        "max_probability": float(np.max(probs)),
+                        "min_probability": float(np.min(probs)),
+                        "entropy": float(-np.sum(probs * np.log(probs + 1e-10))),
+                        "num_classes": len(probs),
+                        "model_type": self.model_type,
+                        "device": str(self.device)
+                    },
+                }
+                results.append(result)
+        
+        return results
+    
+    def get_model_info(self) -> Dict[str, Any]:
+        """Get model information and capabilities."""
+        return {
+            "model_name": self.model_name,
+            "model_type": self.model_type,
+            "model_path": self.model_path,
+            "num_classes": len(self.classes),
+            "classes": self.classes.tolist(),
+            "device": str(self.device),
+            "has_predict_proba": True,
+            "framework": "pytorch",
+            "modern_model": True
+        }

modern_lstm_classifier.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bc096c6cdc0523b378529139c5bcbb9dbcbcd9351ba6babc272a90b1d78bd4f
+size 5921296

requirements.txt

@@ -4,3 +4,6 @@ scikit-learn
 nltk
 joblib
 numpy
+torch
+transformers
+safetensors

summarizer.py

@@ -8,7 +8,7 @@ from preprocessor import preprocess_for_summarization
 class ArabicSummarizer:
     """Arabic text summarizer using TF-IDF scoring."""
     
-    def __init__(self, vectorizer_path: str = "tfidf_vectorizer_text_summarization.joblib"):
+    def __init__(self, vectorizer_path: str = "traditional_tfidf_vectorizer_summarization.joblib"):
         self.vectorizer = joblib.load(vectorizer_path)
     
     def summarize(self, text: str, num_sentences: int = 3) -> Dict[str, Any]:

classifier.py → traditional_classifier.py

@@ -1,16 +1,16 @@
-import joblib
 import numpy as np
+import joblib
 from typing import List, Dict, Any
 from preprocessor import preprocess_for_classification
 
 
-class ArabicClassifier:
-    """Arabic text classifier with probability distributions and metadata."""
+class TraditionalClassifier:
+    """Traditional text classifier with probability distributions and metadata."""
 
     def __init__(
         self,
-        classifier_path: str = "svm_classifier.joblib",
-        vectorizer_path: str = "tfidf_vectorizer_classifier.joblib",
+        classifier_path: str = "traditional_svm_classifier.joblib",
+        vectorizer_path: str = "traditional_tfidf_vectorizer_classifier.joblib",
     ):
         self.model = joblib.load(classifier_path)
         self.vectorizer = joblib.load(vectorizer_path)