update api handle 3

app.py

@@ -22,7 +22,7 @@ app = FastAPI(
 
 class CrawlRequest(BaseModel):
     url: HttpUrl
-    cache_mode: str = "ENABLED"
+    cache_mode: str = "DISABLED"
     excluded_tags: list[str] = ["nav", "footer", "aside", "header", "script", "style"]
     remove_overlay_elements: bool = True
     subject: Optional[str] = None  # Optional subject for content filtering
@@ -185,17 +185,17 @@ def extract_metadata(markdown: str, html: str) -> Dict:
 @app.post("/crawl", response_model=CrawlResponse)
 async def crawl_url(request: CrawlRequest):
     try:
-        cache_mode = getattr(CacheMode, request.cache_mode)
+        # Force cache mode to DISABLED
+        cache_mode = CacheMode.DISABLED
         
-        # Configure markdown generator based on whether subject is provided
+        # Configure markdown generator
         if request.subject:
-            # Use BM25 filter when subject is provided use_stemming=True
             content_filter = BM25ContentFilter(
                 user_query=request.subject,
-                bm25_threshold=1.2
+                bm25_threshold=1.2,
+                min_word_threshold=50
             )
         else:
-            # Use default pruning filter when no subject
             content_filter = PruningContentFilter(
                 threshold=0.48,
                 threshold_type="fixed",
@@ -210,7 +210,7 @@ async def crawl_url(request: CrawlRequest):
         # Create crawler with configuration
         async with AsyncWebCrawler() as crawler:
             config = CrawlerRunConfig(
-                cache_mode=cache_mode,
+                cache_mode=cache_mode,  # Always DISABLED
                 excluded_tags=request.excluded_tags,
                 remove_overlay_elements=request.remove_overlay_elements,
                 markdown_generator=md_generator,
@@ -225,15 +225,13 @@ async def crawl_url(request: CrawlRequest):
                 config=config
             )
             
-            # Extract content
+            # Process results
             markdown = result.markdown_v2.raw_markdown
             html = result.html
             
-            # Extract articles and metadata
             articles = extract_articles(markdown)
             metadata = extract_metadata(markdown, html)
             
-            # Add source URL and subject to metadata
             metadata["subject"] = request.subject
             for article in articles:
                 article.source_url = str(request.url)