add server preview filter

app.py

@@ -1,19 +1,23 @@
 import os
 from datetime import datetime, timedelta
 from sys import platform
+from typing import Any, Dict
 
 import gradio as gr
 import pandas as pd
 from diskcache import Cache
 from dotenv import load_dotenv
 from httpx import Client
-from huggingface_hub import hf_hub_url, list_datasets
+from huggingface_hub import DatasetCard, hf_hub_url, list_datasets
 from tqdm.auto import tqdm
 from tqdm.contrib.concurrent import thread_map
-from huggingface_hub import DatasetCard
+
 
 load_dotenv()
 
+LIMIT = None
+CACHE_TIME = 60 * 60 * 6  # 6 hours
+REMOVE_ORGS = {"HuggingFaceM4", "HuggingFaceBR4", "open-llm-leaderboard"}
 
 HF_TOKEN = os.getenv("HF_TOKEN")
 USER_AGENT = os.getenv("USER_AGENT")
@@ -46,7 +50,7 @@ def get_three_months_ago():
     return now - timedelta(days=90)
 
 
-def get_readme_len(dataset):
+def get_readme_len(dataset: Dict[str, Any]):
     try:
         url = hf_hub_url(dataset["id"], "README.md", repo_type="dataset")
         resp = client.get(url)
@@ -59,6 +63,25 @@ def get_readme_len(dataset):
         return None
 
 
+def check_ds_server_valid(id):
+    url = f"https://datasets-server.huggingface.co/is-valid?dataset={id}"
+    response = client.get(url)
+    if response.status_code != 200:
+        return False
+    try:
+        data = response.json()
+        preview = data.get("preview")
+        return preview is not None
+    except Exception as e:
+        print(e)
+        return False
+
+
+def has_server_preview(dataset):
+    dataset["server_preview"] = check_ds_server_valid(dataset["id"])
+    return dataset
+
+
 def render_model_hub_link(hub_id):
     link = f"https://huggingface.co/datasets/{hub_id}"
     return (
@@ -67,23 +90,27 @@ def render_model_hub_link(hub_id):
     )
 
 
-@cache.memoize(expire=60 * 60 * 12)
+@cache.memoize(expire=CACHE_TIME)
 def get_datasets():
-    return list(tqdm(iter(list_datasets(limit=None, full=True))))
+    return list(
+        tqdm(
+            iter(
+                list_datasets(limit=LIMIT, full=True, sort="lastModified", direction=-1)
+            )
+        )
+    )
 
 
-@cache.memoize(expire=60 * 60 * 12)
+@cache.memoize(expire=CACHE_TIME)
 def load_data():
     datasets = get_datasets()
     datasets = [add_created_data(dataset) for dataset in tqdm(datasets)]
-    filtered = [ds for ds in datasets if ds.get("cardData")]
-    filtered = [ds for ds in filtered if ds["created"] > get_three_months_ago()]
+    filtered = [ds for ds in datasets if ds["created"] > get_three_months_ago()]
     ds_with_len = thread_map(get_readme_len, filtered)
     ds_with_len = [ds for ds in ds_with_len if ds is not None]
-    return ds_with_len
-
-
-remove_orgs = {"HuggingFaceM4", "HuggingFaceBR4", "open-llm-leaderboard"}
+    ds_with_valid_status = thread_map(has_server_preview, ds_with_len)
+    ds_with_valid_status = [ds for ds in ds_with_valid_status if ds is not None]
+    return ds_with_valid_status
 
 
 columns_to_drop = [
@@ -103,7 +130,7 @@ columns_to_drop = [
 ]
 
 
-def prep_dataframe(remove_orgs_and_users=remove_orgs, columns_to_drop=columns_to_drop):
+def prep_dataframe(remove_orgs_and_users=REMOVE_ORGS, columns_to_drop=columns_to_drop):
     ds_with_len = load_data()
     if remove_orgs_and_users:
         ds_with_len = [
@@ -132,8 +159,10 @@ def filter_by_readme_len(df, min_len=None):
     return df
 
 
-def filter_df(max_age_days=None, min_len=None):
+def filter_df(max_age_days=None, min_len=None, needs_server_preview: bool = False):
     df = prep_dataframe()
+    if needs_server_preview:
+        df = df[df["server_preview"] is True]
     if max_age_days is not None:
         df = filter_df_by_max_age(df, max_age_days=max_age_days)
     if min_len is not None:
@@ -164,10 +193,25 @@ with gr.Blocks() as demo:
             step=50,
             interactive=True,
         )
+        needs_server_preview = gr.Checkbox(
+            label="Needs Server Preview", default=False, interactive=True
+        )
 
     output = gr.DataFrame(filter_df, datatype="markdown", min_width=160 * 2.5)
-    max_age_days.input(filter_df, inputs=[max_age_days, min_len], outputs=[output])
-    min_len.input(filter_df, inputs=[max_age_days, min_len], outputs=[output])
-
+    max_age_days.input(
+        filter_df,
+        inputs=[max_age_days, min_len, needs_server_preview],
+        outputs=[output],
+    )
+    min_len.input(
+        filter_df,
+        inputs=[max_age_days, min_len, needs_server_preview],
+        outputs=[output],
+    )
+    needs_server_preview.change(
+        filter_df,
+        inputs=[max_age_days, min_len, needs_server_preview],
+        outputs=[output],
+    )
 
 demo.launch()