Hugging Face's logo

Spaces:
librarian-bots
/
new_hub_datasets
Running

App Files Community
davanstrien HF Staff commited on
Commit
f5faacd
·
1 Parent(s): e62ac39

update'

Browse files
Files changed (3) hide show
  1. app.py +11 -10
  2. requirements.in +1 -0
  3. requirements.txt +10 -7
app.py CHANGED
@@ -11,7 +11,7 @@ from httpx import Client
11
  from huggingface_hub import DatasetCard, hf_hub_url, list_datasets
12
  from tqdm.auto import tqdm
13
  from tqdm.contrib.concurrent import thread_map
14
-
15
 
16
  load_dotenv()
17
 
@@ -30,11 +30,12 @@ client = Client(
30
  headers=headers,
31
  timeout=60,
32
  )
33
- LOCAL = False
34
- if platform == "darwin":
35
- LOCAL = True
36
- cache_dir = "cache" if LOCAL else "/data/diskcache"
37
- cache = Cache(cache_dir)
 
38
 
39
 
40
  def add_created_data(dataset):
@@ -90,7 +91,7 @@ def render_model_hub_link(hub_id):
90
  )
91
 
92
 
93
- @cache.memoize(expire=CACHE_TIME)
94
  def get_datasets():
95
  return list(
96
  tqdm(
@@ -101,7 +102,7 @@ def get_datasets():
101
  )
102
 
103
 
104
- @cache.memoize(expire=CACHE_TIME)
105
  def load_data():
106
  datasets = get_datasets()
107
  datasets = [add_created_data(dataset) for dataset in tqdm(datasets)]
@@ -162,7 +163,7 @@ def filter_by_readme_len(df, min_len=None):
162
  def filter_df(max_age_days=None, min_len=None, needs_server_preview: bool = False):
163
  df = prep_dataframe()
164
  if needs_server_preview:
165
- df = df[df["server_preview"] is True]
166
  if max_age_days is not None:
167
  df = filter_df_by_max_age(df, max_age_days=max_age_days)
168
  if min_len is not None:
@@ -194,7 +195,7 @@ with gr.Blocks() as demo:
194
  interactive=True,
195
  )
196
  needs_server_preview = gr.Checkbox(
197
- label="Needs Server Preview", default=False, interactive=True
198
  )
199
 
200
  output = gr.DataFrame(filter_df, datatype="markdown", min_width=160 * 2.5)
 
11
  from huggingface_hub import DatasetCard, hf_hub_url, list_datasets
12
  from tqdm.auto import tqdm
13
  from tqdm.contrib.concurrent import thread_map
14
+ from cachetools import TTLCache, cached
15
 
16
  load_dotenv()
17
 
 
30
  headers=headers,
31
  timeout=60,
32
  )
33
+ # LOCAL = False
34
+ # if platform == "darwin":
35
+ # LOCAL = True
36
+ # cache_dir = "cache" if LOCAL else "/data/diskcache"
37
+ # cache = Cache(cache_dir)
38
+ cache = TTLCache(maxsize=10, ttl=CACHE_TIME)
39
 
40
 
41
  def add_created_data(dataset):
 
91
  )
92
 
93
 
94
+ @cached(cache)
95
  def get_datasets():
96
  return list(
97
  tqdm(
 
102
  )
103
 
104
 
105
+ @cached(cache)
106
  def load_data():
107
  datasets = get_datasets()
108
  datasets = [add_created_data(dataset) for dataset in tqdm(datasets)]
 
163
  def filter_df(max_age_days=None, min_len=None, needs_server_preview: bool = False):
164
  df = prep_dataframe()
165
  if needs_server_preview:
166
+ df = df[df["server_preview"] == True]
167
  if max_age_days is not None:
168
  df = filter_df_by_max_age(df, max_age_days=max_age_days)
169
  if min_len is not None:
 
195
  interactive=True,
196
  )
197
  needs_server_preview = gr.Checkbox(
198
+ label="Needs Server Preview", value=False, interactive=True
199
  )
200
 
201
  output = gr.DataFrame(filter_df, datatype="markdown", min_width=160 * 2.5)
requirements.in CHANGED
@@ -1,3 +1,4 @@
 
1
  datasets
2
  datasets
3
  diskcache
 
1
+ cachetools
2
  datasets
3
  datasets
4
  diskcache
requirements.txt CHANGED
@@ -16,8 +16,9 @@ altair==5.1.2
16
  # via gradio
17
  annotated-types==0.6.0
18
  # via pydantic
19
- anyio==4.0.0
20
  # via
 
21
  # httpcore
22
  # starlette
23
  async-timeout==4.0.3
@@ -27,6 +28,8 @@ attrs==23.1.0
27
  # aiohttp
28
  # jsonschema
29
  # referencing
 
 
30
  certifi==2023.7.22
31
  # via
32
  # httpcore
@@ -42,7 +45,7 @@ contourpy==1.1.1
42
  # via matplotlib
43
  cycler==0.12.1
44
  # via matplotlib
45
- datasets==2.14.4
46
  # via -r requirements.in
47
  dill==0.3.7
48
  # via
@@ -50,7 +53,7 @@ dill==0.3.7
50
  # multiprocess
51
  diskcache==5.6.3
52
  # via -r requirements.in
53
- fastapi==0.103.0
54
  # via gradio
55
  ffmpy==0.3.1
56
  # via gradio
@@ -62,14 +65,14 @@ frozenlist==1.4.0
62
  # via
63
  # aiohttp
64
  # aiosignal
65
- fsspec[http]==2023.9.2
66
  # via
67
  # datasets
68
  # gradio-client
69
  # huggingface-hub
70
- gradio==3.47.1
71
  # via -r requirements.in
72
- gradio-client==0.6.0
73
  # via gradio
74
  h11==0.14.0
75
  # via
@@ -118,7 +121,7 @@ multidict==6.0.4
118
  # yarl
119
  multiprocess==0.70.15
120
  # via datasets
121
- numpy==1.26.0
122
  # via
123
  # altair
124
  # contourpy
 
16
  # via gradio
17
  annotated-types==0.6.0
18
  # via pydantic
19
+ anyio==3.7.1
20
  # via
21
+ # fastapi
22
  # httpcore
23
  # starlette
24
  async-timeout==4.0.3
 
28
  # aiohttp
29
  # jsonschema
30
  # referencing
31
+ cachetools==5.3.1
32
+ # via -r requirements.in
33
  certifi==2023.7.22
34
  # via
35
  # httpcore
 
45
  # via matplotlib
46
  cycler==0.12.1
47
  # via matplotlib
48
+ datasets==2.14.5
49
  # via -r requirements.in
50
  dill==0.3.7
51
  # via
 
53
  # multiprocess
54
  diskcache==5.6.3
55
  # via -r requirements.in
56
+ fastapi==0.103.2
57
  # via gradio
58
  ffmpy==0.3.1
59
  # via gradio
 
65
  # via
66
  # aiohttp
67
  # aiosignal
68
+ fsspec[http]==2023.6.0
69
  # via
70
  # datasets
71
  # gradio-client
72
  # huggingface-hub
73
+ gradio==3.48.0
74
  # via -r requirements.in
75
+ gradio-client==0.6.1
76
  # via gradio
77
  h11==0.14.0
78
  # via
 
121
  # yarl
122
  multiprocess==0.70.15
123
  # via datasets
124
+ numpy==1.26.1
125
  # via
126
  # altair
127
  # contourpy