Update app.py

app.py

@@ -999,45 +999,31 @@ target_datasets = {
 
 def get_korea_datasets():
     """Korea 관련 데이터셋 검색"""
-    search_terms = ['korea', 'korean', 'kor']  # 검색어 확장
-    all_korea_datasets = []
+    params = {
+        "search": "korea",
+        "full": "True",
+        "limit": 10000
+    }
     
-    for term in search_terms:
-        params = {
-            "search": term,
-            "full": "True",
-            "limit": 10000  # 검색 범위 확장
-        }
+    try:
+        response = requests.get(
+            "https://huggingface.co/api/datasets",
+            headers={'Accept': 'application/json'},  # Authorization 대신 Accept 헤더 사용
+            params=params
+        )
         
-        try:
-            response = requests.get(
-                "https://huggingface.co/api/datasets",
-                headers={'Authorization': f'Bearer {HF_TOKEN}'},
-                params=params
-            )
-            
-            if response.status_code == 200:
-                datasets = response.json()
-                all_korea_datasets.extend(datasets)
-                print(f"Found {len(datasets)} datasets for search term '{term}'")
-            else:
-                print(f"Failed to fetch datasets for term '{term}': {response.status_code}")
-        except Exception as e:
-            print(f"Error fetching datasets for term '{term}': {str(e)}")
-    
-    # 중복 제거
-    seen_ids = set()
-    unique_datasets = []
-    for dataset in all_korea_datasets:
-        dataset_id = dataset.get('id', '')
-        if dataset_id and dataset_id not in seen_ids:
-            seen_ids.add(dataset_id)
-            unique_datasets.append(dataset)
-    
-    print(f"Total unique Korea-related datasets found: {len(unique_datasets)}")
-    return unique_datasets
+        if response.status_code == 200:
+            korea_datasets = response.json()
+            print(f"Found {len(korea_datasets)} Korea-related datasets")
+            return korea_datasets
+        else:
+            print(f"Failed to fetch Korea datasets: {response.status_code}")
+            return []
+    except Exception as e:
+        print(f"Error fetching Korea datasets: {str(e)}")
+        return []
 
-def get_all_datasets(limit=10000):  # 기본 limit 증가
+def get_all_datasets(limit=10000):
     """모든 데이터셋과 Korea 관련 데이터셋 가져오기"""
     all_datasets = []
     page_size = 1000
@@ -1049,55 +1035,64 @@ def get_all_datasets(limit=10000):  # 기본 limit 증가
             'offset': offset
         }
         
-        try:
-            response = requests.get(
-                "https://huggingface.co/api/datasets",
-                headers={'Authorization': f'Bearer {HF_TOKEN}'},
-                params=params
-            )
-            
-            if response.status_code == 200:
-                datasets = response.json()
-                all_datasets.extend(datasets)
-                print(f"Fetched datasets {offset+1} to {offset+len(datasets)}")
-            else:
-                print(f"Failed to fetch datasets at offset {offset}: {response.status_code}")
-                break
-        except Exception as e:
-            print(f"Error fetching datasets at offset {offset}: {str(e)}")
+        response = requests.get(
+            "https://huggingface.co/api/datasets",
+            headers={'Accept': 'application/json'},  # Authorization 대신 Accept 헤더 사용
+            params=params
+        )
+        
+        if response.status_code == 200:
+            all_datasets.extend(response.json())
+            print(f"Fetched datasets {offset+1} to {offset+len(response.json())}")
+        else:
+            print(f"Failed to fetch datasets at offset {offset}: {response.status_code}")
             break
     
-    # Korea 검색 결과 추가
-    korea_datasets = get_korea_datasets()
-    existing_ids = {dataset.get('id', '') for dataset in all_datasets}
+    # Korea 검색 결과도 동일하게 확장
+    korea_params = {
+        "search": "korea",
+        "full": "True",
+        "limit": limit
+    }
     
-    added_count = 0
-    for korea_dataset in korea_datasets:
-        if korea_dataset.get('id', '') not in existing_ids:
-            all_datasets.append(korea_dataset)
-            existing_ids.add(korea_dataset.get('id', ''))
-            added_count += 1
+    korea_response = requests.get(
+        "https://huggingface.co/api/datasets",
+        headers={'Accept': 'application/json'},  # Authorization 대신 Accept 헤더 사용
+        params=korea_params
+    )
     
-    print(f"Added {added_count} additional Korea-related datasets")
-    print(f"Total datasets: {len(all_datasets)}")
+    if korea_response.status_code == 200:
+        korea_datasets = korea_response.json()
+        print(f"Fetched {len(korea_datasets)} Korea-related datasets")
+        
+        # 중복 제거하면서 Korea 데이터셋 추가
+        existing_ids = {dataset.get('id', '') for dataset in all_datasets}
+        for dataset in korea_datasets:
+            if dataset.get('id', '') not in existing_ids:
+                all_datasets.append(dataset)
+                existing_ids.add(dataset.get('id', ''))
     
+    print(f"Total unique datasets: {len(all_datasets)}")
     return all_datasets[:limit]
 
 def get_datasets_data(progress=gr.Progress()):
     def calculate_rank(dataset_id, all_global_datasets, korea_datasets):
+        # 글로벌 순위 확인
         global_rank = next((idx for idx, d in enumerate(all_global_datasets, 1) 
                           if d.get('id', '').strip() == dataset_id.strip()), None)
         
+        # Korea 데이터셋인 경우
         is_korea = any(d.get('id', '').strip() == dataset_id.strip() for d in korea_datasets)
         
         if is_korea:
+            # Korea 데이터셋 중에서의 순위 확인
             korea_rank = next((idx for idx, d in enumerate(korea_datasets, 1)
                              if d.get('id', '').strip() == dataset_id.strip()), None)
             
             if korea_rank:
-                return min(global_rank or 3001, korea_rank + 1000), True
+                return min(global_rank or 10001, korea_rank + 1000), True
         
-        return global_rank if global_rank else 'Not in top 3000', is_korea
+        return global_rank if global_rank else 'Not in top 10000', is_korea
 
     try:
         progress(0, desc="Fetching datasets...")
@@ -1113,9 +1108,15 @@ def get_datasets_data(progress=gr.Progress()):
             empty_df = pd.DataFrame(columns=['Global Rank', 'Dataset ID', 'Title', 'Downloads', 'Likes', 'Korea Search', 'URL'])
             return fig, error_html, empty_df
 
-        all_global_datasets = get_all_datasets(limit=3000)
+        # 일반 데이터셋과 Korea 관련 데이터셋 모두 가져오기
+        all_global_datasets = get_all_datasets(limit=10000)
         korea_datasets = get_korea_datasets()
         
+        print(f"Total global datasets fetched: {len(all_global_datasets)}")
+        print(f"Total Korea datasets fetched: {len(korea_datasets)}")
+        
+
+        
         filtered_datasets = []
         for dataset_id in target_datasets.keys():
             try: