Update app.py

app.py

@@ -12,17 +12,14 @@ username="omnibus"
 dataset_name="tmp"
 save_data=f'https://huggingface.co/datasets/{username}/{dataset_name}/raw/main/'
 api=HfApi(token="")
-filename="test"
-
+filename="urls"
+filename2="pages"
 
 def init():
     r = requests.get(f'{save_data}crawl/{filename}.json') 
     print(f'status code main:: {r.status_code}')
     if r.status_code==200:
         lod = json.loads(r.text)
-        #print(f'lod:: {lod}')
-        #lod[0]['comment']=lod[0]['comment']+1
-        #lod[0]['comment_list'].append({'user':persona[persona2]['name'],'datetime':'','comment':output,'reply_list':[]})
     else:
         lod={} 
     return lod
@@ -338,7 +335,14 @@ def sitemap(url,file_state,level):
                                 print (e)
                 except Exception as e:
                     print (e)
-        uri_key=sort_doc(link_box,file_state,8)
+        #url_page=[]
+        url_front=[]
+        for ea_link in link2['TREE']:
+            url_list=ea_link.split("/")
+            url_front.append(url_list[0])
+            print(f'URL_FRONT:: {url_front}')
+        #url_key=sort
+        uri_key=sort_doc(url_front,file_state,8)
 ######## Save Database ########
     uid=uuid.uuid4()
     #for ea in list(uri_key.keys()):