app.py

import gradio as gr
import requests
import bs4

def link_find(url):
    out = []
    source = requests.get(url)
    if source.status_code ==200:
        print("YES")
        #soup = bs4.BeautifulSoup(source.content,'lxml')
        soup = bs4.BeautifulSoup(source.content,'html.parser')
        
        rawp=(f'RAW TEXT RETURNED: {soup.text}')
        cnt=0
        cnt+=len(rawp)
        rawt=soup.text
        #out.append(rawp)
        #out.append("HTML fragments: ")
        node1 = {"URL":url,"TITLE":soup.title,"STRING":soup.description,"TEXT":rawt,"LINKS":[],"TREE":[]}
        node2 = {"URL":url,"LINKS":[],"TREE":[]}

        q=("a","p","span","content","article")
        for p in soup.find_all("a"):
            url0=p.get('href')
            if url0.startswith("//"):
                uri1=url.split("//")[0]
                #uri2=url.split("//")[1]
                #uri3=uri2.split("/")[0]
                #uri=f'{uri1}//{uri3}'
                uri=f'{uri1}{url0}'
                print(uri)
            if url0.startswith("/"):
                uri1=url.split("//")[0]
                uri2=url.split("//")[1]
                uri3=uri2.split("/")[0]
                uri=f'{uri1}//{uri3}'
                uri=f'{uri}{url0}'
                print(uri)                
            else:
                uri=url0
            node1['LINKS'].append(uri)    
            node1['TREE'].append({"URL":uri,"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","LINKS":[],"TREE":[]})    
            node2['TREE'].append({"URL":uri,"LINKS":[],"TREE":[]}) 
            node2['LINKS'].append(uri)    
            
            #out.append({"URL":p.get('href'),"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","TREE":[]})
    
    else:
        print("NO")
        pass
    return node1,node2
#https://huggingface.co/spaces/Omnibus/crawl

def sitemap(url,level):
    uri=""
    uri0=""
    if url != "" and url != None:    
        link1,link2=link_find(url)
        if level >=2:
            for i,ea in enumerate(link1['TREE']):
                print(ea)
                try:
                    #if not ea['URL'].startswith("http"):
                    #    uri1=url.split("//")[0]
                    #    uri2=url.split("//")[1]
                    #    uri3=uri2.split("/")[0]
                    #    uri=f'{uri1}//{uri3}'
                    #    print(uri)
                    out_list1,out_list2=link_find(f"{uri}{ea['URL']}")
                    link1['TREE'][i]=out_list1
                    link2['TREE'][i]=out_list2
                    #link1['TREE'].append(out_list)
                    
                    if level>=3:
                        for n,na in enumerate(link1['TREE'][i]['TREE']):
                            print(na)
                            try:
                                #if not na['URL'].startswith("http"):
                                #    uri11=url.split("//")[0]
                                #    uri22=url.split("//")[1]
                                #    uri33=uri22.split("/")[0]
                                #    uri0=f'{uri11}//{uri33}'
                                #   print(uri0)
                                out_list1,out_list2=link_find(f"{uri0}{na['URL']}")
                                link1['TREE'][i]['TREE'][n]=out_list1
                                link2['TREE'][i]['TREE'][n]=out_list2
                                #link1['TREE'][i]['TREE'].append(out_list1)
                            except Exception as e:
                                print (e)
                except Exception as e:
                    print (e)
    return link1,link2



def sitemap_OG(url,level):
    uri=""
    if url != "" and url != None:    
        link1=link_find(url)
        if level >=2:
            for i,ea in enumerate(link1):
                print(ea)
                try:
                    if not ea['URL'].startswith("http"):
                        uri1=url.split("//")[0]
                        uri2=url.split("//")[1]
                        uri3=uri2.split("/")[0]
                        uri=f'{uri1}//{uri3}'
                        print(uri)
                    out_list=link_find(f"{uri}{ea['URL']}")
                    link1[i]['TREE']=out_list
                    if level>=3:
                        for n,na in enumerate(link1[i]['TREE']):
                            print(na)
                            try:
                                if not na['URL'].startswith("http"):
                                    uri11=url.split("//")[0]
                                    uri22=url.split("//")[1]
                                    uri33=uri22.split("/")[0]
                                    uri0=f'{uri11}//{uri33}'
                                    print(uri0)
                                out_list1=link_find(f"{uri0}{na['URL']}")
                                link1[i]['TREE'][n]['TREE']=out_list1
                            except Exception as e:
                                print (e)
                except Exception as e:
                    print (e)
    return link1
with gr.Blocks() as app:
    with gr.Row():
        with gr.Column(scale=3):
            with gr.Row():
                inp=gr.Textbox(label="URL")
                level=gr.Slider(minimum=1,maximum=2,step=1,value=1)
            btn=gr.Button()
            outp=gr.JSON()
        with gr.Column(scale=1):
            outmap=gr.JSON()
    btn.click(sitemap,[inp,level],[outp,outmap])
app.launch()