Update app.py

app.py

@@ -21,10 +21,18 @@ def link_find(url):
 
         q=("a","p","span","content","article")
         for p in soup.find_all("a"):
-            node1['LINKS'].append(p.get('href'))    
-            node1['TREE'].append({"URL":p.get('href'),"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","LINKS":[],"TREE":[]})    
-            node2['TREE'].append({"URL":p.get('href'),"LINKS":[],"TREE":[]}) 
-            node2['LINKS'].append(p.get('href'))    
+            url0=p.get('href')
+            if not url0.startswith("http"):
+                uri1=url0.split("//")[0]
+                uri2=url0.split("//")[1]
+                uri3=uri2.split("/")[0]
+                uri=f'{uri1}//{uri3}'
+                print(uri)
+            
+            node1['LINKS'].append(uri)    
+            node1['TREE'].append({"URL":uri,"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","LINKS":[],"TREE":[]})    
+            node2['TREE'].append({"URL":uri,"LINKS":[],"TREE":[]}) 
+            node2['LINKS'].append(uri)    
             
             #out.append({"URL":p.get('href'),"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","TREE":[]})
     
@@ -36,18 +44,19 @@ def link_find(url):
 
 def sitemap(url,level):
     uri=""
+    uri0=""
     if url != "" and url != None:    
         link1,link2=link_find(url)
         if level >=2:
             for i,ea in enumerate(link1['TREE']):
                 print(ea)
                 try:
-                    if not ea['URL'].startswith("http"):
-                        uri1=url.split("//")[0]
-                        uri2=url.split("//")[1]
-                        uri3=uri2.split("/")[0]
-                        uri=f'{uri1}//{uri3}'
-                        print(uri)
+                    #if not ea['URL'].startswith("http"):
+                    #    uri1=url.split("//")[0]
+                    #    uri2=url.split("//")[1]
+                    #    uri3=uri2.split("/")[0]
+                    #    uri=f'{uri1}//{uri3}'
+                    #    print(uri)
                     out_list1,out_list2=link_find(f"{uri}{ea['URL']}")
                     link1['TREE'][i]=out_list1
                     link2['TREE'][i]=out_list2
@@ -57,12 +66,12 @@ def sitemap(url,level):
                         for n,na in enumerate(link1['TREE'][i]['TREE']):
                             print(na)
                             try:
-                                if not na['URL'].startswith("http"):
-                                    uri11=url.split("//")[0]
-                                    uri22=url.split("//")[1]
-                                    uri33=uri22.split("/")[0]
-                                    uri0=f'{uri11}//{uri33}'
-                                    print(uri0)
+                                #if not na['URL'].startswith("http"):
+                                #    uri11=url.split("//")[0]
+                                #    uri22=url.split("//")[1]
+                                #    uri33=uri22.split("/")[0]
+                                #    uri0=f'{uri11}//{uri33}'
+                                #   print(uri0)
                                 out_list1,out_list2=link_find(f"{uri0}{na['URL']}")
                                 link1['TREE'][i]['TREE'][n]=out_list1
                                 link2['TREE'][i]['TREE'][n]=out_list2