from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.common.exceptions import TimeoutException
import time
import datetime
from urllib import parse
import osurls = set([])
def getUrl(url,host):driver = webdriver.Ie()try:#driver = webdriver.Firefox()driver.set_page_load_timeout(10)driver.get(url)#time.sleep(2)all_window_height = []all_window_height.append(driver.execute_script("return document.body.scrollHeight;"))while True:driver.execute_script("scroll(0,100000)")time.sleep(1)check_height = driver.execute_script("return document.body.scrollHeight;")if check_height == all_window_height[-1]:print("我已下拉完毕")breakelse:all_window_height.append(check_height) print("我正在下拉")#for link in driver.find_elements_by_xpath("//*[@href]"):#for link in driver.find_elements_by_tag_name("a"):for link in driver.find_elements_by_xpath("//a[@href]"):try:tempurl1=link.get_attribute('href')if tempurl1.startswith("http"):if tempurl1 not in urls:urls.add(tempurl1)log(host,url+','+tempurl1)print(tempurl1)except:print(link)except Exception as e:print(e)finally:driver.quit()def log(name,msg):filename='D://'+name+'.csv'if not os.path.exists(filename):with open(filename,'w') as f:print('create file:'+filename)f.write('parentUrl,currenturl'+'\n')f.close()with open(filename,'a') as f:f.write(msg+'\n')f.close()url= input("Enter a url")
try:urls.clear()url= url.strip()if len(url)>0:host =parse.urlparse(url).netlocprint(url+"下面的连接:")t1=datetime.datetime.now()getUrl(url,host)l=list(urls)for item in l:print(item+"下面的连接:")getUrl(item,host)t2=datetime.datetime.now()tt =(t2-t1).secondsminutes=tt//60seconds=tt%60print("total cost %d minutes %d seconds" % (minutes,seconds))except Exception as e:print(e)