# 访问网站操作代码
def  runjob():# m, s = divmod(second, 60)# h, m = divmod(m, 60)# hms = "{:02}:{:02}:{:02}".format(h, m, s)# logger.info(hms)#a、 如果人工停止刷量，则直接结束当前线程，并刷新界面if (myframe.stop_refresh_page_thread):# 增加刷量addrefreshnum()# 刷新页面myframe.refresh_run_stop_button()return# b、正常执行任务refreshnum = myframe.refreshnumsiteurl = myframe.siteurlinput.GetValue().strip()my_logger_info(logger,"==开始网站%s第%d次刷量<直接访问>=="%(siteurl,refreshnum+1))try:# 创建浏览器driver = createWebDriver()# 浏览器最大化driver.maximize_window()# 浏览网站viewSite(driver,siteurl)except Exception as e:info = traceback.format_exc()my_logger_info(logger,info)driver.save_screenshot(".\\refreshpage_directvisit_error.png")myframe.stop_refresh_page_thread=Truefinally:# 关闭浏览器driver.quit()# 增加刷量addrefreshnum()# 正常执行后刷新界面myframe.refresh_run_stop_button()# 打印日志my_logger_info(logger,"==完成网站%s第%d次刷量<直接访问>=="%(siteurl,refreshnum+1))

#创建浏览器驱动
def createWebDriver():# 配置参数options = webdriver.ChromeOptions()# 设置网页编码options.add_argument('lang=zh_CN.UTF-8')# 禁止加载图片options.add_argument('blink-settings=imagesEnabled=false')# 禁用sandboxoptions.add_argument('--no-sandbox')# 无界面模式options.add_argument('headless')driver = webdriver.Chrome(options=options, keep_alive=True)# 防止selenium访问被识别出来，不算流量driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": """Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"""})return driver

# 模拟浏览网站
def  viewSite(driver,url):my_logger_info(logger,"<<开始访问网站:%s"% url)driver.get(url)pagesource = driver.page_sourcerunIdleSomeTime(random.randint(3, 5))linklist = []linklist.append("桥架国标")linklist.append("桥架价格")linklist.append("桥架安装")prodlist = []prodlist.append("桥架配件")prodlist.append("桥架规格")for i in range(len(prodlist)):# 支持人工停止刷量if (myframe.stop_refresh_page_thread):breaklanmu = prodlist[i]viewProductOfLanmu(driver, lanmu)for i in range(len(linklist)):# 支持人工停止刷量if (myframe.stop_refresh_page_thread):breaklanmu = linklist[i]viewArticleOfLanmu(driver, lanmu)# 支持人工停止刷量if (myframe.stop_refresh_page_thread):my_logger_info(logger, "已经停止刷量")my_logger_info(logger,">>完成访问网站:%s" % url)

# 查看栏目产品
def viewProductOfLanmu(driver,lanmu):# 浏览相关栏目link_d = driver.find_element_by_link_text(lanmu)# 不直接使用link.click()，避免被其他咨询窗口遮挡driver.execute_script("arguments[0].click();", link_d)# 等待栏目第一页加载完成runIdleSomeTime(random.randint(3, 5))# 支持人工停止刷量if (myframe.stop_refresh_page_thread):returnpagesource = driver.page_sourcesoup = BeautifulSoup(pagesource, "html.parser")# logger.debug(soup.prettify())while True:# 查看当前页所有文章newsdiv_s = soup.find("div", class_="list").find_all("div", class_="mask")for i in range(len(newsdiv_s)):link=newsdiv_s[i].find("a")my_logger_info(logger,"访问页面：%s" % link['href'])# 在新的窗口打开文章js = "window.open('" + link['href'] + "','_blank');"try:driver.execute_script(js)except Exception as e:info = traceback.format_exc()my_logger_info(logger, info)continue# driver.implicitly_wait(3)# 查看打开的文章内容runIdleSomeTime(random.randint(5, 7))# 支持人工停止刷量if (myframe.stop_refresh_page_thread):break# driver.close()# 获取当前打开的所有窗口windows = driver.window_handles# 转换到最新打开的窗口driver.switch_to.window(windows[-1])driver.close()# 转换到父窗口driver.switch_to.window(windows[0])# 在当前页浏览文章runIdleSomeTime(random.randint(1, 3))# 支持人工停止刷量if (myframe.stop_refresh_page_thread):break# 支持人工停止刷量if (myframe.stop_refresh_page_thread):break# 判断是否有下一页pagediv_s = soup.find("div", class_="pageBox")nextpagelink_s=pagediv_s.find("a",text="下一页")if (not nextpagelink_s):break# 尝试翻转到下一页,翻页失败则不再进行nextpagelink_d = driver.find_element_by_link_text("下一页")driver.execute_script("arguments[0].click();", nextpagelink_d)# 等待栏目当前页加载完成runIdleSomeTime(random.randint(3, 5))pagesource = driver.page_sourcesoup = BeautifulSoup(pagesource, "html.parser")

# 查看栏目文章
def viewArticleOfLanmu(driver,lanmu):# 浏览相关栏目link_d = driver.find_element_by_link_text(lanmu)# 不直接使用link.click()，避免被其他咨询窗口遮挡driver.execute_script("arguments[0].click();", link_d)# 等待栏目第一页加载完成runIdleSomeTime(random.randint(3, 5))# 支持人工停止刷量if (myframe.stop_refresh_page_thread):returnpagesource = driver.page_sourcesoup = BeautifulSoup(pagesource, "html.parser")# logger.debug(soup.prettify())while True:# 查看当前页所有文章newsdiv_s = soup.find("div", class_="newsList")for link in newsdiv_s.find_all("a", class_="look"):my_logger_info(logger,"访问页面：%s" % link['href'])# 在新的窗口打开文章js = "window.open('" + link['href'] + "','_blank');"driver.execute_script(js)# driver.implicitly_wait(3)# 查看打开的文章内容runIdleSomeTime(random.randint(5, 7))# 支持人工停止刷量if (myframe.stop_refresh_page_thread):break# driver.close()# 获取当前打开的所有窗口windows = driver.window_handles# 转换到最新打开的窗口driver.switch_to.window(windows[-1])driver.close()# 转换到父窗口driver.switch_to.window(windows[0])# 在当前页浏览文章runIdleSomeTime(random.randint(5, 7))# 支持人工停止刷量if (myframe.stop_refresh_page_thread):break# 支持人工停止刷量if (myframe.stop_refresh_page_thread):break# 判断是否有下一页pagediv_s = soup.find("div", class_="pageBox")nextpagelink_s=pagediv_s.find("a",text="下一页")if (not nextpagelink_s):break# 尝试翻转到下一页,翻页失败则不再进行nextpagelink_d = driver.find_element_by_link_text("下一页")driver.execute_script("arguments[0].click();", nextpagelink_d)# 等待栏目当前页加载完成runIdleSomeTime(random.randint(3, 5))pagesource = driver.page_sourcesoup = BeautifulSoup(pagesource, "html.parser")