import os
import shutil
import time
from time import sleep
from framework.base.BaseFrame import BaseFrame
from sprider.business.DownLoad import DownLoad
from sprider.business.SeleniumTools import SeleniumTools
from sprider.business.SpriderTools import SpriderTools
from selenium import webdriver
from selenium.webdriver.common.by import By
from sprider.model.SpriderEntity import SpriderEntity
from sprider.access.SpriderAccess import SpriderAccesssprider_name = "ASP搜索链接源码"class CNSourceCoder:base_url = "http://www.downcode.com"base_path = "D:\\Freedom\\Sprider\\CNSource"sprider_count =111sprider_start_count =116   #ASP新闻文章源码21是大文件 从20开始first_folder_name = "ASP"max_pager = 10  # 正在采集第22页的第3个资源，共39页资源 正在采集第54页的第1个资源，共64页资源word_image_count = 5  # word插入图片数量 同时也是明细采集图片和描述的数量page_end_number = 0word_content_list = []haved_sprider_count = 0  # 已经采集的数量sprider_detail_index = 0  # 明细方法采集的数量 累加tempFileList=[]def sprider(self,sprider_name):"""ASP留言日记源码 http://www.downcode.com/sort/j_1_2_1.shtmlASP交友会员源码 http://www.downcode.com/sort/j_1_5_1.shtmlASP投票调查源码 http://www.downcode.com/sort/j_1_6_1.shtmlASP上传下载源码 http://www.downcode.com/sort/j_1_7_1.shtmlASP搜索链接源码 http://www.downcode.com/sort/j_1_8_1.shtml:param sprider_name::return:"""if sprider_name == "ASP聊天室源码":self.first_column_name = "sort"self.second_column_name = "j_1_1"elif sprider_name == "ASP搜索链接源码":self.first_column_name = "sort"self.second_column_name = "j_1_8"elif sprider_name=="ASP投票调查源码":self.first_column_name = "sort"self.second_column_name = "j_1_6"elif sprider_name=="ASP留言日记源码":self.first_column_name = "sort"self.second_column_name = "j_1_2"elif sprider_name == "ASP江湖论坛源码":self.first_column_name = "sort"self.second_column_name = "j_1_3"elif sprider_name == "ASP新闻文章源码":self.first_column_name = "sort"self.second_column_name = "j_1_4"elif sprider_name == "ASP交友会员源码":self.first_column_name = "sort"self.second_column_name = "j_1_5"elif sprider_name == "ASP上传下载源码":self.first_column_name = "sort"self.second_column_name = "j_1_7"BaseFrame().debug("开始采集" + self.first_folder_name + "...")BaseFrame().right("本次采集参数：日志路径=" + str(BaseFrame().LOG_PATH) + "")BaseFrame().right("本次采集参数：sprider_count=" + str(self.sprider_count) + "")BaseFrame().right("本次采集参数：sprider_name=" + sprider_name + "")sprider_url = self.base_url + "/{0}/{1}_1.shtml".format(self.first_column_name, self.second_column_name)# 根据栏目构建URLBaseFrame().debug("本次采集参数：sprider_url=:" + sprider_url)self.second_folder_name = str(self.sprider_count) + "个" + sprider_name  # 二级目录也是wordTitleself.merchant = int(self.sprider_start_count) // int(self.max_pager) + 1  # 起始页码用于效率采集self.file_path = self.base_path + os.sep + self.first_folder_name + os.sep + self.second_folder_nameBaseFrame().right("本次采集参数：file_path=" + self.file_path + "")# 浏览器的下载路径self.down_path = self.base_path + os.sep + self.first_folder_name+ os.sep + self.second_folder_name+ "\\Temp\\"BaseFrame().right("本次采集参数：down_path=" + self.down_path + "")# First/PPT/88个动态PPT模板/动态PPT模板self.save_path = self.base_path + os.sep + self.first_folder_name + os.sep + self.second_folder_name + os.sep + sprider_nameBaseFrame().right("本次采集参数：save_path=" + self.save_path + "")if os.path.exists(self.down_path) is True:shutil.rmtree(self.down_path)if os.path.exists(self.down_path) is False:os.makedirs(self.down_path)if os.path.exists(self.save_path) is True:shutil.rmtree(self.save_path)if os.path.exists(self.save_path) is False:os.makedirs(self.save_path)chrome_options = webdriver.ChromeOptions()diy_prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': '{0}'.format(self.down_path)}# 添加路径到selenium配置中chrome_options.add_experimental_option('prefs', diy_prefs)chrome_options.add_argument('--headless')  # 隐藏浏览器# 实例化chrome浏览器时，关联忽略证书错误driver = webdriver.Chrome(options=chrome_options)driver.set_window_size(1280, 800)  # 分辨率 1280*800driver.get(sprider_url)element_list = driver.find_elements(By.CLASS_NAME, "j_text_sort_a")# element_list = element_ul.find_elements(By.TAG_NAME, "h2")  #page_end_number = 1page_end_url = ""a_elments = driver.find_elements(By.TAG_NAME, "a")for a in a_elments:page_end_title = a.get_attribute("title")if page_end_title == "最后页":page_end_url = a.get_attribute("href")print(page_end_url)breakif page_end_url is None or page_end_url == "":page_end_number = 1else:page_end_number = (page_end_url.split(".shtml")[0].split("_")[3])self.page_count = self.merchantwhile self.page_count <= int(page_end_number):  # 翻完停止try:if self.page_count == 1:self.sprider_detail(driver,element_list, page_end_number)passelse:if self.haved_sprider_count == self.sprider_count:BaseFrame().debug("采集到达数量采集停止...")BaseFrame().debug("开始写文章...")SpriderTools.builder_word(self.second_folder_name,self.word_content_list,self.file_path,self.word_image_count,self.first_folder_name )SpriderTools.copy_file(self.word_content_list,self.save_path,self.second_folder_name,self.file_path)SpriderTools.gen_passandtxt(self.second_folder_name, self.word_content_list, self.file_path)BaseFrame().debug("文件编写完毕，请到对应的磁盘查看word文件和下载文件！")breaknext_url = self.base_url + "/{0}/{1}_{2}.shtml".format(self.first_column_name,self.second_column_name,self.page_count)driver.get(next_url)element_list = driver.find_elements(By.CLASS_NAME, "j_text_sort_a")self.sprider_detail(driver, element_list, page_end_number)passself.page_count = self.page_count + 1  # 页码增加1except Exception as e:print("sprider()执行过程出现错误:" + str(e))

import os
import shutildef void_folder(path):# 访问path路径下的文件或文件夹lst = os.listdir(path)# 打印每一层的文件或文件夹for name in lst:# 拼接名称，得到绝对路径，判断该文件是否符合是文件夹real_path = os.path.join(path, name)# 如果是文件夹，则打空格表示，并且递归访问下一层if os.path.isdir(real_path):# print(name)files = os.listdir(real_path)if len(files) == 0:print("void_folder()："+name)shutil.rmtree(real_path)endindex = len(real_path) - len(name)real_path = real_path[0:endindex]void_folder(real_path)else:void_folder(real_path)# 如果不是文件夹，直接打印，不再递归访问下一层else:#print(name)pass
def void_file(dirPath):dirs = os.listdir(dirPath)  # 查找该层文件夹下所有的文件及文件夹，返回列表for file in dirs:file_full_name = dirPath + '/' + filefile_ext = os.path.splitext(file_full_name)[-1]if file_ext is None  or file_ext=="":continueif "rar" == str(file_ext.split(".")[1]):os.remove(file_full_name)if "zip" == str(file_ext.split(".")[1]):os.remove(file_full_name)if "gz" == str(file_ext.split(".")[1]):os.remove(file_full_name)if "tgz" == str(file_ext.split(".")[1]):os.remove(file_full_name)# 查找指定文件夹下所有相同名称的文件
def search_file(dirPath, fileName):dirs = os.listdir(dirPath)  # 查找该层文件夹下所有的文件及文件夹，返回列表for currentFile in dirs:  # 遍历列表absPath = dirPath + '/' + currentFileif os.path.isdir(absPath):  # 如果是目录则递归，继续查找该目录下的文件search_file(absPath, fileName)elif currentFile == fileName:print(absPath)  # 文件存在，则打印该文件的绝对路径os.remove(absPath)