PyQT项目优化---添加多线程数控制

就是筑龙网的爬虫项目优化，添加多线程控制数，避免卡顿

# -*- coding: utf-8 -*-# Form implementation generated from reading ui file 'ZL.ui'
#
# Created by: PyQt5 UI code generator 5.13.0
#
# WARNING! All changes made in this file will be lost!from PyQt5 import QtCore, QtGui, QtWidgets
import sys
from PyQt5.QtWidgets import QApplication,QMainWindow,QMessageBox,QFileDialog
import time
import requests
import _thread
from PyQt5.QtGui import *
from PyQt5.QtCore import pyqtSignal,QObject
from PyQt5.QtWidgets import *
import threadingLogin_Cookies = 'eduumuid=69ed6e952f8f70234c1004fd7dc14038; accessId=8dca1f50-97fb-11e9-99e2-5fd95b30638f; pcid=10794842; redirecturl=https%3A//www.zhulong.com/; ZLID=8c37VwQUnH9KaBdWS8zBopbxKW-vxS-FV97b4EQlL3FG5o2sFePJcKH4KceLZtBW; uid=10794842; username1=%D2%BB%C3%CE%C1%BD%C8%FD%C4%EA; isCorp=0; username=%E4%B8%80%E6%A2%A6%E4%B8%A4%E4%B8%89%E5%B9%B4; specialty_id=1; UM_distinctid=16e38f2c7291ee-0c11be12911fc3-7711b3e-1fa400-16e38f2c72a86e; userid=10794842; token=dXNlcmlkPTEwNzk0ODQyJmFjdGl2YXRlPTE%3D.17c9b0d8930c264246613246e651ebdd.1573518264; bad_id8dca1f50-97fb-11e9-99e2-5fd95b30638f=9f4296d1-ff62-11e9-822e-b3380482fef0; __root_domain_v=.zhulong.com; _qddaz=QD.l6w1jq.3ldlhu.k2lq1yr2; Hm_lvt_bec1297fa47df126a5cd56f7e753d377=1572951810; active_2019116=1; as=%7B%223619%22%3A1573007757%7D; CNZZDATA1547351=cnzz_eid%3D1470027985-1572850933-https%253A%252F%252Fpassport.zhulong.com%252F%26ntime%3D1573022521; Hm_lvt_49541358a94eea717001819b500f76c8=1572957374,1573002562,1573007755,1573024976; Hm_lpvt_49541358a94eea717001819b500f76c8=1573024976; CNZZDATA5078536=cnzz_eid%3D1290042964-1572852079-https%253A%252F%252Fpassport.zhulong.com%252F%26ntime%3D1573020387; qimo_seosource_8dca1f50-97fb-11e9-99e2-5fd95b30638f=%E7%AB%99%E5%86%85; qimo_seokeywords_8dca1f50-97fb-11e9-99e2-5fd95b30638f=; href=https%3A%2F%2Fwww.zhulong.com%2F; pageViewNum=31; Hm_lvt_d165b0df9d8b576128f53e461359a530=1572957375,1573002563,1573007755,1573024977; Hm_lpvt_d165b0df9d8b576128f53e461359a530=1573024977; nice_id8dca1f50-97fb-11e9-99e2-5fd95b30638f=409aea61-0066-11ea-b383-1d483339e6a8' #登录Cookie
#Login_Cookies =''
DOWN_URL = ''sem = threading.Semaphore(4)  # 设定计时器 我感觉 #全局总的信号量def convertCookie(cookie_val):# 转成字典格式return dict(map(lambda x: x.split('='), cookie_val.split(";")))def formatFloat(num):return '{:.2f}'.format(num)class Ui_MainWindow(QObject):signal_single_link = pyqtSignal(str,str,str)  #定义下载单条链接的信号参数(文件名，文件大小，文件进度)signal_success_down = pyqtSignal()signal_show_partinfo = pyqtSignal(str,str)signal_show_all = pyqtSignal(str,str,str,str) #显示所有信息signal_show_success = pyqtSignal(str, str, str, str)  # 显示所有信息def __init__(self):self.table_index = 0 #默认项行数为0开始self.download_url = ''self.file_path = ''self.url_list = [] #下载的链接列表self.download_nums = 0 #下载总数self.thread_poll = [] #线程池super(Ui_MainWindow, self).__init__()#定义测试信号def setupUi(self, MainWindow):MainWindow.setObjectName("MainWindow")MainWindow.resize(700, 600)MainWindow.setMinimumSize(QtCore.QSize(700, 600))MainWindow.setMaximumSize(QtCore.QSize(700, 600))font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)MainWindow.setFont(font)MainWindow.setStyleSheet("")self.centralwidget = QtWidgets.QWidget(MainWindow)self.centralwidget.setObjectName("centralwidget")self.horizontalLayoutWidget = QtWidgets.QWidget(self.centralwidget)self.horizontalLayoutWidget.setGeometry(QtCore.QRect(10, 10, 681, 41))self.horizontalLayoutWidget.setObjectName("horizontalLayoutWidget")self.horizontalLayout = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget)self.horizontalLayout.setContentsMargins(0, 0, 0, 0)self.horizontalLayout.setSpacing(10)self.horizontalLayout.setObjectName("horizontalLayout")self.label = QtWidgets.QLabel(self.horizontalLayoutWidget)font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)self.label.setFont(font)self.label.setObjectName("label")self.horizontalLayout.addWidget(self.label)self.lineEdit_Cookies = QtWidgets.QLineEdit(self.horizontalLayoutWidget)self.lineEdit_Cookies.setEnabled(True)font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)self.lineEdit_Cookies.setFont(font)self.lineEdit_Cookies.setObjectName("lineEdit_Cookies")self.lineEdit_Cookies.setText(Login_Cookies)self.horizontalLayout.addWidget(self.lineEdit_Cookies)self.horizontalLayoutWidget_2 = QtWidgets.QWidget(self.centralwidget)self.horizontalLayoutWidget_2.setGeometry(QtCore.QRect(10, 60, 681, 41))self.horizontalLayoutWidget_2.setObjectName("horizontalLayoutWidget_2")self.horizontalLayout_2 = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget_2)self.horizontalLayout_2.setContentsMargins(0, 0, 0, 0)self.horizontalLayout_2.setObjectName("horizontalLayout_2")self.label_2 = QtWidgets.QLabel(self.horizontalLayoutWidget_2)font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)self.label_2.setFont(font)self.label_2.setObjectName("label_2")self.horizontalLayout_2.addWidget(self.label_2)self.lineEdit_SingleLink = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)self.lineEdit_SingleLink.setFont(font)self.lineEdit_SingleLink.setStyleSheet("")self.lineEdit_SingleLink.setObjectName("lineEdit_SingleLink")self.lineEdit_SingleLink.setText(DOWN_URL)self.horizontalLayout_2.addWidget(self.lineEdit_SingleLink)self.btn_DownloadSingleLink = QtWidgets.QPushButton(self.horizontalLayoutWidget_2)font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)self.btn_DownloadSingleLink.setFont(font)self.btn_DownloadSingleLink.setObjectName("btn_DownloadSingleLink")self.btn_DownloadSingleLink.clicked.connect(self.DownloadSingleLink)self.horizontalLayout_2.addWidget(self.btn_DownloadSingleLink)self.horizontalLayoutWidget_3 = QtWidgets.QWidget(self.centralwidget)self.horizontalLayoutWidget_3.setGeometry(QtCore.QRect(10, 110, 681, 41))self.horizontalLayoutWidget_3.setObjectName("horizontalLayoutWidget_3")self.horizontalLayout_3 = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget_3)self.horizontalLayout_3.setContentsMargins(0, 0, 0, 0)self.horizontalLayout_3.setObjectName("horizontalLayout_3")self.label_3 = QtWidgets.QLabel(self.horizontalLayoutWidget_3)font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)self.label_3.setFont(font)self.label_3.setObjectName("label_3")self.horizontalLayout_3.addWidget(self.label_3)self.btn_OpenTxtPath = QtWidgets.QPushButton(self.horizontalLayoutWidget_3)self.btn_OpenTxtPath.setObjectName("btn_OpenTxtPath")self.btn_OpenTxtPath.clicked.connect(self.OpenTxtPath)self.horizontalLayout_3.addWidget(self.btn_OpenTxtPath)self.label_txt_path = QtWidgets.QLabel(self.horizontalLayoutWidget_3)self.label_txt_path.setObjectName("label_txt_path")self.horizontalLayout_3.addWidget(self.label_txt_path)spacerItem = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)self.horizontalLayout_3.addItem(spacerItem)#self.btn_SetThreadNum = QtWidgets.QPushButton(self.horizontalLayoutWidget_3)#self.btn_SetThreadNum.setObjectName("btn_SetThreadNum")#self.horizontalLayout_3.addWidget(self.btn_SetThreadNum)self.horizontalLayoutWidget_5 = QtWidgets.QWidget(self.centralwidget)self.horizontalLayoutWidget_5.setGeometry(QtCore.QRect(10, 560, 681, 41))self.horizontalLayoutWidget_5.setObjectName("horizontalLayoutWidget_5")self.horizontalLayout_5 = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget_5)self.horizontalLayout_5.setContentsMargins(0, 0, 0, 0)self.horizontalLayout_5.setObjectName("horizontalLayout_5")spacerItem1 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)self.horizontalLayout_5.addItem(spacerItem1)self.btn_OneDownload = QtWidgets.QPushButton(self.horizontalLayoutWidget_5)self.btn_OneDownload.setObjectName("btn_OneDownload")self.horizontalLayout_5.addWidget(self.btn_OneDownload)spacerItem2 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)self.horizontalLayout_5.addItem(spacerItem2)self.verticalLayoutWidget = QtWidgets.QWidget(self.centralwidget)self.verticalLayoutWidget.setGeometry(QtCore.QRect(10, 159, 681, 391))self.verticalLayoutWidget.setObjectName("verticalLayoutWidget")self.verticalLayout = QtWidgets.QVBoxLayout(self.verticalLayoutWidget)self.verticalLayout.setContentsMargins(0, 0, 0, 0)self.verticalLayout.setObjectName("verticalLayout")self.tableWidget = QtWidgets.QTableWidget(self.verticalLayoutWidget)font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)self.tableWidget.setFont(font)self.tableWidget.setObjectName("tableWidget")self.tableWidget.setColumnCount(4)self.tableWidget.setRowCount(10000)self.tableWidget.verticalHeader().hide()self.tableWidget.horizontalHeader().setSectionResizeMode(QHeaderView.Fixed)self.tableWidget.setColumnWidth(0,80)self.tableWidget.setColumnWidth(1,230)self.tableWidget.setColumnWidth(2,130)self.tableWidget.setColumnWidth(3,220)item = QtWidgets.QTableWidgetItem()font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)item.setFont(font)self.tableWidget.setHorizontalHeaderItem(0, item)item = QtWidgets.QTableWidgetItem()font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)item.setFont(font)self.tableWidget.setHorizontalHeaderItem(1, item)item = QtWidgets.QTableWidgetItem()font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)item.setFont(font)self.tableWidget.setHorizontalHeaderItem(2, item)item = QtWidgets.QTableWidgetItem()font = QtGui.QFont()font.setFamily("Consolas")font.setPointSize(10)item.setFont(font)self.tableWidget.setHorizontalHeaderItem(3, item)self.verticalLayout.addWidget(self.tableWidget)self.label_4 = QtWidgets.QLabel(self.verticalLayoutWidget)self.label_4.setObjectName("label_4")self.verticalLayout.addWidget(self.label_4)MainWindow.setCentralWidget(self.centralwidget)self.retranslateUi(MainWindow)QtCore.QMetaObject.connectSlotsByName(MainWindow)self.signal_single_link.connect(self.showSingalInfo)self.signal_success_down.connect(self.showDownSuccess)self.signal_show_partinfo.connect(self.showPartialTableInfo)self.tableWidget.setEditTriggers(QAbstractItemView.NoEditTriggers)self.btn_OneDownload.clicked.connect(self.OneDownload)self.signal_show_all.connect(self.showAllInfo)def showAllInfo(self,index_num,file_name,file_size,file_progress):index_item = QTableWidgetItem(index_num)file_name_item = QTableWidgetItem(file_name)file_size_item = QTableWidgetItem(file_size)file_progress_item = QTableWidgetItem(file_progress)self.tableWidget.setItem(int(index_num),0,index_item)self.tableWidget.setItem(int(index_num),1,file_name_item)self.tableWidget.setItem(int(index_num),2,file_size_item)self.tableWidget.setItem(int(index_num),3,file_progress_item)def showSingalInfo(self,file_name,file_size,file_progress):self.label_txt_path.setText('文件名：'+file_name+'文件大小:'+file_size+'文件进度:'+file_progress)def showDownSuccess(self):self.label_txt_path.setText('下载完成！')def showPartialTableInfo(self,index_num,cur_file_name):# 显示名字和序号index_item = QTableWidgetItem(index_num)file_item = QTableWidgetItem(cur_file_name)self.tableWidget.setItem(int(index_num),0,index_item)self.tableWidget.setItem(int(index_num),1,file_item)def retranslateUi(self, MainWindow):_translate = QtCore.QCoreApplication.translateMainWindow.setWindowTitle(_translate("MainWindow", "筑龙批量下载工具"))self.label.setText(_translate("MainWindow", "请输入登录Cookies："))self.label_2.setText(_translate("MainWindow", "请输入单条链接："))self.btn_DownloadSingleLink.setText(_translate("MainWindow", "点击下载"))self.label_3.setText(_translate("MainWindow", "请选择txt文件："))self.btn_OpenTxtPath.setText(_translate("MainWindow", "点击打开txt文件"))self.label_txt_path.setText(_translate("MainWindow", "显示txt文件路径"))#self.btn_SetThreadNum.setText(_translate("MainWindow", "设置线程数"))self.btn_OneDownload.setText(_translate("MainWindow", "一键下载"))self.tableWidget.setHorizontalHeaderLabels(['序号','文件名','文件大小','文件进度'])self.label_4.setText(_translate("MainWindow", "          tips：登录Cookies为必填项，默认线程数为5,一次下载最多10000条数据！"))def DownloadSingleLink(self):# brief:下载单条链接self.cookie = self.lineEdit_Cookies.text().strip()if self.cookie == '':QMessageBox.warning(None,'下载单条链接','cookie为空，请输入cookie后再下载！')returnif self.cookie != '':cookie_str = convertCookie(self.cookie)self.download_url = self.lineEdit_SingleLink.text().strip()if self.download_url !='':download_list = self.download_url.split('aid=')download_name = download_list[len(download_list)-1]+'.pdf'_thread.start_new_thread(self.downloadFile,(download_name,self.download_url,cookie_str))#Pdf下载def OpenTxtPath(self):#打开文件路径fileName, filetype = QFileDialog.getOpenFileName(None, "选择文件", "/", "Text Files (*.txt)")if fileName == '':QMessageBox.warning(None,'未选择文件','请重新选择txt文件')returnelse:self.file_path = fileName # 设置文件名self.label_txt_path.setText(fileName) #在文件路径上选择f = open(fileName)index_num_cur = 0#遍历》》》》显示到前端for line in f:line_str = line.strip()if line_str == '':continueline_sp = line_str.split('aid=')line_name = line_sp[len(line_sp)-1]+'.pdf'self.signal_show_partinfo.emit(str(index_num_cur),line_name)index_num_cur = index_num_cur + 1f.close()def downloadFile(self,name, url, login_cookie):''':param name:下载保存的名称:param url: 下载链接:return:'''headers = {'Proxy-Connection': 'keep-alive'}r = requests.get(url, stream=True, headers=headers, cookies=login_cookie)length = float(r.headers['content-length'])Download_size_M = length / 1048576file_size = str(round(Download_size_M, 1)) + 'M'f = open(name, 'wb')count = 0count_tmp = 0time1 = time.time()for chunk in r.iter_content(chunk_size=512):if chunk:f.write(chunk)count += len(chunk)if time.time() - time1 > 2:p = count / length * 100speed = (count - count_tmp) / 1024 / 1024 / 2count_tmp = countprint(name + ': ' + formatFloat(p) + '%' + ' Speed: ' + formatFloat(speed) + 'M/S')self.signal_single_link.emit(str(name),str(file_size),str(formatFloat(p))+'%')time1 = time.time()f.close()self.signal_success_down.emit()def downloadMoreFile(self,index,name, url, login_cookie):'''# 批量下载 index序号:param name:下载保存的名称:param url: 下载链接:return:'''headers = {'Proxy-Connection': 'keep-alive'}r = requests.get(url, stream=True, headers=headers, cookies=login_cookie)length = float(r.headers['content-length'])Download_size_M = length / 1048576file_size = str(round(Download_size_M, 1)) + 'M'f = open(name, 'wb')count = 0count_tmp = 0time1 = time.time()for chunk in r.iter_content(chunk_size=512):if chunk:f.write(chunk)count += len(chunk)if time.time() - time1 > 2:p = count / length * 100speed = (count - count_tmp) / 1024 / 1024 / 2count_tmp = countprint(name + ': ' + formatFloat(p) + '%' + ' Speed: ' + formatFloat(speed) + 'M/S')self.signal_show_all.emit(str(index),str(name),str(file_size),str(formatFloat(p))+'%')time1 = time.time()f.close()self.signal_show_all.emit(str(index), str(name), str(file_size),'100%')sem.release()#def AnotherThreadToCreateDown(self,indexcur_nums,url_list,cookie,downloadMoreFile):for url in url_list:url_sp = url.split('aid=')url_name = url_sp[len(url_sp) - 1] + '.pdf'cur_cookie = convertCookie(cookie)th = threading.Thread(target=downloadMoreFile, args=(indexcur_nums, url_name, url, cur_cookie))th.start()indexcur_nums = indexcur_nums + 1#self.thread_poll.append(th)sem.acquire()def OneDownload(self):# 意见下载self.cookie = self.lineEdit_Cookies.text().strip()if self.cookie ==  '':QMessageBox.about(None,'Cookie为空','Cookie为空，无法下载')returnif self.file_path == '':QMessageBox.about(None,'错误','未加载txt文件，无法下载')returncur_file_path =self.file_path #文件名fs = open(cur_file_path) #打开该txt文件for line in fs:#print(line) # line为链接line_str = line.strip()print(line_str)if line_str == '':returnself.url_list.append(line_str)#添加链接到链接列表fs.close()self.download_nums = len(self.url_list) #下载总数index_nums = 0_thread.start_new_thread(self.AnotherThreadToCreateDown,(index_nums,self.url_list,self.cookie,self.downloadMoreFile))if __name__ == '__main__':#加密程序：res = requests.get('https://duhuifeng.github.io/')if res.text.strip() == '1':app = QApplication(sys.argv)MainWindow = QMainWindow()ui = Ui_MainWindow()ui.setupUi(MainWindow)MainWindow.show()sys.exit(app.exec_())else:print('error!')

运行结果：

可以明显的看出，此次的可以控制下载的线程数，一方面可以解决电脑卡顿的问题，另一方面，python的运行速度是真的有点小慢。

PyQT项目优化---添加多线程数控制相关推荐

前端学习（2735）：重读vue电商网站45之项目优化上线
优化Vue项目,部署Vue项目项目优化添加进度条 nprogress 进度条官方文档传送门打开 vue-ui 面板,选择依赖,输入 nprogress 进行安装相关依赖. npm Javascr ...
Shell多线程操作及线程数控制实例
来源:http://www.jb51.net/article/51720.htm 这篇文章主要介绍了Shell多线程操作及线程数控制实例,文中从单线程实现一个需求开始,不断加入代码实现多线程以及线程数 ...
linux脚本多线程,Shell多线程操作及线程数控制实例
前言在业务开发过程中,经常会在后台写一些shell脚本处理数据,但估计很多人不知道shell脚本也可以支持多线程,而且非常简单.本篇文章主要就是介绍shell实现多进程以及进程数量控制. 需求为了 ...
VR项目添加键盘鼠标控制
VR项目添加键盘鼠标控制使用VR模版创建的项目,程序运行后,只能使用VR头盔进行交互场景,不便于调试工作,为此,在场景中添加键盘.鼠标控制功能. 1 .键盘控制添加: 键盘控制主要为ASDW 键控制 ...
JavaScript 大揭秘：React、性能优化以及多线程
@开发者,在后端仅提供原始数据集的情况下,如何让所有搜索逻辑都在前端完成?不仅如此,还要能够实现预测用户输入.忽略错别字输入.快速自动补齐等智能功能?本文的作者就深入 JavaScript 技术特性, ...
Unity项目优化详解（持续补充ing）
Unity开发项目总结的几项优化点,比较适合中小项目优化,拿来即用,大型项目需要考虑定制化渲染管线.剔除.光照等.针对优化更多的还是需要结合项目去考虑. 一.模型 Read/Write:同Textur ...
Vue-cli 项目优化归纳(打包、源码、用户体验)
前言:vue-cli项目开发打包部署后,存在问题有首次首页加载过慢,包括加载缓慢问题,需要进行vue项目优化.下面是对vue性能优化方法进行归纳,后面会对方法进行亲测. 主要包括:代码包打包优化.编码 ...
Java开发技术大杂烩(三）之电商项目优化、rabbitmq、Git、OSI、VIM、Intellj IDEA、HTTP、JS、Java...
前言最近公司让我维护Spring+Servlet+Hibernate+Spring Security+Jsp的老项目,正好可以锻炼我的业务逻辑和掌控项目的能力.虽然项目很老,但是其中还是有很多值我学 ...
【uni-app】uniapp项目优化方式及建议
uniapp项目优化方式及建议介绍:性能优化自古以来就是重中之重,关于uniapp项目优化方式最全整理,会根据开发情况进行补充 1.复杂页面数据区域封装成组件场景: 例如项目里包含类似论坛页面:点 ...

PyQT项目优化---添加多线程数控制

PyQT项目优化---添加多线程数控制相关推荐

最新文章

热门文章