操作excel文件爬取nvd.nist数据

#!/usr/bin/env python
# encoding: utf-8
#@author: jack
import random
from time import sleep
import pandas as pd
from openpyxl import load_workbook
from urllib import request
from lxml import etreewb = load_workbook('cve.xlsx')#要读取的excel文件名，用openpyxl考虑到wrxl库网友反馈对excel后面版本兼容不是太好
sheet = wb['Sheet1']#默认excel右下角表名称
cve_list =[]
for i in sheet["D"][1:25]:#第D列第一行开始读起到24结束cve_code = i.value#读取到的每个列表参数cve_list.append(cve_code)#前面定义的空列表来存放excel读取的数据start_url = 'https://nvd.nist.gov/vuln/detail/'#老美网站待爬数据，右击页面看到数据是静态的爬起来舒服score_li=[]vector3_li=[]vector2_li=[]for url_code in cve_list:#取列表参数url = '{}{}'.format(start_url,url_code)#url拼接response = request.urlopen(url)result = response.read().decode()html = etree.HTML(result)v3BaseScore = html.xpath('//span[@data-testid="vuln-cvssv3-base-score"]/text()')#etree定位so easyVector3 = html.xpath('//span[@data-testid="vuln-cvssv3-vector"]/text()')Vector2 = html.xpath('//span[@data-testid="vuln-cvssv2-vector"]/text()')score_li.append(' '.join(v3BaseScore))#格式化保存页面提取的数据vector3_li.append(' '.join(Vector3))vector2_li.append(' '.join(Vector2))df1 = pd.DataFrame({'v3BaseScore': score_li})#构建表头字段pandas方法df2 = pd.DataFrame({'Vector3': vector3_li})df3 = pd.DataFrame({'Vector2': vector2_li})All = [df1, df2, df3]writer = pd.ExcelWriter('test1.xlsx')#新建excel文件，df1.to_excel(writer, sheet_name='Sheet1', startcol=1, index=False)#指定列逐行写入数据df2.to_excel(writer, sheet_name='Sheet1', startcol=2, index=False)df3.to_excel(writer, sheet_name='Sheet1', startcol=3, index=False)writer.save()writer.close()

业务需求现学pandas和openpyxl，

1、页面分析

2、定位分析

3、数据读写分析

4、网站容易爬挂，并发调低，user-agent代理可以搞起

import base64
import random
from multiprocessing.pool import ThreadPool
import time
import pandas as pd
from openpyxl import load_workbook
from urllib import request
from lxml import etree
from proxies import *def task1():user_agent_list = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1","Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6","Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5","Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3","Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3","Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24","Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"]# count = 0header = {}header['User-Agent'] = random.choice(user_agent_list)header.update({'Host':' nvd.nist.gov','User-Agent:':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',})time.sleep(0.3)# read cve exclewb = load_workbook('cve.xlsx')sheet = wb['Sheet1']cve_list = []# part requestfor i in sheet["D"][1:]:cve_code = i.valuecve_list.append(cve_code)start_url = 'https://nvd.nist.gov/vuln/detail/'score_li = []vector3_li = []vector3_href_li = []vector2_li = []vector2_href_li = []for url_code in cve_list:url = '{}{}'.format(start_url, url_code)res = request.Request(url,headers=header)response = request.urlopen(res)result = response.read().decode()# 数据清洗部分html = etree.HTML(result)score_list = html.xpath('//span[@data-testid="vuln-cvssv3-base-score"]/text()')vector3_list = html.xpath('//span[@data-testid="vuln-cvssv3-vector"]/text()')vector2_list = html.xpath('//span[@data-testid="vuln-cvssv2-vector"]/text()')vector3_href_list = html.xpath('//span[@data-testid="vuln-cvssv3-vector"]//a/@href')vector2_href_list = html.xpath('//span[@data-testid="vuln-cvssv2-vector"]//a/@href')score_li.append(' '.join(score_list))vector3_li.append(' '.join(vector3_list))vector3_href_li.append(' '.join(vector3_href_list))vector2_li.append(' '.join(vector2_list))vector2_href_li.append(' '.join(vector2_href_list))# create some Pandas DateFrame from some datadf1 = pd.DataFrame({'CVSSv3.0BaseScore': score_li})df2 = pd.DataFrame({'CVSS v3.0 Vector': vector3_li})df3 = pd.DataFrame({'CVSS v3.0 Vector link': vector3_href_li})df4 = pd.DataFrame({'CVSS v2.0 Vector': vector2_li})df5 = pd.DataFrame({'CVSS v2.0 Vector link': vector2_href_li})All = [df1, df2, df3, df4, df5]# create a Pandas Excel writer using xlswriterwriter = pd.ExcelWriter('basescore.xlsx')df1.to_excel(writer, sheet_name='Sheet2', startcol=2, index=False)df2.to_excel(writer, sheet_name='Sheet2', startcol=3, index=False)df3.to_excel(writer, sheet_name='Sheet2', startcol=4, index=False)df4.to_excel(writer, sheet_name='Sheet2', startcol=5, index=False)df5.to_excel(writer, sheet_name='Sheet2', startcol=6, index=False)writer.save()writer.close()def process_request(self, request, spider):PROXIES = [{'ip_port': '61.160.233.8', 'user_pass': ''},{'ip_port': '125.93.149.186', 'user_pass': ''},{'ip_port': '58.38.86.181', 'user_pass': ''},{'ip_port': '119.142.86.110', 'user_pass': ''},{'ip_port': '124.161.16.89', 'user_pass': ''},{'ip_port': '61.160.233.8', 'user_pass': ''},{'ip_port': '101.94.131.237', 'user_pass': ''},{'ip_port': '219.157.162.97', 'user_pass': ''},{'ip_port': '61.152.89.18', 'user_pass': ''},{'ip_port': '139.224.132.192', 'user_pass': ''}]proxy = random.choice(PROXIES)if proxy['user_pass'] is not None:request.meta['proxy'] = "http://%s" % proxy['ip_port']encodebytes = base64.b64encode(proxy['user_pass'].encode(encoding='utf-8'))  # 注意encodebytes类型是byte,不是strencoded_user_pass = str(encodebytes, 'utf-8')request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_passelse:request.meta['proxy'] = "http://%s" % proxy['ip_port']if __name__ == '__main__':pool = ThreadPool()pool.apply_async(task1)pool.apply_async(process_request)pool.close()pool.join()

优化版

操作excel文件爬取nvd.nist数据相关推荐

Python操作Excel文件：插入一列数据
问题描述:已有Excel文件,其中包含5列数据,要求在第3列前插入一列数据,保存为新文件. 参考代码: 运行结果: --------董付国老师Python系列教材-------- 1)<Pyth ...
python操作excel及爬取美赛证书
今天美赛成绩出来了,S奖,有点遗憾.但工作还是要继续,因为要下载校队的获奖证书,所以使用python爬虫来爬取证书,省点事. python操作excel 在这里我选择pandas库来操作excel,详 ...
django通过openpyxl操作excel文件，实现批量导入数据到数据库。
1.在一些场景中经常需要管理员批量导入数据,先将数据写到excel表格中,再上传到系统上,最后将excel中的数据批量导入到系统的数据库中. 1.1.test.xlsx文件数据格式首行的名字:必须是 ...
C#在客户端和服务端操作Excel文件
一.在客户端把数据导入到Excel文件步骤 1.创建Excel application对象,打开或生成Excel文件 //服务端创建StringBuilder对象 System.Text.St ...
《崔庆才Python3网络爬虫开发实战教程》学习笔记（5）：将爬虫爬取到的数据存储到TXT，Word，Excel，Json等文件中
本篇博文是自己在学习崔庆才的<Python3网络爬虫开发实战教程>的学习笔记系列,此套教程共5章,加起来共有34节课,内容非常详细丰富!如果你也要这套视频教程的话,关注我公众号[小众技术] ...
python爬取微博数据存入数据库_Python爬取新浪微博评论数据，写入csv文件中
因为新浪微博网页版爬虫比较困难,故采取用手机网页端爬取的方式操作步骤如下: 1. 网页版登陆新浪微博 2.打开m.weibo.cn 3.查找自己感兴趣的话题,获取对应的数据接口链接 4.获取cook ...
爬虫批量保存网页html,2分钟带你学会网络爬虫：Excel批量爬取网页数据（详细图文版）...
面对网页大量的数据,有时候还要翻页,你还在一页一页地复制粘贴吗?别人需要几小时完成的任务,学会这个小技巧你只需要几分钟就能解决.快来学习使用Excel快速批量地爬取网页数据吧! 1.分析网页数据结构 ...
python 生意参谋_如何爬取生意参谋数据？是不是违规操作？
生意参谋是阿里巴巴为商家打造的一站式.个性化.可定制的商务决策体验平台.近期发现,有部分人员使用爬虫爬取生意参谋平台数据,影响商家的用户体验.到底是如何爬取生意参谋数据呢? 官方推荐淘宝.京东.拼多多 ...
对爬虫爬取到的数据进行存储
已写章节第一章网络爬虫入门第二章基本库的使用第三章解析库的使用第四章数据存储第五章动态网页的抓取文章目录已写章节第四章数据存储 4.1 文件存储 4.1.1 TXT文件存储 ...

操作excel文件爬取nvd.nist数据

操作excel文件爬取nvd.nist数据相关推荐

最新文章

热门文章