python爬虫: 爬取boss网站数据保存到mysql

爬取boss数据并保存到mysql
boss的反爬虫控制尤其恶心 cookies大概用3次左右就不能用了
所以爬取不了太多东西只可用来学习
等学习完逆向爬虫课程再来挑战挑战

example.py

import scrapy
from bs4 import BeautifulSoup
from boos_scrapy.items import BoosScrapyItem
from time import sleepclass ExampleSpider(scrapy.Spider):name = 'example'# allowed_domains = ['example.com']start_urls = ['https://www.zhipin.com/c101280100/?query=%E8%BD%AF%E4%BB%B6%E6%B5%8B%E8%AF%95%E5%B7%A5%E7%A8%8B%E5%B8%88&page=1&ka=page-1']url = "https://www.zhipin.com/c101280100/?query=%E8%BD%AF%E4%BB%B6%E6%B5%8B%E8%AF%95%E5%B7%A5%E7%A8%8B%E5%B8%88&page={0}&ka=page-{1}"page_num = 2def parse(self, response):li_list = response.xpath('//*[@class="job-list"]/ul/li')for li in li_list:job_tags = li.xpath('.//div/div[2]/div[2]/text()').extract_first()#福利待遇title = li.xpath('.//div/div[1]/div[1]/div/div[1]/span[1]/a/text()').extract_first()#招聘职位salary = li.xpath('.//div/div[1]/div[1]/div/div[2]/span/text()').extract_first() #薪资待遇particulars = li.xpath('.//div[1]/div[1]/div/div[1]/span[1]/a/@href').extract_first() #详情地址boos_url = "https://www.zhipin.com" + particulars item = BoosScrapyItem()item['job_tags'] = job_tagsitem['title'] = titleitem['salary'] = salary#回调函数yield scrapy.Request(boos_url,callback=self.parse_detail,meta={'item':item})if self.page_num <= 3:new_url = format(self.url%(self.page_num,self.page_num))self.page_num+=1yield scrapy.Request(new_url,callback=self.parse)#进行详情页爬取def parse_detail(self,response):item = response.meta['item']job_sec = ''.join(response.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[1]/div/text()').extract()).strip() #职位描述company = ''.join(response.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[2]/div/text()').extract()).strip() #公司介绍job_location = ''.join(response.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[6]/div/div[1]/text()').extract()).strip()#公司地址item['job_sec'] = job_secitem['company'] = companyitem['job_location'] = job_locationyield item

items.py

import scrapyclass BoosScrapyItem(scrapy.Item):company=scrapy.Field()job_location=scrapy.Field()job_tags=scrapy.Field()title=scrapy.Field()salary=scrapy.Field()job_sec=scrapy.Field()

middlewares.py

from scrapy import signals
import random,json
from itemadapter import is_item, ItemAdapterclass BoosScrapySpiderMiddleware:@classmethoddef from_crawler(cls, crawler):s = cls()crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)return sdef process_spider_input(self, response, spider):return Nonedef process_spider_output(self, response, result, spider):for i in result:yield idef process_spider_exception(self, response, exception, spider):passdef process_start_requests(self, start_requests, spider):for r in start_requests:yield rdef spider_opened(self, spider):spider.logger.info('Spider opened: %s' % spider.name)class BoosScrapyDownloaderMiddleware:user_agent_list = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 ""(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1","Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 ""(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 ""(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 ""(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6","Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 ""(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 ""(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5","Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 ""(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3","Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3","Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3","Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 ""(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 ""(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24","Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 ""(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"]#可用代理池PROXY_http = ['127.0.0.1：7890']PROXY_https = ['127.0.0.1：7890']@classmethoddef from_crawler(cls, crawler):# This method is used by Scrapy to create your spiders.s = cls()crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)return s#拦截请求def process_request(self, request, spider):#UA伪装cookies="填写自己cookies"cookies = {i.split("=")[0]:i.split("=")[1] for i in cookies.split("; ")}request.cookies = cookiesrequest.headers['User-Agent'] = random.choice(self.user_agent_list)return Nonedef process_response(self, request, response, spider):return responsedef process_exception(self, request, exception, spider):if request.url.split(':')[0] == 'http':#代理request.meta['proxy'] = 'http://'+random.choice(self.PROXY_http)else:request.meta['proxy'] = 'https://' + random.choice(self.PROXY_https)return request  #将修正之后的请求对象进行重新的请求发送def spider_opened(self, spider):spider.logger.info('Spider opened: %s' % spider.name)

pipelines.py

import pymysqlclass BoosScrapyPipeline:def process_item(self, item, spider):return itemclass mysqlPileLine(object):conn = Nonecursor = Nonedef open_spider(self,spider):self.conn = pymysql.Connect(host='127.0.0.1',port=3306,user='root',passwd='123456',db='taobao',charset='utf8')def process_item(self,item,spider):self.cursor = self.conn.cursor()try:self.cursor.execute('insert into boos values(null,"%s","%s","%s","%s","%s","%s")'%(item['title'],item['salary'], item['job_sec'],item['company'],item['job_location'],item['job_tags']))self.conn.commit()except Exception as e:print("mysql插入数据失败",e)self.conn.rollback()return itemdef close_spider(self,spider):self.cursor.close()self.conn.close()

settings.py

BOT_NAME = 'boos_scrapy'SPIDER_MODULES = ['boos_scrapy.spiders']
NEWSPIDER_MODULE = 'boos_scrapy.spiders'ROBOTSTXT_OBEY = False
LOG_LEVEL = 'ERROR'
COOKIES_ENABLED = TrueDEFAULT_REQUEST_HEADERS = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'zh-CN,zh;q=0.9,zh-HK;q=0.8',
}DOWNLOADER_MIDDLEWARES = {'boos_scrapy.middlewares.BoosScrapyDownloaderMiddleware': 543,
}#开启访问频率限制
AUTOTHROTTLE_ENABLED = True
#设置访问开始的延迟
AUTOTHROTTLE_START_DELAY = 8
#设置访问之间的最大延迟
AUTOTHROTTLE_MAX_DELAY = 60
#设置Scrapy 并行发给每台远程服务器的请求数量
AUTOTHROTTLE_TARGET_CONCURRENCY= 1.0
#设置下裁之后的自动延迟
DOWNLOAD_DELAY = 3ITEM_PIPELINES = {'boos_scrapy.pipelines.BoosScrapyPipeline': 300,'boos_scrapy.pipelines.mysqlPileLine' : 301
}

main.py
运行

from scrapy import cmdlinecmdline.execute("scrapy crawl example".split())

创建数据库sql语句

CREATE TABLE `boos` (`id` int(11) NOT NULL AUTO_INCREMENT,`title` varchar(100) DEFAULT NULL,`salary` varchar(1000) DEFAULT NULL,`job_sec` varchar(1000) DEFAULT NULL,`company` varchar(1000) DEFAULT NULL,`job_location` varchar(1000) DEFAULT NULL,`job_tags` varchar(1000) DEFAULT NULL,PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=99 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

python爬虫: 爬取boss网站数据保存到mysql相关推荐

mysql scrapy 重复数据_大数据python（scrapy）爬虫爬取招聘网站数据并存入mysql后分析...
基于Scrapy的爬虫爬取腾讯招聘网站岗位数据视频(见本头条号视频) 根据TIOBE语言排行榜更新的最新程序语言使用排行榜显示,python位居第三,同比增加2.39%,为什么会越来越火,越来越受欢迎 ...
python爬虫爬取58网站数据_Python爬虫，爬取58租房数据字体反爬
Python爬虫,爬取58租房数据这俩天项目主管给了个爬虫任务,要爬取58同城上福州区域的租房房源信息.因为58的前端页面做了base64字体加密所以爬取比较费力,前前后后花了俩天才搞完. 项目演示 ...
python爬虫爬取58网站数据_python实战学习笔记：爬取58同城平板电脑数据
学习爬虫一周后独立完成的第一个作业项目:爬取58同城平板电脑数据. 1.首先确定URL,并抓取详情页中需要的信息首先我们确定好需要爬取的网页URL是:http://zhuanzhuan.58.com ...
mysql 查询系统字段自然日_吴裕雄--天生自然python数据清洗与数据可视化：MYSQL、MongoDB数据库连接与查询、爬取天猫连衣裙数据保存到MongoDB...
本博文使用的数据库是MySQL和MongoDB数据库.安装MySQL可以参照我的这篇博文:https://www.cnblogs.com/tszr/p/12112777.html 其中操作Mysql使 ...
python爬虫爬取古诗词内容，并存入mysql
python爬虫爬取古诗词内容,并存入mysql 爬取结果展示: 代码如下: from urllib import request import re,os import pymysql import ...
Python爬虫—爬取某网站上面所有的世界港口信息数据
最近学习了一段时间的Python基础语法后,写了一个爬取世界港口数据的爬虫,并且保存到SQL Server 数据库. 前提公司之前有个需求是想监控集装箱如果进出某个港口的时候能给出信息提示,并且与海 ...
python爬虫-爬取boss直聘的职位信息
爬取boss直聘的相应的职位信息: 一个同学今天想找工作,问我有什么好单位,关键我也不清楚,于是我去各大招聘网站搜索了下,有很多招聘信息,看的我眼花缭乱,刚看完的职位,忘记收藏又过去搜索,感觉太麻烦, ...
python爬取网页代码-python爬虫爬取网页所有数据详细教程
Python爬虫可通过查找一个或多个域的所有 URL 从 Web 收集数据.Python 有几个流行的网络爬虫库和框架.大家熟知的就是python爬取网页数据,对于没有编程技术的普通人来说,怎么才能快 ...
python爬虫爬取某网站图片
学习分享 | 今天刚学完爬虫,就随便写了一个爬虫代码爬取某网站的图片网站就是这个图片网站,我选的是1080p格式,4k的要会员,我反正是还不会导入的包如下 import requests from ...

python爬虫: 爬取boss网站数据保存到mysql

python爬虫: 爬取boss网站数据保存到mysql相关推荐

最新文章

热门文章

python爬虫: 爬取boss网站数据 保存到mysql

python爬虫: 爬取boss网站数据 保存到mysql相关推荐

最新文章

热门文章

python爬虫: 爬取boss网站数据保存到mysql

python爬虫: 爬取boss网站数据保存到mysql相关推荐