# -*- coding:utf-8 -*-
import scrapy
from ..items import ZhilianSecondItem
from scrapy import Request
from bs4 import BeautifulSoup
class spider(scrapy.Spider):name = 'zhilian_second'start_urls =[]def __init__(self):links = open('E:/PythonWorkStation/zhilian_fist/myurls')for line in links:# 一定要去掉换行符，如果有换行符则无法访问网址，真他妈坑爹line=line[:-1]# print('-----------------------------')# print('-----------------------------')# print(line+'测试是否有换行符')# print('-----------------------------')# print('-----------------------------')self.start_urls.append(line)# breakdef parse(self, response):item = ZhilianSecondItem()# print('--------------        start                 -----------------------')title_list = response.xpath('//div/span[@class="post"]/a/text()').extract()company_list = response.xpath('//div/span[@class="company_name"]/a/text()').extract()salary_list = response.xpath('//div/span[@class="salary"]/text()').extract()address_list = response.xpath('//div/span[@class="address"]/text()').extract()release_list = response.xpath('//div/span[@class="release_time"]/text()').extract()if response.xpath('//span[@class="search_page_next"]').extract()!= None:next_url = response.xpath('//span[@class="search_page_next"]/a/@href').extract()next_url=next_url[0].split('/')[2]# print('----b--------')# print('----b--------')# print(response.url)# print(len(response.url.split('/')))# print(next_url)# print(len(next_url))# print('----e--------')# print('----e--------')# self.start_urls.append( Request(response.url[:-9]+next_url[0]))if len(response.url.split('/'))==5:yield Request(response.url+next_url)elif len(response.url.split('/'))>5:i = len(next_url)+1print('***********')# print(i)print(next_url.lstrip('p'))print('***********')if (next_url.lstrip('p') == str(10) or next_url.lstrip('p')==str(100) or next_url.lstrip('p')==str(1000) or next_url.lstrip('p')== str(10000)):print('++++++++++++++++')i = i-1yield Request(response.url[:-(i)] + next_url)for a,s,d,f,g in zip(title_list,company_list,salary_list, address_list,release_list):item['company']=sitem['salary']=ditem['address']=fitem['release']=gitem['title'] = ayield item

items.py文件

# -*- coding: utf-8 -*-# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.htmlimport scrapyclass ZhilianSecondItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()title =scrapy.Field()company =scrapy.Field()salary =scrapy.Field()address =scrapy.Field()release =scrapy.Field()

# -*- coding: utf-8 -*-# Define here the models for your spider middleware
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/spider-middleware.htmlfrom scrapy import signalsclass ZhilianSecondSpiderMiddleware(object):# Not all methods need to be defined. If a method is not defined,# scrapy acts as if the spider middleware does not modify the# passed objects.@classmethoddef from_crawler(cls, crawler):# This method is used by Scrapy to create your spiders.s = cls()crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)return sdef process_spider_input(response, spider):# Called for each response that goes through the spider# middleware and into the spider.# Should return None or raise an exception.return Nonedef process_spider_output(response, result, spider):# Called with the results returned from the Spider, after# it has processed the response.# Must return an iterable of Request, dict or Item objects.for i in result:yield idef process_spider_exception(response, exception, spider):# Called when a spider or process_spider_input() method# (from other spider middleware) raises an exception.# Should return either None or an iterable of Response, dict# or Item objects.passdef process_start_requests(start_requests, spider):# Called with the start requests of the spider, and works# similarly to the process_spider_output() method, except# that it doesn’t have a response associated.# Must return only requests (not items).for r in start_requests:yield rdef spider_opened(self, spider):spider.logger.info('Spider opened: %s' % spider.name)

pipeline.py文件

# -*- coding: utf-8 -*-class ZhilianSecondPipeline(object):def open_spider(self,spider):self.file = open('E:/招聘岗位.txt','w',encoding='utf-8')def process_item(self, item, spider):self.file.write(item['title']+","+item['company']+","+item['salary']+","+item['address']+","+item['release']+'\n')# print('----------------------------------------------------------')# print(item['title'],item['company'],item['salary'],item['address'],item['release'])# print('----------------------------------------------------------')return itemdef spoder_closed(self,spider):self.file.close()

setting.py文件

# -*- coding: utf-8 -*-
BOT_NAME = 'zhilian_second'
SPIDER_MODULES = ['zhilian_second.spiders']
NEWSPIDER_MODULE = 'zhilian_second.spiders'
ROBOTSTXT_OBEY = False
ITEM_PIPELINES = {'zhilian_second.pipelines.ZhilianSecondPipeline': 300,
}
LOG_LEVEL = 'INFO'

由于爬取的太多需要等的时间过长，所以本人在程序没有运行结束之前关终止了运行，但是依旧爬取了数十万岗位信息如下图所示