scrapy crawl xxxx #爬虫名字

ROBOTSTXT_OBEY = False
DEFAULT_REQUEST_HEADERS = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'en','User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)"" Chrome/63.0.3239.132 Safari/537.36"
}
ITEM_PIPELINES = {'zhilian.pipelines.ZhilianPipeline': 300,
}

class ZhilianItem(scrapy.Item):    # define the fields for your item here like:    # name = scrapy.Field()    eduLevel = scrapy.Field()  # 学历    company = scrapy.Field()  # 公司名称    salary = scrapy.Field()  # 薪资    city = scrapy.Field()  # 城市    workingExp = scrapy.Field()  # 工作经验

# -*- coding: utf-8 -*-
import jsonimport scrapyfrom scrapy import Requestfrom zhilian.items import ZhilianItemclass ZlSpiderSpider(scrapy.Spider):name = 'zl_spider'allowed_domains = ['www.zhaopin.com']start_urls = ['https://www.zhaopin.com']first_url = 'https://fe-api.zhaopin.com/c/i/sou?start={page}&pageSize=60&cityId=763&workExperience=-1&education' \'=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3&lastUrlQuery=%7B%22p%22:2,' \'%22jl%22:%22763%22,%22kw%22:%22python%22,%22kt%22:%223%22%7D'def start_requests(self, page=None):  #实现翻页for i in range(0, 11):page = i*60           yield Request(self.first_url.format(page=page), callback=self.parse)passdef parse(self, response):result = json.loads(response.text)item = ZhilianItem()for field in item.fields:if field in result.keys():item[field] = result.get(field)yield item

# -*- coding: utf-8 -*-
import jsonimport scrapy
from scrapy import Request
from zhilian.items import ZhilianItemclass ZlSpiderSpider(scrapy.Spider):name = 'zl_spider'allowed_domains = ['www.zhaopin.com']start_urls = ['https://www.zhaopin.com']first_url = 'https://fe-api.zhaopin.com/c/i/sou?start={page}&pageSize=60&cityId=763&workExperience=-1&education' \'=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3&lastUrlQuery=%7B%22p%22:2,' \'%22jl%22:%22763%22,%22kw%22:%22python%22,%22kt%22:%223%22%7D'def start_requests(self, page=None):  # 翻页for i in range(0, 11):page = i*60yield Request(self.first_url.format(page=page), callback=self.parse)passdef parse(self, response):item = ZhilianItem()result = json.loads(response.text)for i in range(0, 60):  # 一个页面有60个数据字典，遍历每个字典results = result['data']['results'][i]company = results['company']['name']eduLevel = results['eduLevel']['name']workingExp = results['workingExp']['name']city = results['city']['display']item['company'] = companyitem['eduLevel'] = eduLevelitem['salary'] = results.get('salary')item['workingExp'] = workingExpitem['city'] = cityyield item

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongoclass ZhilianPipeline(object):def process_item(self, item, spider):return itemclass MongoPipeline(object):collection_name = 'zl_python'def __init__(self, mongo_uri, mongo_db):self.mongo_uri = mongo_uriself.mongo_db = mongo_db@classmethoddef from_crawler(cls, crawler):return cls(mongo_uri=crawler.settings.get('MONGO_URI'),mongo_db=crawler.settings.get('MONGO_DATABASE'))def open_spider(self, spider):self.client = pymongo.MongoClient(self.mongo_uri)self.db = self.client[self.mongo_db]def close_spider(self, spider):self.client.close()def process_item(self, item, spider):self.db[self.collection_name].update({'company': item['company']}, dict(item), True)  # 去重return item