# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.htmlimport scrapy
class ZhilianItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()# 职位position = scrapy.Field()# 公司名称company = scrapy.Field()# 薪资salary = scrapy.Field()# 工作地点place = scrapy.Field()# 发布时间time = scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from zhilian.items import ZhilianItem
class HighpinSpider(scrapy.Spider):# 爬虫名，创建文件时给定name = "highpin"allowed_domains = ["highpin.cn"]url = 'http://www.highpin.cn/zhiwei/ci_180000_180100_as_50_100'# 用于构造url的参数offset = 1start_urls = [url + '.html']def parse(self, response):# 用xpath对网页内容进行解析，返回的是一个选择器列表position_list = response.xpath('//div[@class="c-list-box"]/div/div[@class="clearfix"]')item = ZhilianItem()print '------------------------------'print len(position_list)print '-----------------------------------'for pos in position_list:# 这里的item对应于items.py文件中的字段item['position'] = pos.xpath('./div/p[@class="jobname clearfix"]/a/text()').extract()[0]item['company'] = pos.xpath('./div/p[@class="companyname"]/a/text()').extract()[0]item['salary'] = pos.xpath('./div/p[@class="s-salary"]/text()').extract()[0]item['place'] = pos.xpath('./div/p[@class="s-place"]/text()').extract()[0]item['time'] = pos.xpath('./div[@class="c-list-search c-wid122 line-h44"]/text()').extract()[0]     yield itemif self.offset < 150:self.offset += 1# 构建下一个要爬取的urlurl = self.url + '_p_' + str(self.offset) + '.html'print url# 发送请求，并调用parse进行数据的解析处理yield scrapy.Request(url,callback=self.parse)     

# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
class ZhilianPipeline(object):def __init__(self):# 初始化是创建本地文件self.filename = open('position.json','w')def process_item(self, item, spider):将python数据通过dumps转换成json数据text = json.dumps(dict(item),ensure_ascii=False) + '\n'# 将数据写入文件self.filename.write(text.encode('utf-8'))return itemdef close_spider(self,spider):# 关闭文件self.filename.close()