emptyempty

# -*- coding: utf-8 -*-# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.htmlimport scrapyclass TcItem(scrapy.Item):# define the fields for your item here like:name = scrapy.Field()   #招聘职位名称Cpname = scrapy.Field() #公司名称pay  = scrapy.Field()   #薪资待遇edu  = scrapy.Field()   #学历要求num  = scrapy.Field()   #招聘人数year = scrapy.Field()   #工作年限FL   = scrapy.Field()   #福利待遇

以上是我给想爬取的数据定义的属性
   3.在spiders中新建了一个tc_spider.py,一下是tc_spider.py的代码:

# -*- coding: utf-8 -*-
import scrapy
from tc.items import TcItem
from scrapy.selector import HtmlXPathSelector,Selector
from scrapy.http import Request
class TcSpider(scrapy.Spider):name='tc'allowed_domains=['jn.58.com']start_urls=["http://jn.58.com/tech/pn1/?utm_source=market&spm=b-31580022738699-me-f-824.bdpz_biaoti&PGTID=0d303655-0010-915b-ca53-cb17de8b2ef6&ClickID=3"]theurl="http://jn.58.com/tech/pn"theurl2="/?utm_source=market&spm=b-31580022738699-me-f-824.bdpz_biaoti&PGTID=0d303655-0010-915b-ca53-cb17de8b2ef6&ClickID=3"for i in range(75):n=i+2the_url=theurl+str(n)+theurl2start_urls.append(the_url)def start_request(self,response):sel = Selector(response)  sites = sel.xpath("//*[@id='infolist']/dl")#items = []  for site in sites:  #item = DmozItem()  #item['namee'] = site.xpath('dt/a/text()').extract()href = site.xpath('dt/a/@href').extract()self.start_urls.append(href)#item['company'] = site.xpath('dd/a/@title').extract()#if site!= " " :#  items.append(item) for url in self.start_urls:yield self.make_requests_from_url()def parse_item(self, response):  items2 = []  item=TcItem()item['name']=response.xpath("//*[@class='headConLeft']/h1/text()").extract()item['Cpname']=response.xpath("//*[@class='company']/a/text()").extract()item['pay']=response.xpath(("//*[@class='salaNum']/strong/text()")).extract()item['edu']=response.xpath("//*[@class='xq']/ul/li[1]/div[2]/text()").extract()item['num']=response.xpath("//*[@class='xq']/ul/li[2]/div[1]/text()").extract()item['year']=response.xpath("//*[@class='xq']/ul/li[2]/div[2]/text()").extract()item['FL']=response.xpath("//*[@class='cbSum']/span/text()").extract()dec=item['num']items2.append(item)return items2  def parse(self, response):sel = HtmlXPathSelector(response)  href = sel.select("//*[@id='infolist']/dl/dt/a/@href").extract()for he in href:yield Request (he,callback=self.parse_item)
# 翻页#    next_page=response.xpath("//*[@class='nextMsg']/a/@href")#   if next_page:#   url=response.urljoin(next_page[0].extract())#    yield scrapy.Request(url,self.parse)

这段代码大体四个部分：①定义爬取的网站以及范围②每个属性的xpath的编写③对于每个职位的链接爬取的循环（能实现进去爬取静态的信息）④连续爬取，网页的循环