# -*- coding: utf-8 -*-
import scrapy
from ..items import TenXunItemclass TenxunSpider(scrapy.Spider):name = 'tenxun'# allowed_domains = ['tenxun.com']  # 域名范围start_urls = ['https://hr.tencent.com/position.php?lid=&tid=87&keywords']burl = 'https://hr.tencent.com/'def parse(self, response):tr_list = response.xpath('//table[@class="tablelist"]/tr')for tr in tr_list[1:-1]:item = TenXunItem()item['position_name']=tr.xpath('./td[1]/a/text()').extract()[0]item['position_link']=self.burl+tr.xpath('./td[1]/a/@href').extract()[0]item['position_type']=tr.xpath('./td[2]/text()').extract()[0]item['position_num']=tr.xpath('./td[3]/text()').extract()[0]item['position_addr']=tr.xpath('./td[4]/text()').extract()[0]item['position_time']=tr.xpath('./td[5]/text()').extract()[0]# yield item# 匹配下一页next_url =self.burl + response.xpath('//div[@class="pagenav"]/a[11]/@href').extract()[0]yield scrapy.Request(url=next_url, callback=self.parse)# 要获取内容，则要发起个新的请求，                      回调函数                回调时传参yield scrapy.Request(url = item['position_link'],callback=self.detail_tent,meta={'items': item})def detail_tent(self,response):# 得到上面传过来的参数item = response.meta.get('items')item['position_con'] = ''.join(response.xpath('//ul[@class="squareli"]//text()').extract())yield item# # 名字# position_name_list = response.xpath('//td[@class="l square"]/a/text()').extract()# # 链接# position_link_list = response.xpath('//td[@class="l square"]/a/@href').extract()# # 类型# position_type_list = response.xpath('//table[@class="tablelist"]/tr/td[2]/text()').extract()# # 人数# position_num_list = response.xpath('//table[@class="tablelist"]/tr/td[3]/text()').extract()# print('====================')# print('====================')# print(self.burl + tr_list[2].xpath('./td[1]/a/@href').extract()[0])# print('====================')# print('====================')

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import jsonclass TenXunPipeline(object):def open_spider(self,spider):self.f = open('tenxun.json', 'w', encoding='utf8')def process_item(self, item, spider):conn = json.dumps(dict(item), ensure_ascii=False)+'\n'self.f.write(conn)return itemdef close_spider(self,spider):self.f.close()

# -*- coding: utf-8 -*-# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.htmlimport scrapyclass TenXunItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()# 名字print('00000000000000001111111111111111')position_name = scrapy.Field()# 链接position_link = scrapy.Field()# 类型position_type = scrapy.Field()# 人数position_num = scrapy.Field()# 地点position_addr = scrapy.Field()# 发布时间position_time = scrapy.Field()# 要求position_con = scrapy.Field()