spider代码

–– coding: utf-8 ––

import scrapy

from lxml import etree

import time

import random

import requests

from …items import LianjiananjingItem

class LianjiananjingSpider(scrapy.Spider):

name = ‘lianjiananjing’

allowed_domains = [‘nj.lianjia.com’]

start_urls = ‘https://nj.lianjia.com/ershoufang/’

def start_requests(self):

user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:60.0) Gecko/20100101 Firefox/60.0'

headers = {'User-Agent': user_agent}

yield scrapy.Request(url=self.start_urls,headers=headers,method='GET',callback=self.parse)

def parse(self,response):

print(response)

user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:60.0) Gecko/20100101 Firefox/60.0'

headers = {'User-Agent': user_agent}

lists = response.body.decode('utf-8')

selector = etree.HTML(lists)

# 在进行网页抓取的时候，分析定位html节点

# 将文件读入，解析成树，然后根据路径定位到每个节点

area_list = selector.xpath('/html/body/div[3]/div/div[1]/dl[2]/dd/div[1]/div/a')

#etree.HTML得到的内容可以直接使用xpath

for area in area_list:

try:

area_hanzi = area.xpath('text()').pop()

print(area_hanzi)

area_pinyin = area.xpath('@href').pop().split('/')[2] #['/ershoufang/pudong/', '/ershoufang/minhang/', '/ershoufang/baoshan/'...]

area_url = 'http://nj.lianjia.com/ershoufang/{}/'.format(area_pinyin)

print(area_url)

yield scrapy.Request(url=area_url, headers=headers, callback=self.detail_url, meta={"id1":area_hanzi, "id2":area_pinyin})

except Exception:

print('44444444444444')

def detail_url(self,response):

for i in range(1, 40): # 获取1-39页

url = 'http://nj.lianjia.com/ershoufang/{}/pg{}/'.format(response.meta["id2"], str(i))

time.sleep(random.randint(1, 5)) # 随机等待1-5秒

try:

contents = requests.get(url)

contents = etree.HTML(contents.content.decode('utf-8'))

houselist = contents.xpath('/html/body/div[4]/div[1]/ul/li')

for house in houselist:

try:

item = LianjiananjingItem()

item['page'] = i

item['title'] = house.xpath('div[1]/div[1]/a/text()').pop()

item['community'] = house.xpath('div[1]/div[2]/div/a/text()').pop()

item['model'] = house.xpath('div[1]/div[2]/div/text()').pop().split('|')[1]

item['area'] = house.xpath('div[1]/div[2]/div/text()').pop().split('|')[2]

item['focus_num'] = house.xpath('div[1]/div[4]/text()').pop().split('/')[0]

item['watch_num'] = house.xpath('div[1]/div[4]/text()').pop().split('/')[1]

item['time'] = house.xpath('div[1]/div[4]/text()').pop().split('/')[2]

item['price'] = house.xpath('div[1]/div[6]/div[1]/span/text()').pop()

item['average_price'] = house.xpath('div[1]/div[6]/div[2]/span/text()').pop()

item['link'] = house.xpath('div[1]/div[1]/a/@href').pop()

item['city'] = response.meta["id1"]

self.url_detail = house.xpath('div[1]/div[1]/a/@href').pop()

except Exception:

print('2222222')

print(item)

yield item

except Exception:

print('2222222')

#setting 要注意把

ROBOTSTXT_OBEY = False，因为带有htpps的请求会被禁

ITEM_PIPELINES = {

‘Lianjiananjing.pipelines.LianjiananjingPipeline’: 300,

}

#item代码

–– coding: utf-8 ––

Define here the models for your scraped items

See documentation in:

import scrapy

class LianjiananjingItem(scrapy.Item):

# define the fields for your item here like:

# name = scrapy.Field()

page = scrapy.Field()

title = scrapy.Field()

community = scrapy.Field()

model = scrapy.Field()

area = scrapy.Field()

focus_num = scrapy.Field()

watch_num =scrapy.Field()

time = scrapy.Field()

price = scrapy.Field()

average_price =scrapy.Field()

link = scrapy.Field()

city = scrapy.Field()

#pipelines代码

import csv

class LianjiananjingPipeline(object):

def process_item(self, item, spider):

f = open(‘V:/Python/scapyProject/Lianjiananjing/lianjiananjing.csv’, ‘a+’)

write = csv.writer(f)

write.writerow((item[‘title’], item[‘community’], item[‘model’], item[‘area’],

item[‘focus_num’], item[‘watch_num’], item[‘time’], item[‘price’], item[‘average_price’],

item[‘link’],

item[‘city’], item[‘page’]))

return item

#main 用于调试等

#!/usr/bin/env python

#-– coding:utf-8 ––

from scrapy.cmdline import execute

import os

import sys

#添加当前项目的绝对地址

sys.path.append(os.path.dirname(os.path.abspath(file)))

#执行 scrapy 内置的函数方法execute， 使用 crawl 爬取并调试，最后一个参数jobbole 是我的爬虫文件名

execute([‘scrapy’, ‘crawl’, ‘lianjiananjing’])