scrapy爬虫简单实例

1.CentOS安装scrapy

安装工具包

[root@localhost ~]# yum groupinstall development
[root@localhost ~]# yum install libffi-devel python-devel openssl-devel  libxslt-devel

如果没有pip安装工具，安装pip
首先安装新的yum软件源，centos7自带的软件源没有pip

[root@localhost ~]# yum install epel-release
[root@localhost ~]# yum install python-pip

安装scrapy:

[root@localhost ~]# pip install scrapy
[root@localhost ~]# pip install scrapy -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
[root@localhost ~]# scrapy -v #查看scrapy 版本

2.scrapy爬虫实例

采用scrapy框架进行爬取。现在一爬取51cto为例子。打开cmd，先用命令行，定位到某个目录，然后执行’scrapy startproject 项目名’,创建爬虫项目，然后打开item文件，代码如下：
创建爬虫项目：

scrapy startproject spiderCTO

紧接着如果想一键创建爬虫文件：

cd spiderCTO
scrapy genspider spiders #这种创建方式无效
scrapy genspider spiders '爬取的初始url'
#如：
scrapy genspider spiders example.com
Created spider 'spiders' using template 'basic' in module:jobs.spiders.crawl

# -*- coding: utf-8 -*-
import scrapy
class SpiderctoItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()title = scrapy.Field()url = scrapy.Field()score = scrapy.Field()hour = scrapy.Field()couse_long = scrapy.Field()student = scrapy.Field()price = scrapy.Field()updata = scrapy.Field()

接下来创建spiders文件里创建MySpider文件，即爬虫文件，代码如下：

# -*- coding: utf-8 -*-</code>import scrapyfrom spiderCTO.items import SpiderctoItemclass MySpider(scrapy.Spider):name = 'MySpider'start_urls = ['http://edu.51cto.com/center/course/index/list?edunav=&amp;page=1']def parse(self,response):item = SpiderctoItem()for box in response.xpath('.//div[@class="Page CourseList"]/div[@class="cList"]/div[@class="cList_Item"]'):item['title'] = box.xpath('div[@class="main"]/h3/a/text()').extract()[0]item['url'] = box.xpath('div[@class="pic"]/a/@href').extract()[0]item['score'] = box.xpath('div[@class="main"]/div[@class="stars02"]/text()').extract()[0].strip()item['student'] = box.xpath('div[@class="main"]/div/p[@class="fr"]/text()').extract()[0]item['price'] = box.xpath('div[@class="main"]/div[@class="price"]/h4/text()').extract()[0]yield scrapy.Request(item['url'],callback=self.parseNext,meta=item)def parseNext(self,response):item = response.metaitem['couse_long'] = response.xpath('.//div[@class="Info"]/div[@class="main   fr"]/div[@class="CourseTime"]/span/text()').extract()[0]item['hour'] = response.xpath('.//div[@class="Info"]/div[@class="main fr"]/div[@class="CourseTime"]/span/text()').extract()[1]item['updata'] = response.xpath('.//div[@class="Info"]/div[@class="main fr"]/div[@class="CourseTime"]/span/text()').extract()[2]yield itemfor x in range(2,162):page = 'http://edu.51cto.com/center/course/index/list?edunav=&amp;page=' + str(x)yield scrapy.Request(page,callback=self.parse)

接下来就是编写输出文件Pipeline.py,代码如下：

# -*- coding: utf-8 -*-</code># Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
#写入json# import codecs# import json# from scrapy.exceptions import DropItem# class SpiderctoPipeline(object):# def __init__(self):# self.file = open('data.json','w')# self.file = codecs.open('data.json','w',encoding='utf-8')# def process_item(self, item, spider):# line = json.dumps(dict(item),ensure_ascii=False) + '\n'# self.file.write(line)# return item
#写入数据库
from twisted.enterprise import adbapiimport pymysqlimport pymysql.cursorsclass SpiderctoPipeline(object):def __init__(self,dbpool):self.dbpool = dbpool@classmethoddef from_settings(cls,setting):dbpool=adbapi.ConnectionPool('pymysql',host='127.0.0.1',db='cto',user='root',password='123456',charset='utf8',cursorclass=pymysql.cursors.DictCursor,use_unicode=True)return cls(dbpool)def process_item(self, item, spider):self.dbpool.runInteraction(self.do_insert,item)def do_insert(self,cursor,item):insert_info = """insert into ctolist(title,url,score,hour,student,couse_long,price,updata)values (%s,%s,%s,%s,%s,%s,%s,%s)"""params = (item['title'],item['url'],item['score'],item['hour'],item['student'],item['couse_long'],item['price'],item['updata'])cursor.execute(insert_info,params)

最后就是把settings文件里设置ITEM_PIPELINES = {‘spiderCTO.pipelines.SpiderctoPipeline’: 1,}
如果想要定时执行爬虫，那就要创建一个定时文件：
一下就是一个定时文件（timeout.py），
然后再终端中执行该文件即可定时执行爬虫语句。
即：

python timeout.py

# -*- coding: utf-8 -*-
import schedule
import time
import os
import subprocess
def job():print("更新数据中。。。。")subprocess.Popen('scrapy crawl Myspider')
#每分钟执行爬虫
schedule.every().minutes.do(job)
#每小时执行
# schedule.every().hour.do(job)
#每天10点40执行
# schedule.every().day.at("10:40").do(job)
#每周一执行
# schedule.every().monday.do(job)
#每周三13点15分执行
# schedule.every().wednesday.at("13:15").do(job)while True:schedule.run_pending()time.sleep(1)

最后介绍一下Ubuntu安装scrapy

sudo apt-get install python-dev
sudo apt-get install libevent-dev
#其中python3.6是你电脑安装的scrapy版本
sudo apt-get install python-twisted-web python3.6-dev
pip install Scrapy

或者执行以下命令：

sudo apt-get update
sudo apt-get install python-pip python-lxml python-crypto python-cssselect python-openssl python-w3lib python-twisted python-dev libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
sudo pip install scrapy

在此过程中如果遇到print打印调试，在控制台不出结果的话，可以把settings中的ROBOTSTXT_OBEY设置为false：

ROBOTSTXT_OBEY = False