#1、执行全局命令：请确保不在某个项目的目录下，排除受该项目配置的影响
scrapy startproject MyProjectcd MyProject
scrapy genspider baidu www.baidu.comscrapy settings --get XXX #如果切换到项目目录下，看到的则是该项目的配置

scrapy runspider baidu.pyscrapy shell https://www.baidu.comresponseresponse.statusresponse.bodyview(response)scrapy view https://www.taobao.com #如果页面显示内容不全，不全的内容则是ajax请求实现的，以此快速定位问题

scrapy fetch --nolog --headers https://www.taobao.comscrapy version #scrapy的版本

scrapy version -v #依赖库的版本#2、执行项目命令：切到项目目录下
scrapy crawl baidu
scrapy check
scrapy list
scrapy parse http://quotes.toscrape.com/ --callback parse
scrapy bench

import scrapyclass XiaoHuarSpider(scrapy.spiders.Spider):name = "xiaohuar"                            # 爬虫名称 *****allowed_domains = ["xiaohuar.com"]  # 允许的域名start_urls = ["http://www.xiaohuar.com/hua/",   # 其实URL
    ]def parse(self, response):# 访问起始URL并获取结果后的回调函数

import sys,os
sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')

response.selector.css()
response.selector.xpath()
可简写为
response.css()
response.xpath()#1 //与/
response.xpath('//body/a/')#
response.css('div a::text')>>> response.xpath('//body/a') #开头的//代表从整篇文档中寻找,body之后的/代表body的儿子
[]
>>> response.xpath('//body//a') #开头的//代表从整篇文档中寻找,body之后的//代表body的子子孙孙
[<Selector xpath='//body//a' data='<a href="image1.html">Name: My image 1 <'>, <Selector xpath='//body//a' data='<a href="image2.html">Name: My image 2 <'>, <Selector xpath='//body//a' data='<a href="
image3.html">Name: My image 3 <'>, <Selector xpath='//body//a' data='<a href="image4.html">Name: My image 4 <'>, <Selector xpath='//body//a' data='<a href="image5.html">Name: My image 5 <'>]#2 text
>>> response.xpath('//body//a/text()')
>>> response.css('body a::text')#3、extract与extract_first:从selector对象中解出内容
>>> response.xpath('//div/a/text()').extract()
['Name: My image 1 ', 'Name: My image 2 ', 'Name: My image 3 ', 'Name: My image 4 ', 'Name: My image 5 ']
>>> response.css('div a::text').extract()
['Name: My image 1 ', 'Name: My image 2 ', 'Name: My image 3 ', 'Name: My image 4 ', 'Name: My image 5 ']>>> response.xpath('//div/a/text()').extract_first()
'Name: My image 1 '
>>> response.css('div a::text').extract_first()
'Name: My image 1 '#4、属性：xpath的属性加前缀@
>>> response.xpath('//div/a/@href').extract_first()
'image1.html'
>>> response.css('div a::attr(href)').extract_first()
'image1.html'#4、嵌套查找
>>> response.xpath('//div').css('a').xpath('@href').extract_first()
'image1.html'#5、设置默认值
>>> response.xpath('//div[@id="xxx"]').extract_first(default="not found")
'not found'#4、按照属性查找
response.xpath('//div[@id="images"]/a[@href="image3.html"]/text()').extract()
response.css('#images a[@href="image3.html"]/text()').extract()#5、按照属性模糊查找
response.xpath('//a[contains(@href,"image")]/@href').extract()
response.css('a[href*="image"]::attr(href)').extract()response.xpath('//a[contains(@href,"image")]/img/@src').extract()
response.css('a[href*="imag"] img::attr(src)').extract()response.xpath('//*[@href="image1.html"]')
response.css('*[href="image1.html"]')#6、正则表达式
response.xpath('//a/text()').re(r'Name: (.*)')
response.xpath('//a/text()').re_first(r'Name: (.*)')#7、xpath相对路径
>>> res=response.xpath('//a[contains(@href,"3")]')[0]
>>> res.xpath('img')
[<Selector xpath='img' data='<img src="data:image3_thumb.jpg">'>]
>>> res.xpath('./img')
[<Selector xpath='./img' data='<img src="data:image3_thumb.jpg">'>]
>>> res.xpath('.//img')
[<Selector xpath='.//img' data='<img src="data:image3_thumb.jpg">'>]
>>> res.xpath('//img') #这就是从头开始扫描
[<Selector xpath='//img' data='<img src="data:image1_thumb.jpg">'>, <Selector xpath='//img' data='<img src="data:image2_thumb.jpg">'>, <Selector xpath='//img' data='<img src="data:image3_thumb.jpg">'>, <Selector xpa
th='//img' data='<img src="data:image4_thumb.jpg">'>, <Selector xpath='//img' data='<img src="data:image5_thumb.jpg">'>]#8、带变量的xpath
>>> response.xpath('//div[@id=$xxx]/a/text()',xxx='images').extract_first()
'Name: My image 1 '
>>> response.xpath('//div[count(a)=$yyy]/@id',yyy=5).extract_first() #求有5个a标签的div的id
'images'

1、
scrapy startproject Amazon
cd Amazon
scrapy genspider spider_goods www.amazon.cn2、settings.py
ROBOTSTXT_OBEY = False
#请求头
DEFAULT_REQUEST_HEADERS = {'Referer':'https://www.amazon.cn/','User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'
}
#打开注释
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'3、items.py
class GoodsItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()#商品名字goods_name = scrapy.Field()#价钱goods_price = scrapy.Field()#配送方式delivery_method=scrapy.Field()4、spider_goods.py
# -*- coding: utf-8 -*-
import scrapyfrom Amazon.items import  GoodsItem
from scrapy.http import Request
from urllib.parse import urlencodeclass SpiderGoodsSpider(scrapy.Spider):name = 'spider_goods'allowed_domains = ['www.amazon.cn']# start_urls = ['http://www.amazon.cn/']def __int__(self,keyword=None,*args,**kwargs):super(SpiderGoodsSpider).__init__(*args,**kwargs)self.keyword=keyworddef start_requests(self):url='https://www.amazon.cn/s/ref=nb_sb_noss_1?'paramas={'__mk_zh_CN': '亚马逊网站','url': 'search - alias = aps','field-keywords': self.keyword}url=url+urlencode(paramas,encoding='utf-8')yield Request(url,callback=self.parse_index)def parse_index(self, response):print('解析索引页:%s' %response.url)urls=response.xpath('//*[contains(@id,"result_")]/div/div[3]/div[1]/a/@href').extract()for url in urls:yield Request(url,callback=self.parse_detail)next_url=response.urljoin(response.xpath('//*[@id="pagnNextLink"]/@href').extract_first())print('下一页的url',next_url)yield Request(next_url,callback=self.parse_index)def parse_detail(self,response):print('解析详情页:%s' %(response.url))item=GoodsItem()# 商品名字item['goods_name'] = response.xpath('//*[@id="productTitle"]/text()').extract_first().strip()# 价钱item['goods_price'] = response.xpath('//*[@id="priceblock_ourprice"]/text()').extract_first().strip()# 配送方式item['delivery_method'] = ''.join(response.xpath('//*[@id="ddmMerchantMessage"]//text()').extract())return item5、自定义pipelines
#sql.py
import pymysql
import settingsMYSQL_HOST=settings.MYSQL_HOST
MYSQL_PORT=settings.MYSQL_PORT
MYSQL_USER=settings.MYSQL_USER
MYSQL_PWD=settings.MYSQL_PWD
MYSQL_DB=settings.MYSQL_DBconn=pymysql.connect(host=MYSQL_HOST,port=int(MYSQL_PORT),user=MYSQL_USER,password=MYSQL_PWD,db=MYSQL_DB,charset='utf8'
)
cursor=conn.cursor()class Mysql(object):@staticmethoddef insert_tables_goods(goods_name,goods_price,deliver_mode):sql='insert into goods(goods_name,goods_price,delivery_method) values(%s,%s,%s)'cursor.execute(sql,args=(goods_name,goods_price,deliver_mode))conn.commit()@staticmethoddef is_repeat(goods_name):sql='select count(1) from goods where goods_name=%s'cursor.execute(sql,args=(goods_name,))if cursor.fetchone()[0] >= 1:return Trueif __name__ == '__main__':cursor.execute('select * from goods;')print(cursor.fetchall())#pipelines.py
from Amazon.mysqlpipelines.sql import Mysqlclass AmazonPipeline(object):def process_item(self, item, spider):goods_name=item['goods_name']goods_price=item['goods_price']delivery_mode=item['delivery_method']if not Mysql.is_repeat(goods_name):Mysql.insert_table_goods(goods_name,goods_price,delivery_mode)6、创建数据库表
create database amazon charset utf8;
create table goods(id int primary key auto_increment,goods_name char(30),goods_price char(20),delivery_method varchar(50)
);7、settings.py
MYSQL_HOST='localhost'
MYSQL_PORT='3306'
MYSQL_USER='root'
MYSQL_PWD='123'
MYSQL_DB='amazon'#数字代表优先级程度（1-1000随意设置，数值越低，组件的优先级越高）
ITEM_PIPELINES = {'Amazon.mysqlpipelines.pipelines.mazonPipeline': 1,
}#8、在项目目录下新建：entrypoint.py
from scrapy.cmdline import execute
execute(['scrapy', 'crawl', 'spider_goods','-a','keyword=iphone8'])