class QiubaiSpider(scrapy.Spider):name = 'qiubai'# allowed_domains = ['www.qiushibaike.com/text']start_urls = ['https://www.qiushibaike.com/text/']# 设计一个通用url模板url = 'https://www.qiushibaike.com/text/page/%d/'page_num = 1def parse(self, response):# 建议使用xpath进行解析（框架集成了xpath解析的接口）div_list = response.xpath('//div[@id="content-left"]/div ')# 存储解析到的页面数据data_list = []for div in div_list:# xpath解析到的指定内容存储到了Selector对象# extract()该方法可以将Selector对象存储中存储的数据值拿到author = div.xpath('./div/a[2]/h2/text()').extract_first()# extract_first = extract()[0]content = div.xpath('.//div[@class="content"]/span/text()').extract_first()# 将解析到的数据值(author和content)存储到items对象item = QiubaibypageseItem()item['author'] = authoritem['content'] = content# 将item对象提交给管道yield item# 请求的手动发送# 13表示最后一页页码if self.page_num <= 13:print("爬取到了第%d页的页面数据" % self.page_num)self.page_num += 1new_url = format(self.url % self.page_num)# callback:将去请求获取到的页面数据进行解析yield scrapy.Request(url=new_url, callback=self.parse)

class PostdemoSpider(scrapy.Spider):name = 'postDemo'# allowed_domains = ['www.baidu.com']start_urls = ['https://fanyi.baidu.com/sug']def start_requests(self):# 该方法其实是父类中的一个方法:该方法可以对star_urls列表中的元素进行get请求的fas# 发送post:# 将Request方法中method参数赋值成post# FormRequest()可以发起post请求(推荐)data = {'kw': 'dog'}print('start requests')for url in self.start_urls:yield scrapy.FormRequest(url=url, formdata=data, callback=self.parse)def parse(self, response):print(response.text)

class DoubanSpider(scrapy.Spider):name = 'douban'start_urls = ['https://accounts.douban.com/j/mobile/login/basic']def start_requests(self):param_data = {'ck': 'dFf2','name': '18629090745','password': 'cs1993413','remember': 'false','ticket': ''}for url in self.start_urls:yield scrapy.FormRequest(url, formdata=param_data, callback=self.parse)def parseBySecondPage(self, response):#  指定个人页面数据进行解析fp = open('second.html', 'w', encoding='utf-8')fp.write(response.text)def parse(self, response):print(response.text)# 获取当前用户的个人主页url = 'https://www.douban.com/people/181322653/'yield scrapy.Request(url=url, callback=self.parseBySecondPage)

# 自定义一个下载中间件的类, 在类中事先process_request（处理中间件拦截到的请求）方法
class MyProxy(object):def process_request(self, request, spider):# 请求ip的更换request.meta['proxy'] = "http://183.233.90.6:8080"

DOWNLOADER_MIDDLEWARES = {# 'postPro.middlewares.PostproDownloaderMiddleware': 543,'postPro.middlewares.MyProxy': 542,
}

# 指定终端输入指定种类的日志信息
LOG_LEVEL = 'ERROR'
# 日志输出到指定的文件
LOG_FILE = 'log.txt'

class Moive195Spider(scrapy.Spider):name = 'moive195'# allowed_domains = ['http://www.1905.com']start_urls = ['http://www.1905.com/mdb/film/list/year-2018']# 用于解析二级子页面中的数据值def parseBySecondPage(self, response):Feature = response.xpath('/html/body/div[2]/div/div[1]/section[1]/div/p/text()').extract_first()director = response.xpath('/html/body/div[2]/section/div/div[2]/div[2]/a[1]/div[2]/text()').extract_first()# 取出Request方法的meta参数传递过来的字典(response.meta)item = response.meta['item']item['feature'] = Feature if Feature else ''item['director'] = director if director else ''# 将item提交给管道yield itemdef parse(self, response):div_list = response.xpath("/html/body/div[2]/div[1]/ul/li")for div in div_list:movie_name = div.xpath("./div/p[1]/a/text()").extract_first()href = div.xpath("./a/@href").extract_first()rank = div.xpath("./div/p[2]/b/text()").extract_first()actors = div.xpath("./div/p[3]/a/text()").extract()print(actors, '演员')movie_type = div.xpath("./div/p[4]/a/text()").extract_first()# 创建item对象item = ParsemoiveItem()item['name'] = movie_name if movie_name else ''item['movie_type'] = movie_type if movie_type else ''item['rank'] = rank if rank else ''item['actors'] = '/'.join(actors) if actors else ''# 需要对url发起请求,获取页面数据进行指定的解析url = 'http://www.1905.com/' + href# meta参数值可以赋值一个字典(将item对象先封装到一个字典中)yield scrapy.Request(url=url, callback=self.parseBySecondPage, meta={'item': item})

class ParsemoivePipeline(object):fp = Nonedef open_spider(self, spider):self.fp = open('movie.txt', 'w', encoding='utf-8')def process_item(self, item, spider):detail = item['name'] + ':' + item['movie_type'] + '' + item['rank'] + '' + item['director'] + "\n" + item['actors'] + "\n" + item['feature'] + "\n\n\n"self.fp.write(detail)return itemdef close_spider(self, spider):self.fp.close()

 scrapy genspider -t crawl 爬虫名称 起始url

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Ruleclass CrawlchoutiSpider(CrawlSpider):name = 'crawlchouti'# allowed_domains = ['dig.chouti.com']start_urls = ['https://dig.chouti.com/']# 实例化了一个链接提取器对象# 链接提取器:用来提取指定的链接(url)# allow参数：赋值一个正则表达式# 链接提取器可以根据正则表达式在页面提取指定的链接# 提取到的链接会全部交给规则解析器link = LinkExtractor(allow=r'/all/hot/recent/\d+')rules = (# 实例化了一个规则解析器对象# 规则解析器接受了链接提取器发送的链接后,就会对这些链接发起请求，获取链接对应的页面内容， 就会根据指定的规则对页面内容中指定的数据值进行解析# callback:指定一个解析规则(方法/函数)# follow:是否将链接提取器继续作用到链接提取器提出的链接所表示的页面数据Rule(link, callback='parse_item', follow=False),)def parse_item(self, response):print(response)

from scrapy.pipelines.images import ImagesPipelineimport scrapyclass ImgproPipeline(object):item = Nonedef process_item(self, item, spider):# print(item)return item#ImagesPipeline专门用于文件下载的管道类，下载过程支持异步和多线程class ImgPipeLine(ImagesPipeline):#对item中的图片进行请求操作def get_media_requests(self, item, info):yield scrapy.Request(item['src'])#定制图片的名称def file_path(self, request, response=None, info=None):url = request.urlfile_name = url.split('/')[-1]return file_namedef item_completed(self, results, item, info):return item  #该返回值会传递给下一个即将被执行的管道类