1 # -*- coding: utf-8 -*-
 2
 3 # Define here the models for your scraped items
 4 #
 5 # See documentation in:
 6 # https://doc.scrapy.org/en/latest/topics/items.html
 7
 8 import scrapy
 9
10
11 class FictionItem(scrapy.Item):
12     # define the fields for your item here like:
13     # name = scrapy.Field()
14
15     bookname = scrapy.Field()
16     statement = scrapy.Field()
17     author = scrapy.Field()
18     simple_content = scrapy.Field()
19
20
21     pass

 1 # -*- coding: utf-8 -*-
 2 import scrapy
 3 from scrapy.linkextractors import LinkExtractor
 4 from ..items import FictionItem
 5
 6
 7 class FictionsSpider(scrapy.Spider):
 8     name = 'fictions'
 9     allowed_domains = ['quanshuwang.com']
10     start_urls = ['http://www.quanshuwang.com/list/8_1.html']
11
12     # 书籍列表的页面解析函数
13     def parse(self,response):
14         # 提取书籍列表页面每一本书的链接
15         pattern = 'http://www.quanshuwang.com/book_'
16         le = LinkExtractor(restrict_xpaths='//*[@class="seeWell cf"]',allow=pattern)
17         for link in le.extract_links(response):
18             yield scrapy.Request(link.url,callback=self.parse_book)
19
20         # 提取下一页链接
21         le = LinkExtractor(restrict_xpaths='//*[@id="pagelink"]/a[@class="next"]')
22         links =  le.extract_links(response)
23         if links:
24             next_url = links[0].url
25             yield scrapy.Request(next_url,callback=self.parse)
26
27         pass
28
29
30     # 书籍界面的解析函数
31     def parse_book(self, response):
32         book=FictionItem()
33         sel = response.css('div.detail')
34         book['bookname']=sel.xpath('./div[1]/h1/text()').extract_first()
35         book['statement']=sel.xpath('//*[@id="container"]/div[2]/section/div/div[4]/div[1]/dl[1]/dd/text()').extract_first()
36         book['author']=sel.xpath('//*[@id="container"]/div[2]/section/div/div[4]/div[1]/dl[2]/dd/text()').extract_first()
37         book['simple_content']=sel.xpath('string(//*[@id="waa"])').extract()
38
39         yield book

scrapy crawl books -o fiction.csv