1 import scrapy
2 class LuooItem(scrapy.Item):
3     url = scrapy.Field()
4     title = scrapy.Field()
5     pic = scrapy.Field()
6     desc = scrapy.Field()

 1 import scrapy
 2 from luoo.items import LuooItem
 3
 4 class LuooSpider(scrapy.Spider):
 5     name = "luoo"
 6     allowed_domains = ["luoo.net"]
 7     start_urls = []
 8     for i in range(750,763):
 9         url = 'http://www.luoo.net/music/%s'%(str(i))
10         start_urls.append(url)
11
12     def parse(self, response):
13         item = LuooItem()
14         item['url'] = response.url
15         item['title'] = response.xpath('//span[@class="vol-title"]/text()').extract()
16         item['pic'] = response.xpath('//img[@class="vol-cover"]/@src').extract()
17         item['desc'] = response.xpath('//div[@class="vol-desc"]/text()').extract()
18         return item

 1 import urllib.request
 2 import re
 3 import time
 4
 5 def openurl(urls):
 6     htmls=[]
 7     for url in urls:
 8         req=urllib.request.Request(url)
 9         req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.107 Safari/537.36')
10   #      Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0
11         response = urllib.request.urlopen(url)
12         htmls.append(response.read())
13         time.sleep(5)
14     return htmls
15
16 def jiexi(htmls):
17     pics=[]
18     titles=[]
19     contents=[]
20     for html in htmls:
21         html = html.decode('utf-8')
22         pics.append(re.findall('<div class="player-wrapper".*?>.*?<img.*?src="(.*?).jp.*?".*?alt=".*"',html,re.S))
23         titles.append(re.findall('class="vol-title">(.*?)</span>',html,re.S))
24         contents.append(re.findall('<div.*?class="vol-desc">.*?(.*?)</div>',html,re.S))
25
26     i = len(titles)
27     with open('C:\\Users\\Administrator\\Desktop\\test.txt', 'w') as f:
28         for x in range(i):
29             print("正在下载期刊：%d" %(746-x))
30             f.write("期刊名："+str(titles[x])[2:-2]+"\n")
31             f.write("图片链接："+str(pics[x])[2:-2]+".jpg\n")
32             content = str(contents[x])[4:-2]
33             content.strip()
34             print(content.count("""<br>\n"""))
35             content.replace("""<br>\n""","#")
36             f.write("配诗："+content+"\n\n\n")
37
38
39 yur='http://www.luoo.net/music/'
40 urls = []
41 for i in range(657,659):
42     urls.append(yur + str(i))
43
44 htmls = openurl(urls)
45 pics = jiexi(htmls)