# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
from scrapy.spiders import Spider
from scrapy_splash import SplashRequest
from scrapy_splash import SplashMiddleware
from scrapy.http import Request, HtmlResponse
from scrapy.selector import Selector
from scrapy_splash import SplashRequest
from splash_test.items import SplashTestItem
import IniFile
import sys
import os
import re
import timereload(sys)
sys.setdefaultencoding('utf-8')sys.stdout = open('output.txt', 'w')class tvoaoSpider(Spider):name = 'tvoao'configfile = os.path.join(os.getcwd(), 'splash_test\spiders\setting.conf')cf = IniFile.ConfigFile(configfile)information_wordlist = cf.GetValue("section", "information_keywords").split(';')websearch_urls = cf.GetValue("tvoao", "websearchurl").split(';')start_urls = []for url in websearch_urls:start_urls.append(url)# request需要封装成SplashRequestdef start_requests(self):for url in self.start_urls:yield SplashRequest(url, self.parse, args={'wait': '2'})def Comapre_to_days(self,leftdate, rightdate):'''比较连个字符串日期，左边日期大于右边日期多少天:param leftdate: 格式：2017-04-15:param rightdate: 格式：2017-04-15:return: 天数'''l_time = time.mktime(time.strptime(leftdate, '%Y-%m-%d'))r_time = time.mktime(time.strptime(rightdate, '%Y-%m-%d'))result = int(l_time - r_time) / 86400return resultdef date_isValid(self, strDateText):'''判断日期时间字符串是否合法：如果给定时间大于当前时间是合法，或者说当前时间给定的范围内:param strDateText: 四种格式 '2小时前'; '2天前' ; '昨天' ;'2017.2.12 ':return: True:合法；False:不合法'''currentDate = time.strftime('%Y-%m-%d')datePattern = re.compile(r'\d{4}-\d{2}-\d{2}')strDate = re.findall(datePattern, strDateText)if len(strDate) == 1:if self.Comapre_to_days(currentDate, strDate[0]) == 0:return True, currentDatereturn False, ''def parse(self, response):site = Selector(response)sels = site.xpath('//li[@class="content_list clearfix"]')for sel in sels:strdates = sel.xpath('.//div[@class="listtext_label"]/span/text()')if len(strdates)>0:flag, date = self.date_isValid(str(strdates[0].extract()))if flag:titles = sel.xpath('.//div[@class="content_listtext"]/a/h3/text()')if len(titles) > 0:title = str(titles[0].extract())url = 'http://www.tvoao.com' + str(sel.xpath('.//a/@href')[0].extract())for keyword in self.information_wordlist:if title.find(keyword) > -1:yield SplashRequest(url, self.parse_item, args={'wait': '1'},meta={'date': date, 'url': url,'keyword': keyword, 'title': title})def parse_item(self, response):site = Selector(response)it = SplashTestItem()it['title'] = response.meta['title']it['url'] = response.meta['url']it['date'] = response.meta['date']it['keyword'] = response.meta['keyword']sources = site.xpath('//div[@class="headlines_source"]/span[2]/text()')if len(sources)>0:it['source'] = str(sources[0].extract()).replace(u'来源： ', '')return it