#coding=utf-8
#爬取每日一文前10页内容
from lxml import etree
import requests
import urllib2,urllib
import sys
import os
import timetmpt_url = 'http://voice.meiriyiwen.com/voice/past?page=%d'
urllist = [tmpt_url%i for i in range(1,11)]def get_url():for url in urllist:try:headers = {'Host':'voice.meiriyiwen.com','Upgrade-Insecure-Requests':'1','User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36' ,'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8','Accept-Encoding':'gzip, deflate, sdch, br','Accept-Language':'zh-CN,zh;q=0.8','Cache-Control':'max-age=0','Connection':'keep-alive'}#proxies = { "http": "dev-proxy.oa.com:8080","https": "dev-proxy.oa.com:8080",}time.sleep(0.5)response = requests.get(url,headers = headers)print response.status_codeget_info(response)except urllib2.URLError, e:print e.reasondef get_info(response):global counthtml = response.content#print htmltree = etree.HTML(html)rez = tree.xpath('//*[@class="img_list"]')for i in rez:title = i.xpath('//*[@class="list_author"]/a/text()')author = i.xpath('//*[@class="author_name"]/text()')for x,y in zip(title,author):count += 1print count,'|',x.replace(u'\xa0','').strip(),'|',y.replace(u'\xa0','').strip()if __name__ == '__main__':count = 0get_url()

# coding=utf-8
import json
import os
import sys
import time
import urllib
import urllib2
import pymongo
import requests
import re
from lxml import etree# reload(sys)
# sys.setdefaultencoding('utf-8')
tmpt_url = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%s&sort=recommend&page_limit=500&page_start=0'
tags = ('热门', '最新', '经典', '豆瓣高分', '冷门佳片', '华语', '欧美', '韩国','日本', '动作', '喜剧', '爱情', '科幻', '悬疑', '恐怖', '文艺')
urllist = [tmpt_url % i for i in tags]
#print urllistdef get_url():for url in urllist:try:headers = {#'Host':'www.douban.com','Upgrade-Insecure-Requests': '1','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8','Accept-Encoding': 'gzip, deflate, sdch, br','Accept-Language': 'zh-CN,zh;q=0.8','Cache-Control': 'max-age=0','Connection': 'keep-alive'}# proxies = { "http": "dev-proxy.oa.com:8080","https":# "dev-proxy.oa.com:8080",}time.sleep(0.5)response = requests.get(url, headers=headers)tag = re.findall('tag=(.*?)&',url)print u"电影类型：",tagget_info(response)except Exception,e:print edef get_info(response):global counthtml = response.contentdictt = json.loads(html, encoding='utf-8')dd = dictt['subjects']for item in dd:count += 1print count,u'电影链接：', item['url'], u'电影名：', item['title'], u'评分：', item['rate']if __name__ == '__main__':count = 0get_url()