#!/usr/bin/env python

#encoding:utf-8

import requests

from bs4 import BeautifulSoup

import urllib

import sys

import re

# 解决编码错误问题

reload(sys)

sys.setdefaultencoding('utf8')

def getHex(words):

mywords = words.split("%")[1:]

result = ""

for i in mywords:

result += chr(int(i, 16))

return result

'''

电影天堂模块

'''

# config-start

maxPage = 5

modelName = "电影天堂"

url = "http://www.dy2018.com"

keyword = sys.argv[1]

pageSize = 20

keywordURLencode = urllib.quote(keyword.decode(sys.stdin.encoding).encode('GBK')) # 将查询关键字进行URL编码

searchUrl = "http://www.dy2018.com/e/search/index.php"

postData = {

'classid':'0',

'show':'title,smalltext',

'tempid':'1',

'keyboard': getHex(keywordURLencode),

'Submit':chr(0xC1) + chr(0xA2) + chr(0xBC) + chr(0xB4) + chr(0xCB) + chr(0xD1) + chr(0xCB) + chr(0xF7)

}

headers = {

'Host' : 'www.dy2018.com',

'Cache-Control' : 'max-age=0',

'Origin' : 'http://www.dy2018.com',

'Upgrade-Insecure-Requests' : '1',

'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',

'Content-Type' : 'application/x-www-form-urlencoded',

'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',

'Referer' : 'http://www.dy2018.com/index.html',

'Accept-Encoding' : 'gzip, deflate',

'Accept-Language' : 'zh-CN,zh;q=0.8,en;q=0.6',

'Connection' : 'close'

}

# config-end

def getContent(url):

global headers

response = requests.get(url, headers=headers)

response.encoding = 'gb2312' # 设置相应体的字符集

return response.text

def getResultNumber(soup):

a = soup.find("a", title="总数")

totalResult = a.find("b").string

return int(totalResult)

def getSoupByPostData(url,postData):

global headers

content = requests.post(url, data=postData, headers=headers).text.decode("UTF-8")

soup = BeautifulSoup(content, "html.parser")

return soup

def getPageNumber(resultNumber):

global pageSize

return int((resultNumber / pageSize)) + 1

def getPageID(soup):

div = soup.find('div', class_="x", style="text-align: center;font-size: 14px;margin: 5px 0;")

firstPage = div.findAll("a")[1]

firstPageStr = firstPage['href']

pageID = firstPageStr.split("-")[1]

return pageID

def getResultDic(soup):

results = []

tables = soup.findAll("table", width="100%", border="0", cellspacing="0", cellpadding="0", class_="tbspan", style="margin-top:6px")

for table in tables:

# 获取结果标题

title = str(table.find("a")["title"])

# 获取结果描述

describe = table.find("td", colspan="2", style="padding-left:3px").string

# 获取页面详细地址

src = url + table.find("a")['href']

# 获取条目时间和点击量

temp = table.find("font", color="#8F8C89").string

time = temp.split("\n")[0].split("：")[1][0:-1] # 注意这里是中文冒号

click = temp.split("\n")[1].split("：")[1] # 注意这里是中文冒号

# 获取下载地址

downloadLinks = []

newContent = getContent(src)

newSoup = BeautifulSoup(newContent, "html.parser")

tbodys = newSoup.findAll("tbody")

for tbody in tbodys:

downloadLinks.append(tbody.find("a")['href'])

result = {

"title":title,

"describe":describe,

'time':time,

'click':click,

"downloadLink":downloadLinks

}

results.append(result)

print "单条数据获取成功 !"

return results

soup = getSoupByPostData(searchUrl, postData)

resultNumber = getResultNumber(soup)

pageNumber = getPageNumber(resultNumber)

pageID = getPageID(soup)

print "查询结果数 :", resultNumber

print "总页面数量 :", pageNumber

print "正在获取第 1 页的结果"

results = getResultDic(soup)

print "该页所有结果获取成功 !"

if pageNumber > maxPage:

for page in range(maxPage):

print "正在获取第",(page + 1),"页的结果"

thisUrl = "http://www.dy2018.com/e/search/result/searchid-" + pageID + "-page-" + str(page) + ".html"

tempContent = getContent(thisUrl)

tempSoup = BeautifulSoup(tempContent, "html.parser")

results += getResultDic(soup)

SIGN = input("已经获取了" + maxPage + "个页面 , 是否需要继续获取 ? [1/0]")

if SIGN == 1:

for page in range(maxPage, pageNumber):

print "正在获取第",(page + 1),"页的结果"

thisUrl = "http://www.dy2018.com/e/search/result/searchid-" + pageID + "-page-" + str(page) + ".html"

tempContent = getContent(thisUrl)

tempSoup = BeautifulSoup(tempContent, "html.parser")

results += getResultDic(soup)

else:

for page in range(pageNumber):

print "正在获取第",(page + 1),"页的结果"

thisUrl = "http://www.dy2018.com/e/search/result/searchid-" + pageID + "-page-" + str(page) + ".html"

tempContent = getContent(thisUrl)

tempSoup = BeautifulSoup(tempContent, "html.parser")

results += getResultDic(soup)

print "数据获取完毕 ! "

# 格式化显示数据 :

for result in results:

file = open(modelName + "-" + keyword + ".txt","a+")

file.write("---------------------------\n")

file.write("标题 : " + result['title'] + "\n")

file.write("描述 : \n\t" + result['describe'] + "\n")

file.write("时间 : " + result['time'] + "\n")

file.write("点击量 : " + result['click'] + "\n")

file.write("下载地址 : " + "\n")

for downloadlink in result['downloadLink']:

file.write("\t" + downloadlink + "\n")

file.write("\n")

file.close()