python 爬取movie时光网

代码如下：欢迎指点不足之处

# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import re
import execjs, urllib
import csv, time
class MVProcess():def __init__(self):self.headers = {'Referer': 'http://movie.mtime.com','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36'}self.run()#获取每一页的链接def get_html(self, url):res = requests.get(url, headers=self.headers, timeout=5)i = 0while res.status_code == 521:time.sleep(1)cookie_dic = self.error_http_521res(res)# 将cookie放入 重新请求res = requests.get(url, headers=self.headers, cookies=cookie_dic, timeout=5)if i == 3:returni += 1return res.text#爬虫返回521解决办法def error_http_521res(self, response):# 分析521返回的js信息，并返回重新生成的cookie用于重新请求res_521_html = response.content.decode('utf-8')# print(res_521_html)# 提取js加密函数res_521_js_func_str = ''.join(re.findall(r'(function .*?)</script>', res_521_html))# print(res_521_jscode_func)# 提取js函数参数res_521_js_arg = ''.join(re.findall(r'setTimeout\(\"\D+\((\d+)\)\"', res_521_html))res_521_js_name = re.findall(r'function (\w+)', res_521_js_func_str)[0]# print(res_521_js_arg)# print(res_521_js_name)# 修改js函数，使其返回cookie内容js_func_str = res_521_js_func_str.replace('eval("qo=eval;qo(po);")', 'return po')# print(js_func_str)js_func = execjs.compile(js_func_str)# 执行获取cookie,注意参数顺序cookie_str = js_func.call(res_521_js_name, res_521_js_arg)# 将cookie 转换成 字典格式str = cookie_str.replace("document.cookie='", "")clearance = str.split(';')[0]cookie_dic = {clearance.split('=')[0]: clearance.split('=')[1]}return cookie_dic#数据保存到csv文件def write_data(self, data):with open("retdata_new.csv", "a+", encoding="utf-8", errors='ignore', newline="") as f:f_csv = csv.writer(f)f_csv.writerow(data)#获取每一页电影的连接def get_mv_urls(self, html):urls = re.compile(r'<h3 class=\\"normal mt6\\"><a target=\\"_blank\\" href=\\"(.*?)\\">.*?</h3>')urls_list = urls.findall(html, re.S)return urls_list#获取第一页的信息def get_frist_page_info(self, html):soup = BeautifulSoup(html, 'lxml')lable1 = soup.find_all('div', attrs={'class': 'clearfix'})try:title = lable1[0].find_all("h1")[0].get_text()except:title = "NAN"try:year = lable1[0].find_all("a")[0].get_text()except:year = "NAN"try:lable2 = soup.find_all('div', attrs={"class": "otherbox __r_c_"})[0].get_text()except:lable2 = "NAN"try:lable_url = soup.find_all('p', attrs={"class": "tr mt15 px12 __r_c_"})[0].find_all('a')[0].get('href')except:lable_url = "NAN"try:page3_url = soup.find_all('dd', attrs={'pan' : 'M14_Movie_Overview_BaseInfo'})[-1].find_all('a')[-1].get('href')except:page3_url = "NAN"ret_list = [title, year, lable2, lable_url, page3_url]return ret_list#获取票房信息def get_check_info(self, html):info_reg = re.compile(r'"RatingFinal":(.*?),.*?"Usercount":(.*?),"AttitudeCount":(.*?),.*?"TotalBoxOffice":"(.*?)","TotalBoxOfficeUnit":"(.*?)","TodayBoxOffice')infos = info_reg.findall(html, re.S)try:data = [x for x in infos[0]]except:data = [0, 0, 0, 0, 0]return data#获取每一个人的详细信息def get_writer_info(self, html, write_id, writer_url):soup = BeautifulSoup(html, 'lxml')try:move_num = soup.find_all('div', attrs={'class' : 'per_iawardsboxer __r_c_'})[0].get_text().replace("\n", "").replace('更多获奖记录', "")except:move_num = "获奖0提名0"a = urllib.parse.quote(writer_url)like_url = 'http://service.library.mtime.com/Person.api?Ajax_CallBack=true&Ajax_CallBackType=Mtime.Library.Services&Ajax_CallBackMethod=GetPersonRatingInfo&Ajax_CrossDomain=1&Ajax_RequestUrl=' + a + '&Ajax_CallBackArgument0=' + write_idlike_html = self.get_html(like_url)like = re.compile(r'"finalRating":(.*?),"')like_mean = like.findall(like_html)return [move_num] + like_mean#获取第二页的信息  职位和对应的连接 就是编剧导演。。。def get_page2_infos(self, html, data):soup = BeautifulSoup(html, 'lxml')reg = soup.find_all('div', attrs={'class': 'credits_list'})for i in range(3):try:title = reg[i].find_all('h4')[0].get_text()a0 = 0for a in reg[i].find_all('a'):if a0 > 4:breakname, writer_url = title + " " + a.get_text(), a.get('href')writer_id = writer_url.split("/")[-2]writer_html = self.get_html(writer_url)write_list = self.get_writer_info(writer_html, writer_id, writer_url)data += [name] +  write_lista0 += 1except:continuereg1 = soup.find_all('div', attrs={'class' : 'credits_l'})[0].find_all('a')for i in [1, 4]:name, writer_url = "演员" + " " + reg1[i].get_text(), reg1[i].get('href')writer_id = writer_url.split("/")[-2]writer_html = self.get_html(writer_url)write_list = self.get_writer_info(writer_html, writer_id, writer_url)data += [name] + write_listprint(data)self.write_data(data)#获取第三页的信息 主要是发行公司和制作公司def get_page3_info(self, html):soup = BeautifulSoup(html, 'lxml')try:res = soup.find_all('div', attrs={'class': 'clearfix'})[1].find_all('div')make = res[0].get_text()send = res[1].get_text()return [make, send]except:return ["NAN", "NAN"]#主程序def run(self):for i in range(1, 700):print("第%s页"%i)url = 'http://service.channel.mtime.com/service/search.mcs?Ajax_CallBack=true&Ajax_CallBackType=Mtime.Channel.Pages.SearchService&Ajax_CallBackMethod=SearchMovieByCategory&Ajax_CrossDomain=1&Ajax_RequestUrl=http%3A%2F%2Fmovie.mtime.com%2Fmovie%2Fsearch%2Fsection%2F%23pageIndex%3D2%26nation%3D138&t=20202281963365098&Ajax_CallBackArgument0=&Ajax_CallBackArgument1=0&Ajax_CallBackArgument2=138&Ajax_CallBackArgument3=0&Ajax_CallBackArgument4=0&Ajax_CallBackArgument5=0&Ajax_CallBackArgument6=0&Ajax_CallBackArgument7=0&Ajax_CallBackArgument8=&Ajax_CallBackArgument9=0&Ajax_CallBackArgument10=0&Ajax_CallBackArgument11=0&Ajax_CallBackArgument12=0&Ajax_CallBackArgument13=0&Ajax_CallBackArgument14=1&Ajax_CallBackArgument15=0&Ajax_CallBackArgument16=1&Ajax_CallBackArgument17=4&Ajax_CallBackArgument18='+str(i)+'&Ajax_CallBackArgument19=0'html = self.get_html(url)for mv_url in self.get_mv_urls(html):try:bind_id = re.findall(r"\d+", mv_url)[0]check_url = 'http://service.library.mtime.com/Movie.api?Ajax_CallBack=true&Ajax_CallBackType=Mtime.Library.Services&Ajax_CallBackMethod=GetMovieOverviewRating&Ajax_CrossDomain=1&' \'Ajax_RequestUrl=' + mv_url +'&Ajax_CallBackArgument0=' + str(bind_id)check_html = self.get_html(check_url)check_info = self.get_check_info(check_html)piao = check_info[-2]danwei = check_info[-1]if danwei == '万' and float(piao) < 5000:print("票房小于0.5亿")continuemv_html = self.get_html(mv_url)info1_list = self.get_frist_page_info(mv_html)page2_url = info1_list[-2]page3_url = info1_list[-1]if page3_url == "NAN":continuepage3_html = self.get_html(page3_url)page3_list = self.get_page3_info(page3_html)data_list = info1_list[:-2] + check_info + page3_listif page2_url == "NAN":continuepage2_html = self.get_html(page2_url)self.get_page2_infos(page2_html, data_list)except:continue
if __name__ == "__main__":MVProcess()

python 爬取movie时光网相关推荐

python爬取网站时，一键获取headers、url等信息（真的是让我爬取网站时，省了不少力气，作为小秘密分享给大家喽）
python爬取网站时,一键获取headers.url等信息 (真的是让我爬取网站时,省了不少力气,作为小秘密分享给大家喽) 作为Python的使用者,我今天就和大家来分享分享我学习python的经验 ...
python爬取网页时返回http状态码HTTP Error 418以及如何查看自己的User-Agent
** 问题:urllib.error.HTTPError: HTTP Error 418: ** 问题描述:当我使用Python的request爬取网页时返回了http状态码为418, 错误描述:经过 ...
python爬取网页时出现乱码（中文）
在爬取某电子书网站源码的时候,所爬取的源码内的中文内容出现了乱码,代码如下爬取的结果: 中文部分出现了乱码,猜测应该是由编码问题造成的,首先想到的是不是采用utf-8的编码方式于是先改成了这样: ...
python 爬虫输出为空,python爬取文件时，内容为空
解决方式: img_res = requests.get(src,headers=header) 在header中加上referer防盗链加上防盗链header的例子: header = {&quo ...
python多线程爬虫数据顺序_多线程爬取小说时如何保证章节的顺序
前言爬取小说时,以每一个章节为一个线程进行爬取,如果不加以控制的话,保存的时候各个章节之间的顺序会乱掉. 当然,这里说的是一本小说保存为单个txt文件,如果以每个章节为一个txt文件,自然不会存在这 ...
python抓取数据时失败_爬取数据缺失的补坑，Python数据爬取的坑坑洼洼如何铲平...
渣渣业余选手讲解,关于爬取数据缺失的补坑,一点点关于Python数据爬取的坑坑洼洼如何铲平,个人的一些心得体会,还有结合实例的数据缺失的补全,几点参考,仅供观赏,如有雷同,那肯定是我抄袭的! 在使用P ...
python爬取百度使用kw关键字爬取时出现，百度安全验证，解决方法
python爬取百度使用kw关键字爬取时出现,百度安全验证,解决方法之前爬取百度用kw时的代码(没有任何问题) import requestsurl = 'http://www.baidu.com/ ...
Php使用selenium爬虫,selenium,python爬虫_使用selenium爬取网站时输出结果不正确，selenium,python爬虫 - phpStudy...
使用selenium爬取网站时输出结果不正确网站链接:http://www.ncbi.nlm.nih.gov/pubmed?term=(%222013%22%5BDate%20-%20Publica ...
python爬取京东数据加载失败_python 爬京东数据时，无法登陆。
想使用python爬取京东的快递信息,现在需要解决的首要问题是使用python模拟浏览器登陆,遇到了下面的问题. _t _ntNBMNX ({"username":"\u ...
如何使用python编程抢京东优惠券知乎_学好Python爬取京东知乎价值数据
原标题:学好Python爬取京东知乎价值数据 Python爬虫为什么受欢迎如果你仔细观察,就不难发现,懂爬虫.学习爬虫的人越来越多,一方面,互联网可以获取的数据越来越多,另一方面,像 Python这 ...

python 爬取movie时光网

代码如下：欢迎指点不足之处

python 爬取movie时光网相关推荐

最新文章

热门文章