Python爬取《悲伤逆流成河》猫眼信息

1.猫眼电影短评接口

http://maoyan.com/films/1217236
我们直接访问这个，在web端只能看到最热的10条短评，那怎么获取到所有短评呢？
(1) 访问上面的链接，按下F12，然后点击图片上的图标，把浏览模式(响应式设计模式，火狐快捷键Ctrl+Shift+M)改为手机模式，刷新页面。

(2)换用谷歌浏览器，F12下进行上面操作，加载完毕后下拉短评，页面继续加载，找到含有offset和startTime的加载条，发现它的Response中包含我们想要的数据，为json格式。

2.获取短评

(1)简单分析

通过上面分析

Request URL: http://m.maoyan.com/mmdb/comments/movie/1217236.json?v=yes&offset=0&startTime=0%2021%3A09%3A31
Request Method: GET

下滑了几次次，我发现了下面规律：
分析上面数据变化，可以大致猜测出：offset表示该接口显示评论开始位置，每个页面15条，比如：15，则显示15-30这中间的15条评论； startTime表示当前评论的时间，固定格式（2018-10-06）。
另外接口最后的%2021%3A09%3A31是不变的。
(2)代码获取

import requests
from fake_useragent import UserAgent
import json
'''
小编准备的python学习资料，加群：1136201545 即可免费获取！
'''
headers = {"User-Agent": UserAgent(verify_ssl=False).random,"Host":"m.maoyan.com","Referer":"http://m.maoyan.com/movie/1217236/comments?_v_=yes"}
# 猫眼电影短评接口
offset = 0
# 电影是2018.9.21上映的
startTime = '2018-09-21'
comment_api = 'http://m.maoyan.com/mmdb/comments/movie/1217236.json?_v_=yes&offset={0}&startTime={1}%2021%3A09%3A31'.format(offset,startTime)
# 发送get请求
response_comment = requests.get(comment_api,headers = headers)
json_comment = response_comment.text
json_comment = json.loads(json_comment)
print(json_comment)

返回数据：
(3)数据简单介绍
(4)数据提取

# 获取数据并存储
def get_data(self,json_comment):json_response = json_comment["cmts"]  # 列表list_info = []for data in json_response:cityName = data["cityName"]content = data["content"]if "gender" in data:gender = data["gender"]else:gender = 0nickName = data["nickName"]userLevel = data["userLevel"]score = data["score"]list_one = [self.time,nickName,gender,cityName,userLevel,score,content]list_info.append(list_one)self.file_do(list_info)

3.存储数据

# 存储文件
def file_do(list_info):# 获取文件大小file_size = os.path.getsize(r'G:\maoyan\maoyan.csv')if file_size == 0:# 表头name = ['评论日期', '评论者昵称', '性别', '所在城市','猫眼等级','评分','评论内容']# 建立DataFrame对象file_test = pd.DataFrame(columns=name, data=list_info)# 数据写入file_test.to_csv(r'G:\maoyan\maoyan.csv', encoding='gbk', index=False)else:with open(r'G:\maoyan\maoyan.csv', 'a+', newline='') as file_test:# 追加到文件后面writer = csv.writer(file_test)# 写入文件writer.writerows(list_info)

数据显示

数据分析可视化

1.提取数据

代码：

def read_csv():content = ''# 读取文件内容with open(r'G:\maoyan\maoyan.csv', 'r', encoding='utf_8_sig', newline='') as file_test:# 读文件reader = csv.reader(file_test)i = 0for row in reader:if i != 0:time.append(row[0])nickName.append(row[1])gender.append(row[2])cityName.append(row[3])userLevel.append(row[4])score.append(row[5])content = content + row[6]# print(row)i = i + 1print('一共有：' + str(i - 1) + '条数据')return content

一共有：15195条数据

评论者性别分布可视化

# 评论者性别分布可视化
def sex_distribution(gender):# print(gender)from pyecharts import Pielist_num = []list_num.append(gender.count('0')) # 未知list_num.append(gender.count('1')) # 男list_num.append(gender.count('2')) # 女attr = ["其他","男","女"]pie = Pie("性别饼图")pie.add("", attr, list_num, is_label_show=True)pie.render("H:\PyCoding\spider_maoyan\picture\sex_pie.html")

评论者所在城市分布可视化

# 评论者所在城市分布可视化
def city_distribution(cityName):city_list = list(set(cityName))city_dict = {city_list[i]:0 for i in range(len(city_list))}for i in range(len(city_list)):city_dict[city_list[i]] = cityName.count(city_list[i])# 根据数量(字典的键值)排序sort_dict = sorted(city_dict.items(), key=lambda d: d[1], reverse=True)city_name = []city_num = []for i in range(len(sort_dict)):city_name.append(sort_dict[i][0])city_num.append(sort_dict[i][1])import randomfrom pyecharts import Barbar = Bar("评论者城市分布")bar.add("", city_name, city_num, is_label_show=True, is_datazoom_show=True)bar.render("H:\PyCoding\spider_maoyan\picture\city_bar.html")# 地图可视化
def render_city(cities):

每日评论总数可视化分析

def time_num_visualization(time):from pyecharts import Linetime_list = list(set(time))time_dict = {time_list[i]: 0 for i in range(len(time_list))}time_num = []for i in range(len(time_list)):time_dict[time_list[i]] = time.count(time_list[i])# 根据数量(字典的键值)排序sort_dict = sorted(time_dict.items(), key=lambda d: d[0], reverse=False)time_name = []time_num = []print(sort_dict)for i in range(len(sort_dict)):time_name.append(sort_dict[i][0])time_num.append(sort_dict[i][1])line = Line("评论数量日期折线图")line.add("日期-评论数",time_name,time_num,is_fill=True,area_color="#000",area_opacity=0.3,is_smooth=True,)line.render("H:\PyCoding\spider_maoyan\picture\c_num_line.html")

评论者猫眼等级、评分可视化

def level_score_visualization(userLevel,score):from pyecharts import PieuserLevel_list = list(set(userLevel))userLevel_num = []for i in range(len(userLevel_list)):userLevel_num.append(userLevel.count(userLevel_list[i]))score_list = list(set(score))score_num = []for i in range(len(score_list)):score_num.append(score.count(score_list[i]))pie01 = Pie("等级环状饼图", title_pos='center', width=900)pie01.add("等级",userLevel_list,userLevel_num,radius=[40, 75],label_text_color=None,is_label_show=True,legend_orient="vertical",legend_pos="left",)pie01.render("H:\PyCoding\spider_maoyan\picture\level_pie.html")pie02 = Pie("评分玫瑰饼图", title_pos='center', width=900)pie02.add("评分",score_list,score_num,center=[50, 50],is_random=True,radius=[30, 75],rosetype="area",is_legend_show=False,is_label_show=True,)pie02.render("H:\PyCoding\spider_maoyan\picture\score_pie.html")