爬取豆瓣电影数据并进行分析可视化

学习爬虫爬取豆瓣电影数据并进行分析，整体流程如下：

1、爬取豆瓣电影数据

2、读取豆瓣电影数据

3、统计各个电影的评论数

4、读取某个电影的全部评论内容

5、获取某个电影的关键词并生成词云图

6、对电影数据的关键词和评分进行辩证分析并生成热力图

让我们开始吧！废话不多说，直接上代码 #------#

爬取豆瓣电影数据

import requests
from bs4 import BeautifulSoup
from collections import OrderedDict
import pandas as pd
# 设定headers
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
movie_info = OrderedDict()def detail_handle(url):html = requests.get(url,headers = headers)soup = BeautifulSoup(html.text,'lxml')movie_info['movie_rank'] = soup.find_all('span',class_="top250-no")[0].stringmovie_info['movie_name'] = soup.find_all('span',property="v:itemreviewed")[0].stringsoup_div = soup.find(id="info")movie_info['movie_director'] = handle_mul_tags(soup_div.find_all('span')[0].find_all('a'))movie_info['movie_writer'] = handle_mul_tags(soup_div.find_all('span')[3].find_all('a'))movie_info['movie_starring'] = handle_mul_tags(soup_div.find_all('span')[6].find_all('a'))movie_info['movie_type'] = handle_mul_tags(soup_div.find_all('span',property="v:genre"))movie_info['movie_country'] = soup_div.find(text = '制片国家/地区:').next_element.lstrip().rstrip()movie_info['movie_language'] = soup_div.find(text = '语言:').next_element.lstrip().rstrip()movie_info['movie_release_date'] = handle_mul_tags(soup_div.find_all('span',property="v:initialReleaseDate"))movie_info['movie_run_time'] = handle_mul_tags(soup_div.find_all('span',property="v:runtime"))movie_second_name = ''try:movie_info['movie_second_name'] = soup_div.find(text = '又名:').next_element.lstrip().rstrip()except AttributeError:print('{}没有别名'.format(movie_info['movie_name']))movie_info['movie_second_name'] = movie_second_namemovie_info['movie_rating'] = soup.find_all('strong',property="v:average")[0].stringmovie_info['movie_comment_users'] = soup.find_all('span',property="v:votes")[0].stringsoup_div_for_ratings = soup.find('div',class_="ratings-on-weight")movie_info['movie_five_star_ratio'] = soup_div_for_ratings.find_all('div')[0].find(class_="rating_per").stringmovie_info['movie_four_star_ratio'] = soup_div_for_ratings.find_all('div')[2].find(class_="rating_per").stringmovie_info['movie_three_star_ratio'] = soup_div_for_ratings.find_all('div')[4].find(class_="rating_per").stringmovie_info['movie_two_star_ratio'] = soup_div_for_ratings.find_all('div')[6].find(class_="rating_per").stringmovie_info['movie_one_star_ratio'] = soup_div_for_ratings.find_all('div')[8].find(class_="rating_per").stringreturn movie_infodef handle_mul_tags(soup_span): # 获取多个标签结果将合并在一起，以/分隔info = ''for second_span in soup_span:info = ('' if (info == '') else '/').join((info,second_span.string))return infodef crawl():htmls = ['https://movie.douban.com/top250?start={}&filter='.format(str(page)) for page in range(0,250,25)]for html in htmls:html_url = requests.get(html,headers = headers)soup = BeautifulSoup(html_url.text,'lxml')movie_htmls = soup.select('.pic')for movie_html in movie_htmls:url = movie_html.select('a')[0]['href']return detail_handle(url)

对电影数据进行分析和可视化

import sqlite3
import pandas as pd
import jieba
import math
import pyecharts.options as opts
from pyecharts.charts import WordCloud
import os
os.chdir('C:\\Users\\Theo.chen\\Desktop\\数据分析项目\\')import matplotlib as mpl
mpl.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams['axes.unicode_minus'] = Falseconn = sqlite3.connect('douban_comment_data.db')
comment_data = pd.read_sql_query('select * from comment;', conn)
movie_data = pd.read_excel('douban_movie_data.xlsx')
FILTER_WORDS = ['知道','影评','影片','小编','没有','一个','\n','good','is','thing','这个','就是','什么','真的','of','我们','最后','一部','the','片子','这么','那么','不是','还是','时候','觉得','电影','但是','hope','Hope','best','因为','只是','故事','看过','豆瓣','maybe','这部']def get_movie_idList(min_comment_count):movie_list = comment_data['MOVIEID'].value_counts()movie_list = movie_list[movie_list.values > min_comment_count]
# 筛选出评论数>100的电影return movie_list.indexdef get_comment_keywords(movie_id,count):comment_list = comment_data[comment_data['MOVIEID'] == movie_id]['CONTENT']comment_str_all = ''for comment in comment_list:comment_str_all += comment + '\n'seg_list = list(jieba.cut(comment_str_all))keywords_counts = pd.Series(seg_list)keywords_counts = keywords_counts[keywords_counts.str.len() > 1]keywords_counts = keywords_counts[~keywords_counts.str.contains('|'.join(FILTER_WORDS))]keywords_counts = keywords_counts.value_counts()[:count]return keywords_countsdef get_movie_name_and_score(movie_id):movie_link = 'https://movie.douban.com/subject/{}/'.format(movie_id)search_result = movie_data[movie_data['链接'] == movie_link].iloc[0]movie_name = search_result['电影名']movie_score = search_result['评分']return (movie_name,movie_score)def generate_wordcloud(word_list,path_name):wordcloud = WordCloud()wordcloud.add("",tuple(zip(keywords_counts.index,keywords_counts)),word_size_range = [20,100])wordcloud.render(path_name)print(f"Generate word cloud file done: {path_name}")# 创建列表, 每个列表都含有10个列表
kw_list_by_score=[[] for i in range(10)]
kw_counts_by_score = [[] for i in range(10)]movie_id_list = get_movie_idList(300)
for movie_id in movie_id_list:word_list = get_comment_keywords(movie_id,30)movie_name, movie_score = get_movie_name_and_score(movie_id)try:kw_list_by_score[math.floor(movie_score)].extend(word_list.index)kw_counts_by_score[math.floor(movie_score)].extend(word_list.values)except:print('Something Error!!!')for i in range(10):if kw_list_by_score[i]:kw30_with_counts = pd.DataFrame({'kw':kw_list_by_score[i],'count':kw_counts_by_score[i]})kw30_with_counts = kw30_with_counts.groupby('kw').sum()kw30_with_counts = kw30_with_counts.sort_values(by = 'count', ascending = False)[:30]counts_sum = kw30_with_counts['count'].sum()kw30_with_counts['percentage'] = kw30_with_counts['count'] / counts_sumkw30_with_counts.to_csv('{}_movie_keywords.csv'.format(i))from pyecharts.charts import HeatMapkw_counts_by_score=[[] for _ in range(10)]
for i in range(4,10):kw_counts_by_score[i] = pd.read_csv('{}_movie_keywords.csv'.format(i))kw_percentage_df = pd.DataFrame([],columns = list(range(4,10)),index=kw_counts_by_score[9]['kw'][:10])for i in range(4,10):kw=kw_counts_by_score[i]kw=kw[kw['kw'].isin(kw_percentage_df.index)]kw_percentage_df[i] = pd.Series(list(kw['percentage']),index=kw['kw'])kw_percentage_df.fillna(0,inplace=True)data=[]
i = 0
for index in kw_percentage_df.index:j=0for column in kw_percentage_df.columns:data.append([j,i,kw_percentage_df[column][index]*100])j+=1i+=iheatmap = HeatMap()
heatmap.add_xaxis(list(kw_percentage_df.columns))
heatmap.add_yaxis("电影评论关键词热力图",list(kw_percentage_df.index),data)
heatmap.set_global_opts(visualmap_opts=opts.VisualMapOpts(min_= 0,max_=10,orient='horizontal'),)
heatmap.render(path="heatmap.html")

爬取豆瓣电影数据并进行分析可视化相关推荐

python实现爬虫探探_全栈 - 9 实战爬取豆瓣电影数据
这是全栈数据工程师养成攻略系列教程的第九期:9 实战爬取豆瓣电影数据. 掌握了爬虫的基本原理和代码实现,现在让我们通过实战项目巩固一下. 确定目标在写爬虫之前应当想清楚:我需要哪方面的数据?需要包 ...
Java爬取豆瓣电影数据
所用到的技术有Jsoup,HttpClient. Jsoup jsoup 是一款Java 的HTML解析器,可直接解析某个URL地址.HTML文本内容.它提供了一套非常省力的API,可通过DOM,CS ...
python爬取百部电影数据，我分析出了一个残酷的真相
2019年就这么匆匆过去了,就在前几天国家电影局发布了2019年中国电影市场数据,数据显示去年总票房为642.66亿元,同比增长5.4%:国产电影总票房411.75亿元,同比增长8.65%,市场占比 ...
BeautifulSoup爬取豆瓣电影数据
BeautifulSoup爬取豆瓣TOP250 豆瓣爬取地址 https://movie.douban.com/top250?format=text BeautifulSoup官网地址 https:/ ...
【爬虫实践】记一次Scrapy框架入门使用爬取豆瓣电影数据
本次的学习分享主要是使用一次Scrapy框架,毕竟在很多次的时候,自己在提取一些或是需要实验数据的时候,数据量要求不大,很快便能通过简单的request等库进行调用,然后获取数据. 这次,则是想要使用 ...
Java爬取豆瓣电影数据，京东高级java面试
.build(); return config; } 根据请求地址获取响应信息方法,获取成功后返回响应信息. public static String doGetHtml(String url, Ma ...
python爬取豆瓣电影信息数据
题外话+ 大家好啊,最近自己在做一个属于自己的博客网站(准备辞职回家养老了,明年再战)在家里琐事也很多, 加上自己一回到家就懒了(主要是家里冷啊! 广东十几度,老家几度,躲在被窝瑟瑟发抖,) 由于 ...
python爬取豆瓣电影并分析_Python实战之如何爬取豆瓣电影？本文教你
爬虫又称为网页蜘蛛,是一种程序或脚本. 但重点在于,它能够按照一定的规则,自动获取网页信息. 爬虫的基本原理--通用框架 1.挑选种子URL: 2.讲这些URL放入带抓取的URL列队: 3.取出带抓取 ...
爬虫练习-爬取豆瓣电影TOP250的数据
前言: 爬取豆瓣电影TOP250的数据,并将爬取的数据存储于Mysql数据库中本文为整理代码,梳理思路,验证代码有效性--2020.1.4 环境: Python3(Anaconda3) PyChar ...
爬取豆瓣电影短评并使用词云简单分析top50
先使用进程池爬取豆瓣电影短评 import requests import re import random import time import pandas as pd from pymongo ...

爬取豆瓣电影数据并进行分析可视化

爬取豆瓣电影数据并进行分析可视化相关推荐

最新文章

热门文章