作者：华仔徐

来源：https://zhuanlan.zhihu.com/p/28318502

文中代码较多，建议边敲边看

以下我按照代码来详细描述自己在编写爬虫代码时碰到的问题：

1.Python爬虫爬取短评代码：

# -*- coding:utf-8 -*-import randomimport time

from downloader import download as ddfrom parser import movieparser as psimport codecs

if __name__ == '__main__':templateurl = 'https://movie.douban.com/subject/26363254/comments?start={}&limit=20&sort=new_score&status=P';     with codecs.open('pjl_comment.txt', 'a', encoding='utf-8') as f:        # 4249for i in range(4249):                print ('开始爬取{}页评论...', i)targeturl = templateurl.format(i * 20)res = dd.download_page(targeturl)f.writelines(ps.get_douban_comments(res))time.sleep(1 + float(random.randint(1, 20)) / 20)

# -*- coding:utf-8 -*-from bs4 import BeautifulSoup

def get_douban_comments(res):comments_list = []  # 评论列表soup = BeautifulSoup(res)comment_nodes = soup.select('.comment > p')    for node in comment_nodes:comments_list.append(node.get_text().strip().replace("\n", "") + u'\n')    return comments_list

# -*- coding:utf-8 -*-

import requests

# 下载源代码def download_page(url):header={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0',            'Cookie':'input your login cookie'}html=requests.get(url,headers=header).content    return html

提醒：爬取短评需要登录操作，可以注册一个小号作为爬虫的工具。

爬取完成后得到一个10M的文件夹：

2. 对电影评论分词,并绘制词云

# -*- coding:utf-8 -*-import codecsfrom os import path

import jiebafrom scipy.misc import imreadfrom wordcloud import WordCloud

def get_all_keywords(file_name):word_lists = []  # 关键词列表with codecs.open(file_name, 'r', encoding='utf-8') as f:Lists = f.readlines()  # 文本列表for List in Lists:cut_list = list(jieba.cut(List))                       for word in cut_list:word_lists.append(word)word_lists_set = set(word_lists)  # 去除重复元素sort_count = []word_lists_set = list(word_lists_set)length = len(word_lists_set)        print u"共有%d个关键词" % lengthk = 1for w in word_lists_set:sort_count.append(w + u':' + unicode(word_lists.count(w)) + u"次\n")        print u"%d---" % k + w + u":" + unicode(word_lists.count(w)) + u"次"k += 1with codecs.open('count_word.txt', 'w', encoding='utf-8') as f:f.writelines(sort_count)# 绘制词云

def save_jieba_result():# 设置多线程切割jieba.enable_parallel(4)dirs = path.join(path.dirname(__file__), '../pjl_comment.txt')     with codecs.open(dirs, encoding='utf-8') as f:comment_text = f.read()cut_text = " ".join(jieba.cut(comment_text))  # 将jieba分词得到的关键词用空格连接成为字符串with codecs.open('pjl_jieba.txt', 'a', encoding='utf-8') as f:f.write(cut_text)def draw_wordcloud2():

def draw_wordcloud2():dirs = path.join(path.dirname(__file__), 'pjl_jieba.txt')    with codecs.open(dirs, encoding='utf-8') as f:comment_text = f.read()color_mask = imread("/Users/huazi/Desktop/music.jpg")  # 读取背景图片stopwords = [u'就是', u'电影', u'你们', u'这么', u'不过', u'但是', u'什么', u'没有', u'这个', u'那个', u'大家', u'比较', u'看到', u'真是',                 u'除了', u'时候', u'已经', u'可以']cloud = WordCloud(font_path="/Users/huazi/Desktop/simsunttc/simsun.ttc", background_color='white',max_words=2000, max_font_size=200, min_font_size=4, mask=color_mask, stopwords=stopwords)word_cloud = cloud.generate(comment_text)  # 产生词云word_cloud.to_file("pjl_cloud.jpg")save_jieba_result()
draw_wordcloud2()

碰到的一些问题：

1. 因为评论的数据较多，使用jieba分词工具时，时间消耗很大，提高速度的方法， 设置多线程切割： jieba.enable_parallel(4)

2. WordCloud使用时，对stopwords使用中文，一直没有效果，通过修改源代码函数达到目标：

3.最后的结果