文章目录

前言

一、数据集介绍

二、数据预处理

1.引入库

2.剔除无价值数据

三、情感信息提取及可视化

四、使用线性支持向量分类模型进行情感预测

优化处理操作——向下采样：

五、使用LDA主题分类模型进行数据分析

1. 积极评论：

2. 消极评论：

六、结果分析及结论

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_splitimport jieba
import replt.rcParams["font.sans-serif"] = ["SimHei"]plt.rcParams["axes.unicode_minus"] = False
import warnings
warnings.filterwarnings("ignore")

data["review"] = data["review"].astype('str')import re
# 去除数字、携程等词语
strinfo = re.compile('[0-9]|酒店|携程|年月日|北京|上海|重庆|广州|杭州|南京|成都|苏州|西安|东莞|长沙|济南|深圳|西路|东路')
data["review"] =  data["review"].apply(lambda x: strinfo.sub('',x))import re
# 由于有的时候jupyternotebook会出bug,去除一次并不能去除掉年月日，所以保险起见，再去除一次
strinfo = re.compile('[0-9]|酒店|携程|年月日')
data["review"] =  data["review"].apply(lambda x: strinfo.sub('',x))#第一步 将空字符的行替换为nan，方便进行删除
data.replace(to_replace=r'^\s*$', value=np.nan, regex=True, inplace=True)
data.replace(to_replace=r'[a-zA-Z]', value=np.nan, regex=True, inplace=True)
print(data)#第二步 删除所有值为nan的行
data.dropna(axis=0, how='any', inplace=True)

# 提取正负面评论信息
posdata = data[data['label'] == 1]['review']
negdata = data[data['label'] == 0]['review']

from wordcloud import WordCloud
import jieba
import matplotlib.pyplot as plt
import PILtext = ''
for s in posdata:text += s
data_cut = ' '.join(jieba.lcut(text))path = '停用词汇总.txt'
f = open(path,"r",encoding='utf-8').read()  #设置文件对象word_cloud = WordCloud(font_path="simsun.ttc", background_color="white", stopwords=f)word_cloud.generate(data_cut)plt.subplots(figsize=(12,8))
plt.imshow(word_cloud)
plt.axis("off")
plt.show()

from jieba import analyse
key_words = jieba.analyse.extract_tags(sentence=text, topK=20, withWeight=True, allowPOS=())
key_words

from wordcloud import WordCloud
import jieba
import matplotlib.pyplot as plt
import PILtext1 = ''
for s in negdata:text1 += s
data_cut = ' '.join(jieba.lcut(text1))path = '停用词汇总.txt'
f = open(path,"r",encoding='utf-8').read()  #设置文件对象word_cloud = WordCloud(font_path="simsun.ttc", background_color="white", stopwords=f)word_cloud.generate(data_cut)plt.subplots(figsize=(12,8))
plt.imshow(word_cloud)
plt.axis("off")
plt.show()

from jieba import analyse
key_words = jieba.analyse.extract_tags(sentence=text1, topK=20, withWeight=True, allowPOS=())
key_words

from sklearn.feature_extraction.text import TfidfVectorizer as TFIDF  # 原始文本转化为tf-idf的特征矩阵
from sklearn.svm import LinearSVC
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split# 将有标签的数据集划分成训练集和测试集
train_X,valid_X,train_y,valid_y = train_test_split(data['review'],data['label'],test_size=0.3,random_state=42)train_X.shape,train_y.shape,valid_X.shape,valid_y.shape

# 模型构建
model_tfidf = TFIDF(min_df=5, max_features=5000, ngram_range=(1,3), use_idf=1, smooth_idf=1)
# 学习idf vector
model_tfidf.fit(train_X)
# 把文档转换成 X矩阵（该文档中该特征词出现的频次），行是文档个数，列是特征词的个数
train_vec = model_tfidf.transform(train_X)# 模型训练
model_SVC = LinearSVC()
clf = CalibratedClassifierCV(model_SVC)
clf.fit(train_vec,train_y)

# 把文档转换成矩阵
valid_vec = model_tfidf.transform(valid_X)
# 验证
pre_valid = clf.predict_proba(valid_vec)pre_valid = clf.predict(valid_vec)
print('正例:',sum(pre_valid == 1))
print('负例:',sum(pre_valid == 0))

from sklearn.metrics import accuracy_scorescore = accuracy_score(pre_valid,valid_y)
print("准确率:",score)

def get_balanced_words(size,positive_comment=data[data['label'] == 1],negtive_comment=data[data['label'] == 0]):word_size = size // 2#获取正负评论数num_pos = positive_comment.shape[0]num_neg = negtive_comment.shape[0]#     当 正(负)品论数中<采样数量/2 时，进行上采样，否则都是下采样；#     其中pandas的sample方法里的repalce参数代表是否进行上采样，默认不进行balanced_words = pd.concat([positive_comment.sample(word_size,replace=num_pos < word_size,random_state=0),negtive_comment.sample(word_size,replace=num_neg < word_size,random_state=0)])#     打印样本个数print('样本总数：', balanced_words.shape[0])print('正样本数：', balanced_words[data['label'] == 1].shape[0])print('负样本数：', balanced_words[data['label'] == 0].shape[0])print('')return balanced_wordsdata_4888 = get_balanced_words(4888)

from sklearn.feature_extraction.text import TfidfVectorizer as TFIDF  # 原始文本转化为tf-idf的特征矩阵
from sklearn.svm import LinearSVC
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split# 将有标签的数据集划分成训练集和测试集
train_X,valid_X,train_y,valid_y = train_test_split(data_4888['review'],data_4888['label'],test_size=0.3,random_state=23)train_X.shape,train_y.shape,valid_X.shape,valid_y.shape

# 模型构建
model_tfidf = TFIDF(min_df=2, max_features=5000, ngram_range=(1,3), use_idf=1, smooth_idf=1)
# 学习idf vector
model_tfidf.fit(train_X)
# 把文档转换成 X矩阵（该文档中该特征词出现的频次），行是文档个数，列是特征词的个数
train_vec = model_tfidf.transform(train_X)# 模型训练
model_SVC = LinearSVC()
clf = CalibratedClassifierCV(model_SVC)
clf.fit(train_vec,train_y)

# 把文档转换成矩阵
valid_vec = model_tfidf.transform(valid_X)
# 验证
pre_valid = clf.predict_proba(valid_vec)pre_valid = clf.predict(valid_vec)
print('正例:',sum(pre_valid == 1))
print('负例:',sum(pre_valid == 0))

from sklearn.metrics import accuracy_scorescore = accuracy_score(pre_valid,valid_y)
print("准确率:",score)

posdata=pd.DataFrame(posdata).dropna(axis=0)
posdata.columns=['comment']negdata=pd.DataFrame(negdata).dropna(axis=0)
negdata.columns=['comment']

#利用jieba中文分词
import jieba
import jieba.posseg as psg#格式转换 否则会报错  'float' object has no attribute 'decode'
df1 = pd.DataFrame(posdata.astype(str))def chinese_word_cut(mytext):return ' '.join(jieba.cut(mytext))#增加一列数据
df1['content_cutted'] = df1['comment'].apply(chinese_word_cut)#格对负面评论进行操作
df2 = pd.DataFrame(negdata.astype(str))def chinese_word_cut(mytext):return ' '.join(jieba.cut(mytext))#增加一列数据
df2['content_cutted'] = df2['comment'].apply(chinese_word_cut)

path = '停用词汇总.txt'
f = open(path,"r",encoding='utf-8').read()
stopwords=list(f)#计算TF-IDF值
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer#设置特征数
n_features = 2000tf_vectorizer = TfidfVectorizer(strip_accents = 'unicode',max_features=n_features,stop_words=stopwords,max_df = 0.99,min_df = 0.002) #去除文档内出现几率过大或过小的词汇
tf = tf_vectorizer.fit_transform(df1.content_cutted)print(tf.shape)
print(tf)

#LDA分析
from sklearn.decomposition import LatentDirichletAllocation#设置主题数
n_topics = 3lda = LatentDirichletAllocation(n_components=n_topics,max_iter=100,learning_method='online',learning_offset=50,random_state=0)
lda.fit(tf)#显示主题数 model.topic_word_
print(lda.components_)
#几个主题就是几行 多少个关键词就是几列
print(lda.components_.shape)                         #主题-关键词分布
def print_top_words(model, tf_feature_names, n_top_words):for topic_idx,topic in enumerate(model.components_):    # lda.component相当于model.topic_word_print('Topic #%d:' % topic_idx)print(' '.join([tf_feature_names[i] for i in topic.argsort()[:-n_top_words-1:-1]]))print("")#定义好函数之后 暂定每个主题输出前20个关键词
n_top_words = 20
tf_feature_names = tf_vectorizer.get_feature_names()
#调用函数
print_top_words(lda, tf_feature_names, n_top_words)

import pyLDAvis
import pyLDAvis.gensim_models
red_vis_data = pyLDAvis.sklearn.prepare(lda,tf,tf_vectorizer)
pyLDAvis.display(red_vis_data)

#计算TF-IDF值
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizern_features = 2000tf_vectorizer = TfidfVectorizer(strip_accents = 'unicode',max_features=n_features,stop_words=stopwords,max_df = 0.99,min_df = 0.002)
tf = tf_vectorizer.fit_transform(df2.content_cutted)print(tf.shape)
print(tf)

#LDA分析
from sklearn.decomposition import LatentDirichletAllocation#设置主题数
n_topics = 2lda = LatentDirichletAllocation(n_components=n_topics,max_iter=100,learning_method='online',learning_offset=50,random_state=0)
lda.fit(tf)#显示主题数 model.topic_word_
print(lda.components_)
#几个主题就是几行 多少个关键词就是几列
print(lda.components_.shape)                         #主题-关键词分布
def print_top_words(model, tf_feature_names, n_top_words):for topic_idx,topic in enumerate(model.components_):    # lda.component相当于model.topic_word_print('Topic #%d:' % topic_idx)print(' '.join([tf_feature_names[i] for i in topic.argsort()[:-n_top_words-1:-1]]))print("")#定义好函数之后 暂定每个主题输出前20个关键词
n_top_words = 20
tf_feature_names = tf_vectorizer.get_feature_names()
#调用函数
print_top_words(lda, tf_feature_names, n_top_words)

import pyLDAvis
import pyLDAvis.gensim_models
red_vis_data = pyLDAvis.sklearn.prepare(lda,tf,tf_vectorizer)
pyLDAvis.display(red_vis_data)