# coding: utf-8
import re
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import confusion_matrix
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import itertools###########################词袋模型特征############################################
#重组为新的句子
def clean_text(text):"""去掉html标签、移除标点、切分成词/token、去掉停用词、重组为新的句子:param text::return:"""print(text)text = BeautifulSoup(text, 'html.parser').get_text()text = re.sub(r'[^a-zA-Z]', ' ', text)words = text.lower().split()stopwords = {}.fromkeys([line.rstrip() for line in open('../stopwords/stopwords_english.txt')])eng_stopwords = set(stopwords)print(eng_stopwords)words = [w for w in words if w not in eng_stopwords]print(words)return ' '.join(words)#混淆矩阵
def plot_confusion_matrix(cm, classes,title='Confusion matrix',cmap=plt.cm.Blues):"""This function prints and plots the confusion matrix."""plt.imshow(cm, interpolation='nearest', cmap=cmap)plt.title(title)plt.colorbar()tick_marks = np.arange(len(classes))plt.xticks(tick_marks, classes, rotation=0)plt.yticks(tick_marks, classes)thresh = cm.max() / 2.for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):plt.text(j, i, cm[i, j],horizontalalignment="center",color="white" if cm[i, j] > thresh else "black")plt.tight_layout()plt.ylabel('True label')plt.xlabel('Predicted label')if __name__=='__main__':#读取数据df = pd.read_csv('../data/labeledTrainData.tsv', sep='\t', escapechar='\\')print(df.head(5))#数据清洗,对df中的每一个Serial进行清洗df['clean_review'] = df.review.apply(clean_text)print(df['clean_review'])#抽取bag of words特征(用sklearn的CountVectorizer)vectorizer = CountVectorizer(max_features=5000)train_data_features = vectorizer.fit_transform(df.clean_review).toarray()print(train_data_features)# 数据切分X_train, X_test, y_train, y_test = train_test_split(train_data_features, df.sentiment, test_size=0.2,random_state=0)print(X_train,X_test,y_train,y_test)# ### 训练分类器LR_model = LogisticRegression()LR_model = LR_model.fit(X_train, y_train)y_pred = LR_model.predict(X_test)cnf_matrix = confusion_matrix(y_test, y_pred)print("Recall metric in the testing dataset: ", cnf_matrix[1, 1] / (cnf_matrix[1, 0] + cnf_matrix[1, 1]))print("accuracy metric in the testing dataset: ", (cnf_matrix[1, 1] + cnf_matrix[0, 0]) / (cnf_matrix[0, 0] + cnf_matrix[1, 1] + cnf_matrix[1, 0] + cnf_matrix[0, 1]))# Plot non-normalized confusion matrixclass_names = [0, 1]plt.figure()plot_confusion_matrix(cnf_matrix, classes=class_names, title='Confusion matrix')plt.show()

import re
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
from sklearn.metrics import confusion_matrix
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import nltk
import warnings
from gensim.models.word2vec import Word2Vec
from nltk.corpus import stopwords
import matplotlib.pyplot as plt
import itertools
warnings.filterwarnings("ignore")def clean_text(text, remove_stopwords=False):text = BeautifulSoup(text, 'html.parser').get_text()text = re.sub(r'[^a-zA-Z]', ' ', text)words = text.lower().split()eng_stopwords = set(stopwords.words('english'))if remove_stopwords:words = [w for w in words if w not in eng_stopwords]return wordsdef split_sentences(review):#print(type(review))raw_sentences=tokenizer.tokenize(str(review).strip())sentences = [clean_text(s) for s in raw_sentences if s]return sentencesdef to_review_vector(review):global word_vecreview = clean_text(review, remove_stopwords=True)# print (review)# words = nltk.word_tokenize(review)word_vec = np.zeros((1, 300))for word in review:# word_vec = np.zeros((1,300))if word in model:word_vec += np.array([model[word]])# print (word_vec.mean(axis = 0))return pd.Series(word_vec.mean(axis=0))def plot_confusion_matrix(cm, classes,title='Confusion matrix',cmap=plt.cm.Blues):"""This function prints and plots the confusion matrix."""plt.imshow(cm, interpolation='nearest', cmap=cmap)plt.title(title)plt.colorbar()tick_marks = np.arange(len(classes))plt.xticks(tick_marks, classes, rotation=0)plt.yticks(tick_marks, classes)thresh = cm.max() / 2.for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):plt.text(j, i, cm[i, j],horizontalalignment="center",color="white" if cm[i, j] > thresh else "black")plt.tight_layout()plt.ylabel('True label')plt.xlabel('Predicted label')if __name__ == '__main__':#读取数据df = pd.read_csv('../data/labeledTrainData.tsv', sep='\t', escapechar='\\')#数据清洗df['clean_review'] = df.review.apply(clean_text)review_part = df['clean_review']#nltk库分词tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')sentences = sum(review_part.apply(split_sentences), [])sentences_list = []for line in sentences:sentences_list.append(nltk.word_tokenize(str(line).strip()))#word2vecnum_features = 300  # Word vector dimensionalitymin_word_count = 40  # Minimum word countnum_workers = 4  # Number of threads to run in parallelcontext = 10  # Context window sizemodel_name = '{}features_{}minwords_{}context.model'.format(num_features, min_word_count, context)model = Word2Vec(sentences_list, workers=num_workers, size=num_features, min_count=min_word_count, window=context)model.init_sims(replace=True)model.save('word2vec.models')train_data_features = df.review.apply(to_review_vector)X_train, X_test, y_train, y_test = train_test_split(train_data_features, df.sentiment, test_size=0.2, random_state=0)LR_model = LogisticRegression()LR_model = LR_model.fit(X_train, y_train)y_pred = LR_model.predict(X_test)cnf_matrix = confusion_matrix(y_test, y_pred)print("Recall metric in the testing dataset: ", cnf_matrix[1, 1] / (cnf_matrix[1, 0] + cnf_matrix[1, 1]))print("accuracy metric in the testing dataset: ", (cnf_matrix[1, 1] + cnf_matrix[0, 0]) / (cnf_matrix[0, 0] + cnf_matrix[1, 1] + cnf_matrix[1, 0] + cnf_matrix[0, 1]))# Plot non-normalized confusion matrixclass_names = [0, 1]plt.figure()plot_confusion_matrix(cnf_matrix , classes=class_names, title='Confusion matrix')plt.show()