数据准备

使用jieba分词

data_processing.py

import jiebadata_path = "sources/weibo_senti_100k.csv"
data_stop_path = "sources/hit_stopword"
data_list = open(data_path,encoding='utf-8').readlines()[1:]
#处理停用词
stops_word = open(data_stop_path,encoding='utf-8').readlines()
stops_word = [line.strip() for line in stops_word]#去掉换行符
stops_word.append(" ")#防止被strip过滤掉
stops_word.append("\n")
#根据分词结果统计字典
voc_dict = {}
min_seq = 1
top_n = 1000
UNK="<UNK>"
PAD = "<PAD>"
for item in data_list[:]:label = item[0]content = item[2:].strip()#去掉结尾的换行符seg_list = jieba.cut(content, cut_all=False)seg_res = []for seg_item in seg_list:print(seg_item)if seg_item in stops_word: #去掉停用词continueseg_res.append(seg_item)if seg_item in voc_dict.keys(): #使用字典统计词频voc_dict[seg_item] = voc_dict[seg_item] + 1else:voc_dict[seg_item] = 1print(content)print(seg_res)#排序字典词频，取topN的词定义字典
voc_list = sorted([_ for _ in voc_dict.items() if _[1] > min_seq],key=lambda x:x[1], reverse=True)[:top_n]
voc_dict = {word_count[0]: idx for idx,word_count in enumerate(voc_list)}
#将字典以外的词固定为特定字符UNK
voc_dict.update({UNK:len(voc_dict),PAD:len(voc_dict) + 1})print(voc_dict)#保存字典
ff = open("sources/dict","w")
for item in voc_dict.keys():ff.writelines("{},{}\n".format(item,voc_dict[item]))

此时如果分词结果里还有不想用到的词，就把这些词复制到停用词词典里

加载数据集

datasets.py

from torch.utils.data import Dataset,DataLoader
import jieba
import numpy as npdef read_dict(voc_dict_path):voc_dict = {}dict_list = open(voc_dict_path,encoding='utf-8').readlines()for item in dict_list:item = item.split(",")voc_dict[item[0]] = int(item[1].strip())return voc_dictdef load_data(data_path,data_stop_path):data_list = open(data_path,encoding='utf-8').readlines()[1:]stops_word = open(data_stop_path,encoding='utf-8').readlines()stops_word = [line.strip() for line in stops_word]stops_word.append(" ")stops_word.append("\n")voc_dict = {}data = []max_len_seq = 0 #统计最长的句子长度np.random.shuffle(data_list)for item in data_list[:]:label = item[0]content = item[2:].strip()seg_list = jieba.cut(content, cut_all=False)seg_res = []for seg_item in seg_list:if seg_item in stops_word:continueseg_res.append(seg_item)if seg_item in voc_dict.keys():voc_dict[seg_item] = voc_dict[seg_item] + 1else:voc_dict[seg_item] = 1if len(seg_res) > max_len_seq:max_len_seq = len(seg_res)data.append([label, seg_res])return data, max_len_seqclass text_ClS(Dataset):def __init__(self, voc_dict_path,data_path,data_stop_path):self.data_path = data_pathself.data_stop_path = data_stop_pathself.voc_dict = read_dict(voc_dict_path)self.data, self.max_len_seq = \load_data(self.data_path,self.data_stop_path)np.random.shuffle(self.data)def __len__(self):return len(self.data)def __getitem__(self, item):data = self.data[item]label = int(data[0])word_list = data[1]input_idx = []for word in word_list:if word in self.voc_dict.keys():input_idx.append(self.voc_dict[word])else:input_idx.append(self.voc_dict["<UNK>"])if len(input_idx) < self.max_len_seq:#长度是否是和最大的句子的长度对齐input_idx += [self.voc_dict["<PAD>"]for _ in range(self.max_len_seq - len(input_idx))]data = np.array(input_idx)return label, datadef data_loader(dataset, config):return DataLoader(dataset, batch_size=config.batch_size, shuffle=config.is_shuffle)#data_path = "sources/weibo_senti_100k.csv"#data_stop_path = "sources/hit_stopword"#dict_path = "sources/dict"if __name__ == '__main__':data_path = "sources/weibo_senti_100k.csv"data_stop_path = "sources/hit_stopword"dict_path = "sources/dict"train_dataloader = data_loader(data_path,data_stop_path,dict_path)for i, batch in enumerate(train_dataloader):print(batch[1].size())

搭建模型结构

models.py

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as npclass Model(nn.Module):def __init__(self,config):super(Model,self).__init__()self.embeding = nn.Embedding(config.n_vocab,config.embed_size,padding_idx=config.n_vocab -1)self.lstm = nn.LSTM(config.embed_size, config.hidden_size,config.num_layers, bidirectional=True,batch_first=True, dropout=config.dropout)self.maxpooling = nn.MaxPool1d(config.pad_size)self.fc = nn.Linear(config.hidden_size * 2 + config.embed_size, config.num_classes)self.softmax = nn.Softmax(dim=1)def forward(self,x):embed = self.embeding(x) #输出为[batchsize, seqlen, embed_size] 标准RNN网络的输入out, _ = self.lstm(embed)out = torch.cat((embed,out),2)out = F.relu(out)out = out.permute(0,2,1)#交换维度out = self.maxpooling(out).reshape(out.size()[0],-1)#转化为2维tensorprint(out.size())out = self.fc(out)out =  self.softmax(out)return outif __name__ == '__main__':from configs import Configcfg = Config()cfg.pad_size = 640model_textcls = Model(config=cfg)input_tensor = torch.tensor([i for i in range(640)]).reshape([1, 640])out_tensor = model_textcls.forward(input_tensor)print(out_tensor.size())print(out_tensor)

cofigs.py

import torch
class Config():def __init__(self):'''self.embeding = nn.Embedding(config.n_vocab,config.embed_size,padding_idx=config.n_vocab - 1)self.lstm = nn.LSTM(config.embed_size,config.hidden_size,config.num_layers,bidirectional=True, batch_first=True,dropout=config.dropout)self.maxpool = nn.MaxPool1d(config.pad_size)self.fc = nn.Linear(config.hidden_size * 2 + config.embed_size,config.num_classes)self.softmax = nn.Softmax(dim=1)'''self.n_vocab = 1002 #字典长度self.embed_size = 128self.hidden_size = 128self.num_layers = 3self.dropout = 0.8self.num_classes = 2 #二分类问题self.pad_size = 32self.batch_size = 128self.is_shuffle = Trueself.learn_rate = 0.001self.num_epochs = 100self.devices = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

训练脚本的搭建

run_train.py

import torch
import torch.nn as nn
from torch import optim
from models import Model
from datasets import data_loader, text_ClS
from configs import Configcfg = Config()#读取数据
data_path = "sources/weibo_senti_100k.csv"
data_stop_path = "sources/hit_stopword"
dict_path = "sources/dict"dataset = text_ClS(dict_path, data_path, data_stop_path)
train_dataloader = data_loader(dataset, cfg)cfg.pad_size = dataset.max_len_seqmodel_text_cls = Model(cfg)
model_text_cls.to(cfg.devices)loss_func = nn.CrossEntropyLoss()
optimizer = optim.Adam(model_text_cls.parameters(),lr=cfg.learn_rate)for epoch in range(cfg.num_epochs):for i, batch in enumerate(train_dataloader):label, data = batchdata = torch.tensor(data).to(cfg.devices)label = torch.tensor(label,dtype=torch.int64).to(cfg.devices)optimizer.zero_grad()pred = model_text_cls.forward(data)loss_val = loss_func(pred, label)# print(pred)# print(label)print("epoch is {}, ite is {}, val is {}".format(epoch,i,loss_val))loss_val.backward()optimizer.step()if epoch % 10 == 0:#每10次迭代存储一次模型torch.save(model_text_cls.state_dict(),"models/{}.pth".format(epoch))

测试脚本的编写

test.py

import torch
import torch.nn as nn
from torch import optim
from models import Model
from datasets import data_loader, text_ClS
from configs import Configcfg = Config()#读取数据
data_path = "sources/weibo_senti_100k.csv"
data_stop_path = "sources/hit_stopword"
dict_path = "sources/dict"dataset = text_ClS(dict_path, data_path, data_stop_path)
train_dataloader = data_loader(dataset, cfg)cfg.pad_size = dataset.max_len_seqmodel_text_cls = Model(cfg)
model_text_cls.to(cfg.devices)
#加载模型
model_text_cls.load_state_dict(torch.load("models/10.pth"))for i, batch in enumerate(train_dataloader):label, data = batchdata = torch.tensor(data).to(cfg.devices)label = torch.tensor(label,dtype=torch.int64).to(cfg.devices)pred_softmax = model_text_cls.forward(data)#print(pred_softmax)print(label)pred = torch.argmax(pred_softmax, dim=1)print(pred)#统计准确率out = torch.eq(pred,label)print(out.sum() * 1.0 / pred.size()[0])

中文文本情感分类实战（weibo_senti_100k为数据集）相关推荐

基于pytorch的Bi-LSTM中文文本情感分类
基于pytorch的Bi-LSTM中文文本情感分类目录基于pytorch的Bi-LSTM中文文本情感分类一.前言二.数据集的准备与处理 2.1 数据集介绍 2.2 文本向量化 2.3 数据集处 ...
中文文本情感分类及情感分析资源大全
摘要:20世纪初以来,文本的情感分析在自然语言处理领域成为了研究的热点,吸引了众多学者越来越多的关注.对于中文文本的情感倾向性研究在这样一大环境下也得到了显著的发展.本文主要是基于机器学习方法的中文文 ...
自然语言处理课程作业中文文本情感分类
摘要:20世纪初以来,文本的情感分析在自然语言处理领域成为了研究的热点,吸引了众多学者越来越多的关注.对于中文文本的情感倾向性研究在这样一大环境下也得到了显著的发展.本文主要是基于机器学习方法的中文文 ...
bert中文文本情感分类微博评论挖掘之Bert实战应用案例-文本情感分类
Bert模型全称Bidirectional Encoder Representations from Transformers,主要分为两个部分:1训练语言模型(language model)的预训练 ...
基于LSTM的中文文本多分类实战
在我之前的博客中我们介绍了文本的多分类的方法,我们还尝试了各种分类模型,比如朴素贝叶斯.逻辑回归.支持向量机和随机森林等并且都取得了非常不错的效果.今天我们使用深度学习中的LSTM(Long Shor ...
中文文本情感分类（基于LSTM和textCNN）
中文新闻数据集负面文本: 正面文本: 数据文本都是用爬虫从网络上爬取的,由人工进行分类,在使用这些数据之前,需要先对文本进行预处理,预处理包括去除标点符号,停用词过滤和分词等,由于篇幅有限,这里就不 ...
python英文文本情感分析_sentimentpy模块进行中文文本情感分类
sentimentpy是我根据R语言的一个文本情感分析包sentiment进行开发的, 开发的初衷有: R的sentiment已经被弃坑, 没人维护 Python比R更擅长文本处理 sentiment ...
使用python和sklearn的中文文本多分类实战开发
文本分类一般可以分为二分类.多分类.多标签分类三种情况,二分类是指将一组文本分成两个类(0或1),比较常见的应用如垃圾邮件分类.电商网站的用户评价数据的正负面分类等,多分类是指将文本分成若干个类中的某 ...
python情感词典_sentimentpy模块进行中文文本情感分类
sentimentpy是我根据R语言的一个文本情感分析包sentiment进行开发的, 开发的初衷有: R的sentiment已经被弃坑, 没人维护 Python比R更擅长文本处理 sentiment ...

中文文本情感分类实战（weibo_senti_100k为数据集）

中文文本情感分类

数据准备

加载数据集

搭建模型结构

训练脚本的搭建

测试脚本的编写

中文文本情感分类实战（weibo_senti_100k为数据集）相关推荐

最新文章

热门文章