{'科技': 0, '股票': 1, '体育': 2, '娱乐': 3, '时政': 4, '社会': 5, '教育': 6, '财经': 7, '家居': 8, '游戏': 9, '房产': 10, '时尚': 11, '彩票': 12, '星座': 13}

import pandas as pd
train_df = pd.read_csv('../data/train_set.csv', sep='t', nrows=100)#读取前100行

f = lambda x : x ** 2
f(3)Output: 9

import numpy
import matplotlib
from matplotlib import pylab, mlab, pyplot
np = numpy
plt = pyplotfrom IPython.display import display
from IPython.core.pylabtools import figsize, getfigsfrom pylab import *
from numpy import *

%pylab inline
train_df['text_len'] = train_df['text'].apply(lambda x: len(x.split(' ')))
print(train_df['text_len'].describe())Output:
Populating the interactive namespace from numpy and matplotlib
count    200000.000000
mean        907.207110
std         996.029036
min           2.000000
25%         374.000000
50%         676.000000
75%        1131.000000
max       57921.000000
Name: text_len, dtype: float64

_ = plt.hist(train_df['text_len'], bins=200) #使用了%pylab，可以直接使用'plt'
plt.xlabel('Text char count')
plt.title("Histogram of char count")
Text(0.5, 1.0, 'Histogram of char count')

train_df['label'].value_counts().plot(kind='bar')
plt.title('News class count')
plt.xlabel("category")
Text(0.5, 0, 'category')

from collections import Counter
c = Counter()
for ch in 'datawhale':c[ch] = c[ch] + 1
cOutput: Counter({'d': 1, 'a': 3, 't': 1, 'w': 1, 'h': 1, 'l': 1, 'e': 1})

c.update('mynlp')
cOutput: Counter({'d': 1, 'a': 3, 't': 1, 'w': 1, 'h': 1, 'l': 2, 'e': 1, 'm': 1, 'y': 1, 'n': 1, 'p': 1})

from collections import Counter
all_lines = ' '.join(list(train_df['text']))
word_count = Counter(all_lines.split(" "))
word_count = sorted(word_count.items(), key=lambda d:d[1], reverse = True)print(len(word_count))print(word_count[0])print(word_count[-1])Output:
6869
('3750',7482224)
('3133',1)

train_df['text_unique'] = train_df['text'].apply(lambda x: ' '.join(list(set(x.split(' ')))))
all_lines = ' '.join(list(train_df['text_unique']))
word_count = Counter(all_lines.split(" "))
word_count = sorted(word_count.items(), key=lambda d:int(d[1]), reverse = True)print(word_count[0])print(word_count[1])print(word_count[2])Output:
('3750', 197997)
('900', 197653)
('648', 191975)

句子1：我 爱 北 京 天 安 门
句子2：我 喜 欢 上 海

{'我': 1, '爱': 2, '北': 3, '京': 4, '天': 5,'安': 6, '门': 7, '喜': 8, '欢': 9, '上': 10, '海': 11
}

我：[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
爱：[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]
...
海：[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]

句子1：我 爱 北 京 天 安 门
句子2：我 喜 欢 上 海

句子1：我 爱 北 京 天 安 门
转换为 [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]句子2：我 喜 欢 上 海
转换为 [1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1]

from sklearn.feature_extraction.text import CountVectorizer
corpus = ['This is the first document.','This document is the second document.','And this is the third one.','Is this the first document?',
]
vectorizer = CountVectorizer()
vectorizer.fit_transform(corpus).toarray()

句子1：我爱 爱北 北京 京天 天安 安门
句子2：我喜 喜欢 欢上 上海

TF(t)= 该词语在当前文档出现的次数 / 当前文档中词语的总数
IDF(t)= log_e（文档总数 / （出现该词语的文档总数+1））#在实际工作中，可能先有词表，再处理文档语料，该词有可能是不存在任何文档中的

# Count Vectors + RidgeClassifierimport pandas as pdfrom sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import RidgeClassifier
from sklearn.metrics import f1_scoretrain_df = pd.read_csv('../data/train_set.csv', sep='t', nrows=15000)vectorizer = CountVectorizer(max_features=3000)
train_test = vectorizer.fit_transform(train_df['text'])clf = RidgeClassifier()
clf.fit(train_test[:10000], train_df['label'].values[:10000])val_pred = clf.predict(train_test[10000:])
print(f1_score(train_df['label'].values[10000:], val_pred, average='macro'))
# 0.74

# TF-IDF +  RidgeClassifierimport pandas as pdfrom sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import RidgeClassifier
from sklearn.metrics import f1_scoretrain_df = pd.read_csv('../data/train_set.csv', sep='t', nrows=15000)tfidf = TfidfVectorizer(ngram_range=(1,3), max_features=3000)
train_test = tfidf.fit_transform(train_df['text'])clf = RidgeClassifier()
clf.fit(train_test[:10000], train_df['label'].values[:10000])val_pred = clf.predict(train_test[10000:])
print(f1_score(train_df['label'].values[10000:], val_pred, average='macro'))
# 0.87

pip install fasttext

git clone https://github.com/facebookresearch/fastText.git
cd fastText
sudo pip install .

import pandas as pd
from sklearn.metrics import f1_score# 转换为FastText需要的格式
train_df = pd.read_csv('../data/train_set.csv', sep='t', nrows=15000)
train_df['label_ft'] = '__label__' + train_df['label'].astype(str)
train_df[['text','label_ft']].iloc[:-5000].to_csv('train.csv', index=None, header=None, sep='t')import fasttext
model = fasttext.train_supervised('train.csv', lr=1.0, wordNgrams=2, verbose=2, minCount=1, epoch=25, loss="hs")val_pred = [model.predict(x)[0][0].split('__')[-1] for x in train_df.iloc[-5000:]['text']]
print(f1_score(train_df['label'].values[-5000:].astype(str), val_pred, average='macro'))
# 0.82

label2id = {}
for i in range(total):label = str(all_labels[i])if label not in label2id:label2id[label] = [i]else:label2id[label].append(i)

import logging
import randomimport numpy as np
import torchlogging.basicConfig(level=logging.INFO, format='%(asctime)-15s %(levelname)s: %(message)s')# set seed
seed = 666
random.seed(seed)
np.random.seed(seed)
torch.cuda.manual_seed(seed)
torch.manual_seed(seed)# split data to 10 fold
fold_num = 10
data_file = '../data/train_set.csv'
import pandas as pddef all_data2fold(fold_num, num=10000):fold_data = []f = pd.read_csv(data_file, sep='t', encoding='UTF-8')texts = f['text'].tolist()[:num]labels = f['label'].tolist()[:num]total = len(labels)index = list(range(total))np.random.shuffle(index)all_texts = []all_labels = []for i in index:all_texts.append(texts[i])all_labels.append(labels[i])label2id = {}for i in range(total):label = str(all_labels[i])if label not in label2id:label2id[label] = [i]else:label2id[label].append(i)all_index = [[] for _ in range(fold_num)]for label, data in label2id.items():# print(label, len(data))batch_size = int(len(data) / fold_num)other = len(data) - batch_size * fold_numfor i in range(fold_num):cur_batch_size = batch_size + 1 if i < other else batch_size# print(cur_batch_size)batch_data = [data[i * batch_size + b] for b in range(cur_batch_size)]all_index[i].extend(batch_data)batch_size = int(total / fold_num)other_texts = []other_labels = []other_num = 0start = 0for fold in range(fold_num):num = len(all_index[fold])texts = [all_texts[i] for i in all_index[fold]]labels = [all_labels[i] for i in all_index[fold]]if num > batch_size:fold_texts = texts[:batch_size]other_texts.extend(texts[batch_size:])fold_labels = labels[:batch_size]other_labels.extend(labels[batch_size:])other_num += num - batch_sizeelif num < batch_size:end = start + batch_size - numfold_texts = texts + other_texts[start: end]fold_labels = labels + other_labels[start: end]start = endelse:fold_texts = textsfold_labels = labelsassert batch_size == len(fold_labels)# shuffleindex = list(range(batch_size))np.random.shuffle(index)shuffle_fold_texts = []shuffle_fold_labels = []for i in index:shuffle_fold_texts.append(fold_texts[i])shuffle_fold_labels.append(fold_labels[i])data = {'label': shuffle_fold_labels, 'text': shuffle_fold_texts}fold_data.append(data)logging.info("Fold lens %s", str([len(data['label']) for data in fold_data]))return fold_datafold_data = all_data2fold(10)# build train data for word2vec
fold_id = 9train_texts = []
for i in range(0, fold_id):data = fold_data[i]train_texts.extend(data['text'])logging.info('Total %d docs.' % len(train_texts))logging.info('Start training...')
from gensim.models.word2vec import Word2Vecnum_features = 100     # Word vector dimensionality
num_workers = 8       # Number of threads to run in paralleltrain_texts = list(map(lambda x: list(x.split()), train_texts))
model = Word2Vec(train_texts, workers=num_workers, size=num_features)
model.init_sims(replace=True)# save model
model.save("./word2vec.bin")# load model
model = Word2Vec.load("./word2vec.bin")# convert format
model.wv.save_word2vec_format('./word2vec.txt', binary=False)

import logging
import randomimport numpy as np
import torchlogging.basicConfig(level=logging.INFO, format='%(asctime)-15s %(levelname)s: %(message)s')# set seed
seed = 666
random.seed(seed)
np.random.seed(seed)
torch.cuda.manual_seed(seed)
torch.manual_seed(seed)# set cuda
gpu = 0
use_cuda = gpu >= 0 and torch.cuda.is_available()
if use_cuda:torch.cuda.set_device(gpu)device = torch.device("cuda", gpu)
else:device = torch.device("cpu")
logging.info("Use cuda: %s, gpu id: %d.", use_cuda, gpu)# split data to 10 fold
fold_num = 10
data_file = '../data/train_set.csv'
import pandas as pddef all_data2fold(fold_num, num=10000):fold_data = []f = pd.read_csv(data_file, sep='t', encoding='UTF-8')texts = f['text'].tolist()[:num]labels = f['label'].tolist()[:num]total = len(labels)index = list(range(total))np.random.shuffle(index)all_texts = []all_labels = []for i in index:all_texts.append(texts[i])all_labels.append(labels[i])label2id = {}for i in range(total):label = str(all_labels[i])if label not in label2id:label2id[label] = [i]else:label2id[label].append(i)all_index = [[] for _ in range(fold_num)]for label, data in label2id.items():# print(label, len(data))batch_size = int(len(data) / fold_num)other = len(data) - batch_size * fold_numfor i in range(fold_num):cur_batch_size = batch_size + 1 if i < other else batch_size# print(cur_batch_size)batch_data = [data[i * batch_size + b] for b in range(cur_batch_size)]all_index[i].extend(batch_data)batch_size = int(total / fold_num)other_texts = []other_labels = []other_num = 0start = 0for fold in range(fold_num):num = len(all_index[fold])texts = [all_texts[i] for i in all_index[fold]]labels = [all_labels[i] for i in all_index[fold]]if num > batch_size:fold_texts = texts[:batch_size]other_texts.extend(texts[batch_size:])fold_labels = labels[:batch_size]other_labels.extend(labels[batch_size:])other_num += num - batch_sizeelif num < batch_size:end = start + batch_size - numfold_texts = texts + other_texts[start: end]fold_labels = labels + other_labels[start: end]start = endelse:fold_texts = textsfold_labels = labelsassert batch_size == len(fold_labels)# shuffleindex = list(range(batch_size))np.random.shuffle(index)shuffle_fold_texts = []shuffle_fold_labels = []for i in index:shuffle_fold_texts.append(fold_texts[i])shuffle_fold_labels.append(fold_labels[i])data = {'label': shuffle_fold_labels, 'text': shuffle_fold_texts}fold_data.append(data)logging.info("Fold lens %s", str([len(data['label']) for data in fold_data]))return fold_datafold_data = all_data2fold(10)# build train, dev, test data
fold_id = 9# dev
dev_data = fold_data[fold_id]# train
train_texts = []
train_labels = []
for i in range(0, fold_id):data = fold_data[i]train_texts.extend(data['text'])train_labels.extend(data['label'])train_data = {'label': train_labels, 'text': train_texts}# test
test_data_file = '../data/test_a.csv'
f = pd.read_csv(test_data_file, sep='t', encoding='UTF-8')
texts = f['text'].tolist()
test_data = {'label': [0] * len(texts), 'text': texts}# build vocab
from collections import Counter
from transformers import BasicTokenizerbasic_tokenizer = BasicTokenizer()class Vocab():def __init__(self, train_data):self.min_count = 5self.pad = 0self.unk = 1self._id2word = ['[PAD]', '[UNK]']self._id2extword = ['[PAD]', '[UNK]']self._id2label = []self.target_names = []self.build_vocab(train_data)reverse = lambda x: dict(zip(x, range(len(x))))self._word2id = reverse(self._id2word)self._label2id = reverse(self._id2label)logging.info("Build vocab: words %d, labels %d." % (self.word_size, self.label_size))def build_vocab(self, data):self.word_counter = Counter()for text in data['text']:words = text.split()for word in words:self.word_counter[word] += 1for word, count in self.word_counter.most_common():if count >= self.min_count:self._id2word.append(word)label2name = {0: '科技', 1: '股票', 2: '体育', 3: '娱乐', 4: '时政', 5: '社会', 6: '教育', 7: '财经',8: '家居', 9: '游戏', 10: '房产', 11: '时尚', 12: '彩票', 13: '星座'}self.label_counter = Counter(data['label'])for label in range(len(self.label_counter)):count = self.label_counter[label]self._id2label.append(label)self.target_names.append(label2name[label])def load_pretrained_embs(self, embfile):with open(embfile, encoding='utf-8') as f:lines = f.readlines()items = lines[0].split()word_count, embedding_dim = int(items[0]), int(items[1])index = len(self._id2extword)embeddings = np.zeros((word_count + index, embedding_dim))for line in lines[1:]:values = line.split()self._id2extword.append(values[0])vector = np.array(values[1:], dtype='float64')embeddings[self.unk] += vectorembeddings[index] = vectorindex += 1embeddings[self.unk] = embeddings[self.unk] / word_countembeddings = embeddings / np.std(embeddings)reverse = lambda x: dict(zip(x, range(len(x))))self._extword2id = reverse(self._id2extword)assert len(set(self._id2extword)) == len(self._id2extword)return embeddingsdef word2id(self, xs):if isinstance(xs, list):return [self._word2id.get(x, self.unk) for x in xs]return self._word2id.get(xs, self.unk)def extword2id(self, xs):if isinstance(xs, list):return [self._extword2id.get(x, self.unk) for x in xs]return self._extword2id.get(xs, self.unk)def label2id(self, xs):if isinstance(xs, list):return [self._label2id.get(x, self.unk) for x in xs]return self._label2id.get(xs, self.unk)@propertydef word_size(self):return len(self._id2word)@propertydef extword_size(self):return len(self._id2extword)@propertydef label_size(self):return len(self._id2label)vocab = Vocab(train_data)# build module
import torch.nn as nn
import torch.nn.functional as Fclass Attention(nn.Module):def __init__(self, hidden_size):super(Attention, self).__init__()self.weight = nn.Parameter(torch.Tensor(hidden_size, hidden_size))self.weight.data.normal_(mean=0.0, std=0.05)self.bias = nn.Parameter(torch.Tensor(hidden_size))b = np.zeros(hidden_size, dtype=np.float32)self.bias.data.copy_(torch.from_numpy(b))self.query = nn.Parameter(torch.Tensor(hidden_size))self.query.data.normal_(mean=0.0, std=0.05)def forward(self, batch_hidden, batch_masks):# batch_hidden: b x len x hidden_size (2 * hidden_size of lstm)# batch_masks:  b x len# linearkey = torch.matmul(batch_hidden, self.weight) + self.bias  # b x len x hidden# compute attentionoutputs = torch.matmul(key, self.query)  # b x lenmasked_outputs = outputs.masked_fill((1 - batch_masks).bool(), float(-1e32))attn_scores = F.softmax(masked_outputs, dim=1)  # b x len# 对于全零向量，-1e32的结果为 1/len, -inf为nan, 额外补0masked_attn_scores = attn_scores.masked_fill((1 - batch_masks).bool(), 0.0)# sum weighted sourcesbatch_outputs = torch.bmm(masked_attn_scores.unsqueeze(1), key).squeeze(1)  # b x hiddenreturn batch_outputs, attn_scores# build word encoder
bert_path = '../emb/bert-mini/'
dropout = 0.15from transformers import BertModelclass WordBertEncoder(nn.Module):def __init__(self):super(WordBertEncoder, self).__init__()self.dropout = nn.Dropout(dropout)self.tokenizer = WhitespaceTokenizer()self.bert = BertModel.from_pretrained(bert_path)self.pooled = Falselogging.info('Build Bert encoder with pooled {}.'.format(self.pooled))def encode(self, tokens):tokens = self.tokenizer.tokenize(tokens)return tokensdef get_bert_parameters(self):no_decay = ['bias', 'LayerNorm.weight']optimizer_parameters = [{'params': [p for n, p in self.bert.named_parameters() if not any(nd in n for nd in no_decay)],'weight_decay': 0.01},{'params': [p for n, p in self.bert.named_parameters() if any(nd in n for nd in no_decay)],'weight_decay': 0.0}]return optimizer_parametersdef forward(self, input_ids, token_type_ids):# input_ids: sen_num x bert_len# token_type_ids: sen_num  x bert_len# sen_num x bert_len x 256, sen_num x 256sequence_output, pooled_output = self.bert(input_ids=input_ids, token_type_ids=token_type_ids)if self.pooled:reps = pooled_outputelse:reps = sequence_output[:, 0, :]  # sen_num x 256if self.training:reps = self.dropout(reps)return repsclass WhitespaceTokenizer():"""WhitespaceTokenizer with vocab."""def __init__(self):vocab_file = bert_path + 'vocab.txt'self._token2id = self.load_vocab(vocab_file)self._id2token = {v: k for k, v in self._token2id.items()}self.max_len = 256self.unk = 1logging.info("Build Bert vocab with size %d." % (self.vocab_size))def load_vocab(self, vocab_file):f = open(vocab_file, 'r')lines = f.readlines()lines = list(map(lambda x: x.strip(), lines))vocab = dict(zip(lines, range(len(lines))))return vocabdef tokenize(self, tokens):assert len(tokens) <= self.max_len - 2tokens = ["[CLS]"] + tokens + ["[SEP]"]output_tokens = self.token2id(tokens)return output_tokensdef token2id(self, xs):if isinstance(xs, list):return [self._token2id.get(x, self.unk) for x in xs]return self._token2id.get(xs, self.unk)@propertydef vocab_size(self):return len(self._id2token)# build sent encoder
sent_hidden_size = 256
sent_num_layers = 2class SentEncoder(nn.Module):def __init__(self, sent_rep_size):super(SentEncoder, self).__init__()self.dropout = nn.Dropout(dropout)self.sent_lstm = nn.LSTM(input_size=sent_rep_size,hidden_size=sent_hidden_size,num_layers=sent_num_layers,batch_first=True,bidirectional=True)def forward(self, sent_reps, sent_masks):# sent_reps:  b x doc_len x sent_rep_size# sent_masks: b x doc_lensent_hiddens, _ = self.sent_lstm(sent_reps)  # b x doc_len x hidden*2sent_hiddens = sent_hiddens * sent_masks.unsqueeze(2)if self.training:sent_hiddens = self.dropout(sent_hiddens)return sent_hiddens# build model
class Model(nn.Module):def __init__(self, vocab):super(Model, self).__init__()self.sent_rep_size = 256self.doc_rep_size = sent_hidden_size * 2self.all_parameters = {}parameters = []self.word_encoder = WordBertEncoder()bert_parameters = self.word_encoder.get_bert_parameters()self.sent_encoder = SentEncoder(self.sent_rep_size)self.sent_attention = Attention(self.doc_rep_size)parameters.extend(list(filter(lambda p: p.requires_grad, self.sent_encoder.parameters())))parameters.extend(list(filter(lambda p: p.requires_grad, self.sent_attention.parameters())))self.out = nn.Linear(self.doc_rep_size, vocab.label_size, bias=True)parameters.extend(list(filter(lambda p: p.requires_grad, self.out.parameters())))if use_cuda:self.to(device)if len(parameters) > 0:self.all_parameters["basic_parameters"] = parametersself.all_parameters["bert_parameters"] = bert_parameterslogging.info('Build model with bert word encoder, lstm sent encoder.')para_num = sum([np.prod(list(p.size())) for p in self.parameters()])logging.info('Model param num: %.2f M.' % (para_num / 1e6))def forward(self, batch_inputs):# batch_inputs(batch_inputs1, batch_inputs2): b x doc_len x sent_len# batch_masks : b x doc_len x sent_lenbatch_inputs1, batch_inputs2, batch_masks = batch_inputsbatch_size, max_doc_len, max_sent_len = batch_inputs1.shape[0], batch_inputs1.shape[1], batch_inputs1.shape[2]batch_inputs1 = batch_inputs1.view(batch_size * max_doc_len, max_sent_len)  # sen_num x sent_lenbatch_inputs2 = batch_inputs2.view(batch_size * max_doc_len, max_sent_len)  # sen_num x sent_lenbatch_masks = batch_masks.view(batch_size * max_doc_len, max_sent_len)  # sen_num x sent_lensent_reps = self.word_encoder(batch_inputs1, batch_inputs2)  # sen_num x sent_rep_sizesent_reps = sent_reps.view(batch_size, max_doc_len, self.sent_rep_size)  # b x doc_len x sent_rep_sizebatch_masks = batch_masks.view(batch_size, max_doc_len, max_sent_len)  # b x doc_len x max_sent_lensent_masks = batch_masks.bool().any(2).float()  # b x doc_lensent_hiddens = self.sent_encoder(sent_reps, sent_masks)  # b x doc_len x doc_rep_sizedoc_reps, atten_scores = self.sent_attention(sent_hiddens, sent_masks)  # b x doc_rep_sizebatch_outputs = self.out(doc_reps)  # b x num_labelsreturn batch_outputsmodel = Model(vocab)# build optimizer
learning_rate = 2e-4
bert_lr = 5e-5
decay = .75
decay_step = 1000
from transformers import AdamW, get_linear_schedule_with_warmupclass Optimizer:def __init__(self, model_parameters, steps):self.all_params = []self.optims = []self.schedulers = []for name, parameters in model_parameters.items():if name.startswith("basic"):optim = torch.optim.Adam(parameters, lr=learning_rate)self.optims.append(optim)l = lambda step: decay ** (step // decay_step)scheduler = torch.optim.lr_scheduler.LambdaLR(optim, lr_lambda=l)self.schedulers.append(scheduler)self.all_params.extend(parameters)elif name.startswith("bert"):optim_bert = AdamW(parameters, bert_lr, eps=1e-8)self.optims.append(optim_bert)scheduler_bert = get_linear_schedule_with_warmup(optim_bert, 0, steps)self.schedulers.append(scheduler_bert)for group in parameters:for p in group['params']:self.all_params.append(p)else:Exception("no nameed parameters.")self.num = len(self.optims)def step(self):for optim, scheduler in zip(self.optims, self.schedulers):optim.step()scheduler.step()optim.zero_grad()def zero_grad(self):for optim in self.optims:optim.zero_grad()def get_lr(self):lrs = tuple(map(lambda x: x.get_lr()[-1], self.schedulers))lr = ' %.5f' * self.numres = lr % lrsreturn res# build dataset
def sentence_split(text, vocab, max_sent_len=256, max_segment=16):words = text.strip().split()document_len = len(words)index = list(range(0, document_len, max_sent_len))index.append(document_len)segments = []for i in range(len(index) - 1):segment = words[index[i]: index[i + 1]]assert len(segment) > 0segment = [word if word in vocab._id2word else '<UNK>' for word in segment]segments.append([len(segment), segment])assert len(segments) > 0if len(segments) > max_segment:segment_ = int(max_segment / 2)return segments[:segment_] + segments[-segment_:]else:return segmentsdef get_examples(data, word_encoder, vocab, max_sent_len=256, max_segment=8):label2id = vocab.label2idexamples = []for text, label in zip(data['text'], data['label']):# labelid = label2id(label)# wordssents_words = sentence_split(text, vocab, max_sent_len-2, max_segment)doc = []for sent_len, sent_words in sents_words:token_ids = word_encoder.encode(sent_words)sent_len = len(token_ids)token_type_ids = [0] * sent_lendoc.append([sent_len, token_ids, token_type_ids])examples.append([id, len(doc), doc])logging.info('Total %d docs.' % len(examples))return examples# build loaderdef batch_slice(data, batch_size):batch_num = int(np.ceil(len(data) / float(batch_size)))for i in range(batch_num):cur_batch_size = batch_size if i < batch_num - 1 else len(data) - batch_size * idocs = [data[i * batch_size + b] for b in range(cur_batch_size)]yield docsdef data_iter(data, batch_size, shuffle=True, noise=1.0):"""randomly permute data, then sort by source length, and partition into batchesensure that the length of  sentences in each batch"""batched_data = []if shuffle:np.random.shuffle(data)lengths = [example[1] for example in data]noisy_lengths = [- (l + np.random.uniform(- noise, noise)) for l in lengths]sorted_indices = np.argsort(noisy_lengths).tolist()sorted_data = [data[i] for i in sorted_indices]else:sorted_data =databatched_data.extend(list(batch_slice(sorted_data, batch_size)))if shuffle:np.random.shuffle(batched_data)for batch in batched_data:yield batch# some function
from sklearn.metrics import f1_score, precision_score, recall_scoredef get_score(y_ture, y_pred):y_ture = np.array(y_ture)y_pred = np.array(y_pred)f1 = f1_score(y_ture, y_pred, average='macro') * 100p = precision_score(y_ture, y_pred, average='macro') * 100r = recall_score(y_ture, y_pred, average='macro') * 100return str((reformat(p, 2), reformat(r, 2), reformat(f1, 2))), reformat(f1, 2)def reformat(num, n):return float(format(num, '0.' + str(n) + 'f'))# build trainerimport time
from sklearn.metrics import classification_reportclip = 5.0
epochs = 1
early_stops = 3
log_interval = 50test_batch_size = 16
train_batch_size = 16save_model = './bert.bin'
save_test = './bert.csv'class Trainer():def __init__(self, model, vocab):self.model = modelself.report = Trueself.train_data = get_examples(train_data, model.word_encoder, vocab)self.batch_num = int(np.ceil(len(self.train_data) / float(train_batch_size)))self.dev_data = get_examples(dev_data, model.word_encoder, vocab)self.test_data = get_examples(test_data, model.word_encoder, vocab)# criterionself.criterion = nn.CrossEntropyLoss()# label nameself.target_names = vocab.target_names# optimizerself.optimizer = Optimizer(model.all_parameters, steps=self.batch_num * epochs)# countself.step = 0self.early_stop = -1self.best_train_f1, self.best_dev_f1 = 0, 0self.last_epoch = epochsdef train(self):logging.info('Start training...')for epoch in range(1, epochs + 1):train_f1 = self._train(epoch)dev_f1 = self._eval(epoch)if self.best_dev_f1 <= dev_f1:logging.info("Exceed history dev = %.2f, current dev = %.2f" % (self.best_dev_f1, dev_f1))torch.save(self.model.state_dict(), save_model)self.best_train_f1 = train_f1self.best_dev_f1 = dev_f1self.early_stop = 0else:self.early_stop += 1if self.early_stop == early_stops:logging.info("Eearly stop in epoch %d, best train: %.2f, dev: %.2f" % (epoch - early_stops, self.best_train_f1, self.best_dev_f1))self.last_epoch = epochbreakdef test(self):self.model.load_state_dict(torch.load(save_model))self._eval(self.last_epoch + 1, test=True)def _train(self, epoch):self.optimizer.zero_grad()self.model.train()start_time = time.time()epoch_start_time = time.time()overall_losses = 0losses = 0batch_idx = 1y_pred = []y_true = []for batch_data in data_iter(self.train_data, train_batch_size, shuffle=True):torch.cuda.empty_cache()batch_inputs, batch_labels = self.batch2tensor(batch_data)batch_outputs = self.model(batch_inputs)loss = self.criterion(batch_outputs, batch_labels)loss.backward()loss_value = loss.detach().cpu().item()losses += loss_valueoverall_losses += loss_valuey_pred.extend(torch.max(batch_outputs, dim=1)[1].cpu().numpy().tolist())y_true.extend(batch_labels.cpu().numpy().tolist())nn.utils.clip_grad_norm_(self.optimizer.all_params, max_norm=clip)for optimizer, scheduler in zip(self.optimizer.optims, self.optimizer.schedulers):optimizer.step()scheduler.step()self.optimizer.zero_grad()self.step += 1if batch_idx % log_interval == 0:elapsed = time.time() - start_timelrs = self.optimizer.get_lr()logging.info('| epoch {:3d} | step {:3d} | batch {:3d}/{:3d} | lr{} | loss {:.4f} | s/batch {:.2f}'.format(epoch, self.step, batch_idx, self.batch_num, lrs,losses / log_interval,elapsed / log_interval))losses = 0start_time = time.time()batch_idx += 1overall_losses /= self.batch_numduring_time = time.time() - epoch_start_time# reformatoverall_losses = reformat(overall_losses, 4)score, f1 = get_score(y_true, y_pred)logging.info('| epoch {:3d} | score {} | f1 {} | loss {:.4f} | time {:.2f}'.format(epoch, score, f1,overall_losses,during_time))if set(y_true) == set(y_pred) and self.report:report = classification_report(y_true, y_pred, digits=4, target_names=self.target_names)logging.info('n' + report)return f1def _eval(self, epoch, test=False):self.model.eval()start_time = time.time()data = self.test_data if test else self.dev_datay_pred = []y_true = []with torch.no_grad():for batch_data in data_iter(data, test_batch_size, shuffle=False):torch.cuda.empty_cache()batch_inputs, batch_labels = self.batch2tensor(batch_data)batch_outputs = self.model(batch_inputs)y_pred.extend(torch.max(batch_outputs, dim=1)[1].cpu().numpy().tolist())y_true.extend(batch_labels.cpu().numpy().tolist())score, f1 = get_score(y_true, y_pred)during_time = time.time() - start_timeif test:df = pd.DataFrame({'label': y_pred})df.to_csv(save_test, index=False, sep=',')else:logging.info('| epoch {:3d} | dev | score {} | f1 {} | time {:.2f}'.format(epoch, score, f1,during_time))if set(y_true) == set(y_pred) and self.report:report = classification_report(y_true, y_pred, digits=4, target_names=self.target_names)logging.info('n' + report)return f1def batch2tensor(self, batch_data):'''[[label, doc_len, [[sent_len, [sent_id0, ...], [sent_id1, ...]], ...]]'''batch_size = len(batch_data)doc_labels = []doc_lens = []doc_max_sent_len = []for doc_data in batch_data:doc_labels.append(doc_data[0])doc_lens.append(doc_data[1])sent_lens = [sent_data[0] for sent_data in doc_data[2]]max_sent_len = max(sent_lens)doc_max_sent_len.append(max_sent_len)max_doc_len = max(doc_lens)max_sent_len = max(doc_max_sent_len)batch_inputs1 = torch.zeros((batch_size, max_doc_len, max_sent_len), dtype=torch.int64)batch_inputs2 = torch.zeros((batch_size, max_doc_len, max_sent_len), dtype=torch.int64)batch_masks = torch.zeros((batch_size, max_doc_len, max_sent_len), dtype=torch.float32)batch_labels = torch.LongTensor(doc_labels)for b in range(batch_size):for sent_idx in range(doc_lens[b]):sent_data = batch_data[b][2][sent_idx]for word_idx in range(sent_data[0]):batch_inputs1[b, sent_idx, word_idx] = sent_data[1][word_idx]batch_inputs2[b, sent_idx, word_idx] = sent_data[2][word_idx]batch_masks[b, sent_idx, word_idx] = 1if use_cuda:batch_inputs1 = batch_inputs1.to(device)batch_inputs2 = batch_inputs2.to(device)batch_masks = batch_masks.to(device)batch_labels = batch_labels.to(device)return (batch_inputs1, batch_inputs2, batch_masks), batch_labels# train
trainer = Trainer(model, vocab)
trainer.train()# test
trainer.test()