1.创建模型

import math
import torch
import torch.nn as nn
import torch.nn.functional as Fclass TransformerModel(nn.Module):def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):super(TransformerModel, self).__init__()from torch.nn import TransformerEncoder, TransformerEncoderLayerself.model_type = 'Transformer'self.src_mask = Noneself.pos_encoder = PositionalEncoding(ninp, dropout) # 将位置编码和词向量矩阵相加encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)self.encoder = nn.Embedding(ntoken, ninp)self.ninp = ninpself.decoder = nn.Linear(ninp, ntoken)self.init_weights()def _generate_square_subsequent_mask(self, sz):mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))return maskdef init_weights(self):initrange = 0.1self.encoder.weight.data.uniform_(-initrange, initrange)self.decoder.bias.data.zero_()self.decoder.weight.data.uniform_(-initrange, initrange)def forward(self, src):if self.src_mask is None or self.src_mask.size(0) != len(src):device = src.devicemask = self._generate_square_subsequent_mask(len(src)).to(device)self.src_mask = masksrc = self.encoder(src) * math.sqrt(self.ninp)src = self.pos_encoder(src)output = self.transformer_encoder(src, self.src_mask)output = self.decoder(output)return outputclass PositionalEncoding(nn.Module):def __init__(self, d_model, dropout=0.1, max_len=5000):super(PositionalEncoding, self).__init__()self.dropout = nn.Dropout(p=dropout)pe = torch.zeros(max_len, d_model) # 构建句子矩阵position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) # 给单词添加位置div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))pe[:, 0::2] = torch.sin(position * div_term)pe[:, 1::2] = torch.cos(position * div_term)pe = pe.unsqueeze(0).transpose(0, 1)self.register_buffer('pe', pe)def forward(self, x):x = x + self.pe[:x.size(0), :]return self.dropout(x)

加载数据

import torchtext
from torchtext.data.utils import get_tokenizer
TEXT = torchtext.data.Field(tokenize=get_tokenizer("basic_english"),init_token='<sos>',eos_token='<eos>',lower=True)
train_txt, val_txt, test_txt = torchtext.datasets.WikiText2.splits(TEXT)
TEXT.build_vocab(train_txt)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")def batchify(data, bsz):data = TEXT.numericalize([data.examples[0].text])# Divide the dataset into bsz parts.nbatch = data.size(0) // bsz# Trim off any extra elements that wouldn't cleanly fit (remainders).data = data.narrow(0, 0, nbatch * bsz)# Evenly divide the data across the bsz batches.data = data.view(bsz, -1).t().contiguous()return data.to(device)batch_size = 20
eval_batch_size = 10
train_data = batchify(train_txt, batch_size)
val_data = batchify(val_txt, eval_batch_size)
test_data = batchify(test_txt, eval_batch_size)bptt = 35
def get_batch(source, i):seq_len = min(bptt, len(source) - 1 - i)data = source[i:i+seq_len]target = source[i+1:i+1+seq_len].view(-1)return data, target

3.训练模型

ntokens = len(TEXT.vocab.stoi) # the size of vocabulary
emsize = 200 # embedding dimension
nhid = 200 # the dimension of the feedforward network model in nn.TransformerEncoder
nlayers = 2 # the number of nn.TransformerEncoderLayer in nn.TransformerEncoder
nhead = 2 # the number of heads in the multiheadattention models
dropout = 0.2 # the dropout value
model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(device)criterion = nn.CrossEntropyLoss()
lr = 5.0 # learning rate
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)import time
def train():model.train() # Turn on the train modetotal_loss = 0.start_time = time.time()ntokens = len(TEXT.vocab.stoi)for batch, i in enumerate(range(0, train_data.size(0) - 1, bptt)):data, targets = get_batch(train_data, i)optimizer.zero_grad()output = model(data)loss = criterion(output.view(-1, ntokens), targets)loss.backward()torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)optimizer.step()total_loss += loss.item()log_interval = 200if batch % log_interval == 0 and batch > 0:cur_loss = total_loss / log_intervalelapsed = time.time() - start_timeprint('| epoch {:3d} | {:5d}/{:5d} batches | ''lr {:02.2f} | ms/batch {:5.2f} | ''loss {:5.2f} | ppl {:8.2f}'.format(epoch, batch, len(train_data) // bptt, scheduler.get_lr()[0],elapsed * 1000 / log_interval,cur_loss, math.exp(cur_loss)))total_loss = 0start_time = time.time()def evaluate(eval_model, data_source):eval_model.eval() # Turn on the evaluation modetotal_loss = 0.ntokens = len(TEXT.vocab.stoi)with torch.no_grad():for i in range(0, data_source.size(0) - 1, bptt):data, targets = get_batch(data_source, i)output = eval_model(data)output_flat = output.view(-1, ntokens)total_loss += len(data) * criterion(output_flat, targets).item()return total_loss / (len(data_source) - 1)

参考：
https://pytorch.org/tutorials/beginner/transformer_tutorial.html
http://nlp.seas.harvard.edu/2018/04/03/attention.html
https://my.oschina.net/u/3081982/blog/3192285/print

pytorch transformers相关推荐

基于pytorch+transformers的车牌识别
目录程序流程设计熟悉训练数据集 CCPD2019数据集 CCPD数据集标注信息单例再现加载本地车牌数据集程序流程设计 1,熟悉训练数据集: 2,加载本地车牌数据集: 3,定义网络模型: 4, ...
Pytorch transformers tokenizer 分词器词汇表添加新的词语和embedding
目标: 在NLP领域,基于公开语料的预训练模型,在专业领域迁移时,会遇到专业领域词汇不在词汇表的问题,本文介绍如何添加专有名词到预训练模型. 例如,在bert预训练模型中,并不包含财经词汇,比如'市盈 ...
pyTorch api
应用 pytorch FC_regression pytorch FC_classification pytorch RNN_regression pytorch LSTM_regression py ...
Transformers中的动态学习率
Transformers中的动态学习率 1. 基线模型 2. 动态学习率 2.1 constant 2.2 constant_with_warmup 2.3 linear_with_warmup 2. ...
BERT模型超酷炫，上手又太难？请查收这份BERT快速入门指南！
点击上方"AI遇见机器学习",选择"星标"公众号重磅干货,第一时间送达来自 | GitHub 作者 | Jay Alammar 转自 | 机器之心如 ...
BERT小学生级上手教程，从原理到上手全有图示，还能直接在线运行
作者 Jay Alammar 伊瓢编译量子位出品 | 公众号 QbitAI BERT,作为自然语言处理领域的C位选手,总是NLPer们逃不过的一环. 但是,如果是经验匮乏.基础薄弱的选手,想玩转 ...
Coding and Paper Letter（七十）
资源整理. 1 Coding: 1.JupyterHub的流量模拟器. hubtraf 2.前端面试手册. front end interview handbook 3.Python学习课程. lea ...
初次使用BERT的可视化指南
初次使用BERT的可视化指南在过去几年里,处理语言的机器学习模型的进展一直在迅速加快.这一进步已经离开了研究实验室,开始为一些领先的数字产品提供动力.这方面的一个很好的例子是最近公布的BERT模型如 ...
ChatGPT 和 Elasticsearch：OpenAI 遇见私有数据（二）
在之前的文章 "ChatGPT 和 Elasticsearch:OpenAI 遇见私有数据(一)" 中,我们详细描述了如何结合 ChatGPT 及 Elasticsearch 来进 ...

pytorch transformers

1.创建模型

加载数据

3.训练模型

pytorch transformers相关推荐

最新文章

热门文章