1     def _forward_alg(self, feats):
 2         # do the forward algorithm to compute the partition function
 3         init_alphas = torch.Tensor(1, self.labelSize).fill_(0)
 4         # Wrap in a variable so that we will get automatic backprop
 5         forward_var = autograd.Variable(init_alphas)
 6
 7         # Iterate through the sentence
 8         for idx in range(len(feats)):
 9             feat = feats[idx]
10             alphas_t = []           # The forward variables at this timestep
11             for next_tag in range(self.labelSize):
12                 # broadcast the emission score: it is the same regardless of the previous tag
13                 if idx == 0:
14                     alphas_t.append(feat[next_tag].view(1, -1))
15                 else:
16                     emit_score = feat[next_tag].view(1, -1).expand(1, self.labelSize)
17                     # the ith entry of trans_score is the score of transitioning to next_tag from i
18                     trans_score = self.T[next_tag]
19                     # The ith entry of next_tag_var is the value for the edge (i -> next_tag) before we do log-sum-exp
20                     next_tag_var = forward_var + trans_score + emit_score
21                     # The forward variable for this tag is log-sum-exp of all the scores.
22                     alphas_t.append(self.log_sum_exp(next_tag_var))
23             forward_var = torch.cat(alphas_t).view(1, -1)
24         alpha_score = self.log_sum_exp(forward_var)
25         return alpha_score

1 　　# Compute log sum exp in a numerically stable way for the forward algorithm
2     def log_sum_exp(self, vec):
3         max_score = vec[0, self.argmax(vec)]
4         max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])
5         return max_score + torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))

1     def neg_log_likelihood(self, feats, tags):
2         forward_score = self._forward_alg(feats)                # calculate denominator
3         gold_score = self._score_sentence(feats, tags)
4         return forward_score - gold_score                       # calculate loss

 1         for iter in range(self.hyperParams.maxIter):
 2             print('###Iteration' + str(iter) + "###")
 3             random.shuffle(indexes)
 4             for idx in range(len(trainExamples)):
 5                 # Step 1. Remember that Pytorch accumulates gradients. We need to clear them out before each instance
 6                 self.model.zero_grad()
 7                 # Step 2. Get our inputs ready for the network, that is, turn them into Variables of word indices.
 8                 self.model.LSTMHidden = self.model.init_hidden()
 9                 exam = trainExamples[indexes[idx]]
10                 # Step 3. Run our forward pass. Compute the loss, gradients, and update the parameters by calling optimizer.step()
11                 lstm_feats = self.model(exam.feat)
12                 loss = self.model.crf.neg_log_likelihood(lstm_feats, exam.labelIndexs)
13                 loss.backward()
14                 optimizer.step()
15                 if (idx + 1) % self.hyperParams.verboseIter == 0:
16                     print('current: ', idx + 1,  ", cost:", loss.data[0])

1                 if idx == 0:
2                     viterbi_var.append(feat[next_tag].view(1, -1))

 1             for next_tag in range(self.labelSize):
 2                 if idx == 0:
 3                     viterbi_var.append(feat[next_tag].view(1, -1))
 4                 else:
 5                     emit_score = feat[next_tag].view(1, -1).expand(1, self.labelSize)
 6                     trans_score = self.T[next_tag]
 7                     next_tag_var = forward_var + trans_score + emit_score
 8                     best_label_id = self.argmax(next_tag_var)
 9                     bptrs_t.append(best_label_id)
10                     viterbi_var.append(next_tag_var[0][best_label_id])

 1     def _viterbi_decode(self, feats):
 2         init_score = torch.Tensor(1, self.labelSize).fill_(0)
 3         # forward_var at step i holds the viterbi variables for step i-1
 4         forward_var = autograd.Variable(init_score)
 5         back = []
 6         for idx in range(len(feats)):
 7             feat = feats[idx]
 8             bptrs_t = []                        # holds the backpointers for this step
 9             viterbi_var = []                    # holds the viterbi variables for this step
10             for next_tag in range(self.labelSize):
11                 # next_tag_var[i] holds the viterbi variable for tag i at the previous step,
12                 # plus the score of transitioning from tag i to next_tag.
13                 # We don't include the emission scores here because the max does not
14                 # depend on them (we add them in below)
15                 if idx == 0:
16                     viterbi_var.append(feat[next_tag].view(1, -1))
17                 else:
18                     emit_score = feat[next_tag].view(1, -1).expand(1, self.labelSize)
19                     trans_score = self.T[next_tag]
20                     next_tag_var = forward_var + trans_score + emit_score
21                     best_label_id = self.argmax(next_tag_var)
22                     bptrs_t.append(best_label_id)
23                     viterbi_var.append(next_tag_var[0][best_label_id])
24             # Now add in the emission scores, and assign forward_var to the set of viterbi variables we just computed
25             forward_var = (torch.cat(viterbi_var)).view(1, -1)
26             if idx > 0:
27                 back.append(bptrs_t)
28         best_label_id = self.argmax(forward_var)
29         # Follow the back pointers to decode the best path.
30         best_path = [best_label_id]
31         path_score = forward_var[0][best_label_id]
32         for bptrs_t in reversed(back):
33             best_label_id = bptrs_t[best_label_id]
34             best_path.append(best_label_id)
35         best_path.reverse()
36         return path_score, best_path

 1  　　　　　　# Check predictions after training
 2             eval_dev = Eval()
 3             for idx in range(len(devExamples)):
 4                 predictLabels = self.predict(devExamples[idx])
 5                 devInsts[idx].evalPRF(predictLabels, eval_dev)
 6             print('Dev: ', end="")
 7             eval_dev.getFscore()
 8
 9             eval_test = Eval()
10             for idx in range(len(testExamples)):
11                 predictLabels = self.predict(testExamples[idx])
12                 testInsts[idx].evalPRF(predictLabels, eval_test)
13             print('Test: ', end="")
14             eval_test.getFscore()

1     def predict(self, exam):
2         tag_hiddens = self.model(exam.feat)
3         _, best_path = self.model.crf._viterbi_decode(tag_hiddens)
4         predictLabels = []
5         for idx in range(len(best_path)):
6             predictLabels.append(self.hyperParams.labelAlpha.from_id(best_path[idx]))
7         return predictLabels

 1     def getFscore(self):
 2         if self.predict_num == 0:
 3             self.precision = 0
 4         else:
 5             self.precision = self.correct_num / self.predict_num
 6
 7         if self.gold_num == 0:
 8             self.recall = 0
 9         else:
10             self.recall = self.correct_num / self.gold_num
11
12         if self.precision + self.recall == 0:
13             self.fscore = 0
14         else:
15             self.fscore = 2 * (self.precision * self.recall) / (self.precision + self.recall)
16         print("precision: ", self.precision, ", recall: ", self.recall, ", fscore: ", self.fscore)