三种分词算法比较

dt = {'空调': 1, '调和': 1, '和风': 1, '风扇': 1,'空': 1, '调': 1, '和': 2, '风': 1, '扇': 1}  # 词典
max_len = max(len(w) for w in dt)  # 词最大长度，默认等于词典最长词
total = sum(dt.values())  # 总频数
sentence = '空调和风扇'
length = len(sentence)def maximum_matching(sentence):"""正向最大匹配"""head = 0while head < length:tail = min(head + max_len, length)for middle in range(tail, head + 1, -1):word = sentence[head: middle]if word in dt:head = middlebreakelse:word = sentence[head]head += 1yield worddef reverse_maximum_matching(sentence):"""逆向最大匹配"""words = []tail = lengthwhile tail > 0:head = min(tail - max_len, 0)for middle in range(head, tail - 1):word = sentence[middle: tail]if word in dt:tail = middlebreakelse:tail -= 1word = sentence[tail]words.append(word)  # 比words.insert(0, word)快6%return words[::-1]def probability(sentence):"""贝叶斯网络"""# get DAGDAG = dict()for head in range(length):DAG.update({head: [head]})tail = min(head + max_len, length)for middle in range(head + 2, tail + 1):word = sentence[head: middle]if word in dt:DAG[head].append(middle - 1)# calculate routeroute = {}route[length] = (1, 1)for idx in range(length - 1, -1, -1):route[idx] = max((dt.get(sentence[idx:x + 1], 0) / total * route[x + 1][0], x)for x in DAG[idx])# yieldx = 0while x < length:y = route[x][1] + 1l_word = sentence[x:y]yield l_wordx = yprint(list(maximum_matching(sentence)))
print(reverse_maximum_matching(sentence))
print(list(probability(sentence)))

空调和风扇分词结果: [‘空调’, ‘和风’, ‘扇’]
[‘空’, ‘调和’, ‘风扇’]
[‘空调’, ‘和’, ‘风扇’]

逆向最大匹配

从后往前扫描

sentence = '西樵山'
# 初始化扫描位置，从句尾开始
tail = len(sentence)
# 词最大长度
max_len = 3
# 逆向扫描
while tail > 0:head = tail - max_lenif head < 0:head = 0for middle in range(head, tail):word = sentence[middle: tail]print(head, middle, tail, word)tail -= 1

0 0 3 西樵山
0 1 3 樵山
0 2 3 山
0 0 2 西樵
0 1 2 樵
0 0 1 西

词典匹配

dt = {'桂江二中', '毕业', '二中'}  # 词典
sentence = '桂江二中毕业'tail = len(sentence)
max_len = 4
words = []
while tail > 0:head = max(tail - max_len, 0)for middle in range(head, tail - 1):  # 忽略长度为1的词word = sentence[middle: tail]if word in dt:print(middle, tail - 1, word)words.append(word)tail = middlebreakelse:tail -= 1print(tail, tail, sentence[tail])words.append(sentence[tail])
print(words[::-1])

4 5 毕业
0 3 桂江二中
[‘桂江二中’, ‘毕业’]

概率分词

原理

基于词典，对句子进行词图扫描，生成所有成词情况所构成的有向无环图（Directed Acyclic Graph）
根据DAG，反向计算最大概率路径（使用动态规划算法）
根据路径获取最大概率的分词序列

dt = {'空调': 1, '调和': 1, '和风': 1, '风扇': 1,'空': 1, '调': 1, '和': 2, '风': 1, '扇': 1}  # 词典
max_len = max(len(w) for w in dt)  # 词最大长度，默认等于词典最长词
total = sum(dt.values())  # 总频数
sentence = '空调和风扇'
length = len(sentence)

total=1+1+1+1+1+1+2+1+1=10total=1+1+1+1+1+1+2+1+1=10total=1+1+1+1+1+1+2+1+1=10
P(空调，和风，扇)=110∗110∗110=11000P(空调，和风，扇)=\frac {1} {10} * \frac {1} {10} * \frac {1} {10} = \frac {1} {1000}P(空调，和风，扇)=101∗101∗101=10001
P(空，调和，风扇)=110∗110∗110=11000P(空，调和，风扇)=\frac {1} {10} * \frac {1} {10} * \frac {1} {10} = \frac {1} {1000}P(空，调和，风扇)=101∗101∗101=10001
P(空调，和，风扇)=110∗210∗110=21000P(空调，和，风扇)=\frac {1} {10} * \frac {2} {10} * \frac {1} {10} = \frac {2} {1000}P(空调，和，风扇)=101∗102∗101=10002
P(空调，和，风扇)最大P(空调，和，风扇)最大P(空调，和，风扇)最大

DAG

DAG = dict()
for head in range(length):DAG.update({head: [head]})tail = min(head + max_len, length)for middle in range(head + 2, tail + 1):word = sentence[head: middle]if word in dt:DAG[head].append(middle - 1)
print(DAG)

{0: [0, 1], 1: [1, 2], 2: [2, 3], 3: [3, 4], 4: [4]}

计算大概率路径

route = {}
route[length] = (1, 1)
for idx in range(length - 1, -1, -1):route[idx] = max((dt.get(sentence[idx:x + 1], 0) / total * route[x + 1][0], x)for x in DAG[idx])
print(route)

{5: (1, 1), 4: (0.1, 4), 3: (0.1, 4), 2: (0.02, 2), 1: (0.01, 2), 0: (0.002, 1)}

封装

from os import path
import re
import jieba
from math import log
fname = path.join(path.dirname(jieba.__file__), 'dict.txt')
NA = 'NA'class Tokenizer:re_eng = re.compile('[a-zA-Z]+')re_m = re.compile('[0-9]+')  # jieba数词标注为mdef __init__(self, word2freq, total, word2flag, max_len):self.word2freq = word2freqself.total = totalself.word2flag = word2flagself.max_len = max_len@classmethoddef initialization(cls):word2freq, total, word2flag = dict(), 0, dict()with open(fname, encoding='utf-8') as f:for line in f.read().strip().split('\n'):word, freq, flag = line.split()freq = int(freq)word2freq[word] = freqtotal += freqword2flag[word] = flag# 词最大长度，默认等于词典最长词（超长英文符会识别不出来）max_len = max(len(i) for i in word2freq.keys())return cls(word2freq, total, word2flag, max_len)def _get_DAG(self, sentence):length = len(sentence)DAG = dict()for head in range(length):DAG.update({head: [head]})tail = min(head + self.max_len, length)for middle in range(head + 2, tail + 1):word = sentence[head: middle]# ------------- 词典 + 正则 ------------- #if word in self.word2freq:DAG[head].append(middle - 1)elif self.re_eng.fullmatch(word):DAG[head].append(middle - 1)elif self.re_m.fullmatch(word):DAG[head].append(middle - 1)return DAGdef _calculate(self, sentence):DAG = self._get_DAG(sentence)length = len(sentence)route = dict()route[length] = (0, 0)logtotal = log(self.total)for idx in range(length - 1, -1, -1):route[idx] = max((log(self.word2freq.get(sentence[idx:x + 1], 1)) - logtotal + route[x + 1][0], x)for x in DAG[idx])return routedef cut(self, sentence):route = self._calculate(sentence)length = len(sentence)x = 0while x < length:y = route[x][1] + 1l_word = sentence[x:y]yield l_wordx = ydef lcut(self, sentence):return list(self.cut(sentence))def add_word(self, word, freq=-1, flag=NA):if freq >= 0:self.del_word(word)else:freq = 1original_freq = self.word2freq.get(word, 0)self.word2freq[word] = original_freq + freqself.total = self.total - original_freq + self.word2freq[word]self.word2flag[word] = flagdef del_word(self, word):original_freq = self.word2freq.get(word)if original_freq is not None:del self.word2freq[word]self.total -= original_freqdel self.word2flag[word]def cut2position(self, sentence):route = self._calculate(sentence)x = 0length = len(sentence)while x < length:y = route[x][1] + 1l_word = sentence[x:y]yield l_word, x, yx = ydef get_flag(self, word):return self.word2flag.get(word, NA)def get_flags(self, words):if isinstance(words, str):words = self.cut(words)return [self.get_flag(word) for word in words]# 实例化
tokenizer = Tokenizer.initialization()
cut = tokenizer.cut
lcut = tokenizer.lcut
add_word = tokenizer.add_word
del_word = tokenizer.del_word
cut2position = tokenizer.cut2position
get_flag = tokenizer.get_flag
get_flags = tokenizer.get_flagsif __name__ == '__main__':text = '幻刺斩杀大法师'print(lcut(text))add_word('幻刺', 2, 'N')print(list(cut2position(text)))del_word('大法师')print(lcut(text))print(get_flags(text))

图论知识补充

图的表示方法

networkx

%matplotlib inline
import networkx as nx
# 创建图
G = nx.DiGraph()
# 添加边
G.add_edges_from([(0, 1), (0, 2), (1, 2), (2, 3)])
# 绘图
nx.draw(G, with_labels=True, font_size=36, node_size=1500, width=4, node_color='lightgreen')

矩阵

class G:def __init__(self, nodes):self.matrix = [[0] * nodes for _ in range(nodes)]def add_edge(self, start, end, value=1):self.matrix[start][end] = valueg = G(4)
g.add_edge(0, 1)
g.add_edge(0, 2)
g.add_edge(1, 2)
g.add_edge(2, 3)
print(g.matrix)

字典

class G:def __init__(self):self.dt = dict()def add_edge(self, start, end, value=1):self.dt[start] = self.dt.get(start, dict())self.dt[start][end] = valueg = G()
g.add_edge(0, 1)
g.add_edge(0, 2)
g.add_edge(1, 2)
g.add_edge(2, 3)
print(g.dt)

概率图模型

概率图模型：利用图来表示与模型有关的变量的联合概率分布
贝叶斯网络模型：由有向无环图和条件概率表（Conditional Probability Table，CPT）组成

贝叶斯网络

Python-基于词典-中文分词算法相关推荐

中文分词算法python代码_中文分词算法之最大正向匹配算法（Python版）
最大匹配算法是自然语言处理中的中文匹配算法中最基础的算法,分为正向和逆向,原理都是一样的. 正向最大匹配算法,故名思意,从左向右扫描寻找词的最大匹配. 首先我们可以规定一个词的最大长度,每次扫描的时候 ...
在Hadoop上运行基于RMM中文分词算法的MapReduce程序
原文:http://xiaoxia.org/2011/12/18/map-reduce-program-of-rmm-word-count-on-hadoop/ 在Hadoop上运行基于RMM中文分词 ...
正向最大匹配算法 python代码_中文分词算法之最大正向匹配算法（Python版）
最大匹配算法是自然语言处理中的中文匹配算法中最基础的算法,分为正向和逆向,原理都是一样的. 正向最大匹配算法,故名思意,从左向右扫描寻找词的最大匹配. 首先我们可以规定一个词的最大长度,每次扫描的时候 ...
基于笔画中文分词算法---蚂蚁金服
原标题:AAAI 2018 | 蚂蚁金服公开最新基于笔画的中文词向量算法转载自蚂蚁金服科技查看全文 http://www.taodudu.cc/news/show-5808302.html 相关文 ...
Mmseg中文分词算法解析
Mmseg中文分词算法解析 @author linjiexing 开发中文搜索和中文词库语义自己主动识别的时候,我採用都是基于mmseg中文分词算法开发的Jcseg开源project.使用场景涉及搜索 ...
NLP-基础任务-中文分词算法(1)-基于词典：机械分词（词典字符串匹配）：前向最大匹配、后向最大匹配、双向最大匹配【OOV：基于现有词典，不能进行新词发现处理】
分词与NLP关系:分词是中文自然语言处理的基础,没有中文分词,我们对语言很难量化,进而很能运用数学的知识去解决问题.对于拉丁语系是不需要分词的. 拉丁语系与亚系语言区别拉丁语言系不需要分词,因为他们 ...
基于词典的正向最大匹配中文分词算法，能实现中英文数字混合分词
基于词典的正向最大匹配中文分词算法,能实现中英文数字混合分词.比如能分出这样的词:bb霜.3室.乐phone.touch4.mp3.T恤第一次写中文分词程序,欢迎拍砖. publicclass MM ...
基于词典的逆向最大匹配中文分词算法，更好实现中英文数字混合分词
基于词典的逆向最大匹配中文分词算法,能实现中英文数字混合分词.比如能分出这样的词:bb霜.3室.乐phone.touch4.mp3.T恤.实际分词效果比正向分词效果好 publicclass RMM ...
python最大分词_中文分词算法之最大正向匹配算法（Python版）
最大匹配算法是自然语言处理中的中文匹配算法中最基础的算法,分为正向和逆向,原理都是一样的. 正向最大匹配算法,故名思意,从左向右扫描寻找词的最大匹配. 首先我们可以规定一个词的最大长度,每次扫描的时候 ...

Python-基于词典-中文分词算法

文章目录