https://github.com/toolgood/ToolGood.Words

名为DFA的算法，即Deterministic Finite Automaton算法，翻译成中文就是确定有穷自动机算法。它的基本思想是基于状态转移来检索敏感词，只需要扫描一次待检测文本，就能对所有敏感词进行检测

假设我们有以下5个敏感词需要检测：傻逼、傻子、傻大个、坏蛋、坏人。那么我们可以先把敏感词中有相同前缀的词组合成一个树形结构，不同前缀的词分属不同树形分支，以上述5个敏感词为例，可以初始化成如下2棵树：

image.png

class DFAFilter(object):def __init__(self):self.keyword_chains = {}  # 关键词链表self.delimit = '\x00'  # 限定def add(self, keyword):keyword = keyword.lower()  # 关键词英文变为小写chars = keyword.strip()  # 关键字去除首尾空格和换行if not chars:  # 如果关键词为空直接返回returnlevel = self.keyword_chains# 遍历关键字的每个字for i in range(len(chars)):# 如果这个字已经存在字符链的key中就进入其子字典if chars[i] in level:level = level[chars[i]]else:if not isinstance(level, dict):breakfor j in range(i, len(chars)):level[chars[j]] = {}last_level, last_char = level, chars[j]level = level[chars[j]]last_level[last_char] = {self.delimit: 0}breakif i == len(chars) - 1:level[self.delimit] = 0def parse(self, cache_name, path):with open(path, encoding='utf-8') as f:for keyword in f:self.add(str(keyword).strip())f = f.read()# print(self.keyword_chains)def filter(self, message, repl="*"):message = message.lower()ret = []start = 0while start < len(message):level = self.keyword_chainsstep_ins = 0for char in message[start:]:if char in level:step_ins += 1if self.delimit not in level[char]:level = level[char]else:ret.append(repl * step_ins)start += step_ins - 1breakelse:ret.append(message[start])breakelse:ret.append(message[start])start += 1return ''.join(ret)