python实现中文繁体字与简体字转换
2024-04-12 06:16:37
首先需要两个工具文件,鄙人放在了utils文件夹中:
1. langconv.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-from copy import deepcopytry:import psycopsyco.full()
except:passtry:from utils.zh_wiki import zh2Hant, zh2Hans
except ImportError:from zhtools.zh_wiki import zh2Hant, zh2Hansimport sys
py3k = sys.version_info >= (3, 0, 0)if py3k:UEMPTY = ''
else:_zh2Hant, _zh2Hans = {}, {}for old, new in ((zh2Hant, _zh2Hant), (zh2Hans, _zh2Hans)):for k, v in old.items():new[k.decode('utf8')] = v.decode('utf8')zh2Hant = _zh2Hantzh2Hans = _zh2HansUEMPTY = ''.decode('utf8')# states
(START, END, FAIL, WAIT_TAIL) = list(range(4))
# conditions
(TAIL, ERROR, MATCHED_SWITCH, UNMATCHED_SWITCH, CONNECTOR) = list(range(5))MAPS = {}class Node(object):def __init__(self, from_word, to_word=None, is_tail=True,have_child=False):self.from_word = from_wordif to_word is None:self.to_word = from_wordself.data = (is_tail, have_child, from_word)self.is_original = Trueelse:self.to_word = to_word or from_wordself.data = (is_tail, have_child, to_word)self.is_original = Falseself.is_tail = is_tailself.have_child = have_childdef is_original_long_word(self):return self.is_original and len(self.from_word)>1def is_follow(self, chars):return chars != self.from_word[:-1]def __str__(self):return '<Node, %s, %s, %s, %s>' % (repr(self.from_word),repr(self.to_word), self.is_tail, self.have_child)__repr__ = __str__class ConvertMap(object):def __init__(self, name, mapping=None):self.name = nameself._map = {}if mapping:self.set_convert_map(mapping)def set_convert_map(self, mapping):convert_map = {}have_child = {}max_key_length = 0for key in sorted(mapping.keys()):if len(key)>1:for i in range(1, len(key)):parent_key = key[:i]have_child[parent_key] = Truehave_child[key] = Falsemax_key_length = max(max_key_length, len(key))for key in sorted(have_child.keys()):convert_map[key] = (key in mapping, have_child[key],mapping.get(key, UEMPTY))self._map = convert_mapself.max_key_length = max_key_lengthdef __getitem__(self, k):try:is_tail, have_child, to_word = self._map[k]return Node(k, to_word, is_tail, have_child)except:return Node(k)def __contains__(self, k):return k in self._mapdef __len__(self):return len(self._map)class StatesMachineException(Exception): passclass StatesMachine(object):def __init__(self):self.state = STARTself.final = UEMPTYself.len = 0self.pool = UEMPTYdef clone(self, pool):new = deepcopy(self)new.state = WAIT_TAILnew.pool = poolreturn newdef feed(self, char, map):node = map[self.pool+char]if node.have_child:if node.is_tail:if node.is_original:cond = UNMATCHED_SWITCHelse:cond = MATCHED_SWITCHelse:cond = CONNECTORelse:if node.is_tail:cond = TAILelse:cond = ERRORnew = Noneif cond == ERROR:self.state = FAILelif cond == TAIL:if self.state == WAIT_TAIL and node.is_original_long_word():self.state = FAILelse:self.final += node.to_wordself.len += 1self.pool = UEMPTYself.state = ENDelif self.state == START or self.state == WAIT_TAIL:if cond == MATCHED_SWITCH:new = self.clone(node.from_word)self.final += node.to_wordself.len += 1self.state = ENDself.pool = UEMPTYelif cond == UNMATCHED_SWITCH or cond == CONNECTOR:if self.state == START:new = self.clone(node.from_word)self.final += node.to_wordself.len += 1self.state = ENDelse:if node.is_follow(self.pool):self.state = FAILelse:self.pool = node.from_wordelif self.state == END:# END is a new STARTself.state = STARTnew = self.feed(char, map)elif self.state == FAIL:raise StatesMachineException('Translate States Machine ''have error with input data %s' % node)return newdef __len__(self):return self.len + 1def __str__(self):return '<StatesMachine %s, pool: "%s", state: %s, final: %s>' % (id(self), self.pool, self.state, self.final)__repr__ = __str__class Converter(object):def __init__(self, to_encoding):self.to_encoding = to_encodingself.map = MAPS[to_encoding]self.start()def feed(self, char):branches = []for fsm in self.machines:new = fsm.feed(char, self.map)if new:branches.append(new)if branches:self.machines.extend(branches)self.machines = [fsm for fsm in self.machines if fsm.state != FAIL]all_ok = Truefor fsm in self.machines:if fsm.state != END:all_ok = Falseif all_ok:self._clean()return self.get_result()def _clean(self):if len(self.machines):self.machines.sort(key=lambda x: len(x))# self.machines.sort(cmp=lambda x,y: cmp(len(x), len(y)))self.final += self.machines[0].finalself.machines = [StatesMachine()]def start(self):self.machines = [StatesMachine()]self.final = UEMPTYdef end(self):self.machines = [fsm for fsm in self.machinesif fsm.state == FAIL or fsm.state == END]self._clean()def convert(self, string):self.start()for char in string:self.feed(char)self.end()return self.get_result()def get_result(self):return self.finaldef registery(name, mapping):global MAPSMAPS[name] = ConvertMap(name, mapping)registery('zh-hant', zh2Hant)
registery('zh-hans', zh2Hans)
del zh2Hant, zh2Hansdef run():import sysfrom optparse import OptionParserparser = OptionParser()parser.add_option('-e', type='string', dest='encoding',help='encoding')parser.add_option('-f', type='string', dest='file_in',help='input file (- for stdin)')parser.add_option('-t', type='string', dest='file_out',help='output file')(options, args) = parser.parse_args()if not options.encoding:parser.error('encoding must be set')if options.file_in:if options.file_in == '-':file_in = sys.stdinelse:file_in = open(options.file_in)else:file_in = sys.stdinif options.file_out:if options.file_out == '-':file_out = sys.stdoutelse:file_out = open(options.file_out, 'wb')else:file_out = sys.stdoutc = Converter(options.encoding)for line in file_in:# print >> file_out, c.convert(line.rstrip('\n').decode(file_out.write(c.convert(line.rstrip('\n').decode('utf8')).encode('utf8'))if __name__ == '__main__':run()2. zh_wiki.py
# -*- coding: utf-8 -*-
# copy fom wikipediazh2Hant = {
'呆': '獃',
"打印机": "印表機",
'帮助文件': '說明檔案',
"画": "畫",
"龙": "竜",
"板": "板",
"表": "表",
"才": "才",
"丑": "醜",
"出": "出",
"淀": "澱",
"冬": "冬",
"范": "範",
"丰": "豐",
"刮": "刮",
"后": "後",
"胡": "胡",
"回": "回",
"伙": "夥",
"姜": "薑",
"借": "借",
"克": "克",
"困": "困",
"漓": "漓",
"里": "里",
"帘": "簾",
"霉": "霉",
"面": "面",
"蔑": "蔑",
"千": "千",
"秋": "秋",
"松": "松",
"咸": "咸",
"向": "向",
"余": "餘",
"郁": "鬱",
"御": "御",
"愿": "願",
"云": "雲",
"芸": "芸",
"沄": "沄",
"致": "致",
"制": "制",
"朱": "朱",
"筑": "築",
"准": "準",
"厂": "廠",
"广": "廣",
"辟": "闢",
"别": "別",
"卜": "卜",
"沈": "沈",
"冲": "沖",
"种": "種",
"虫": "蟲",
"担": "擔",
"党": "黨",
"斗": "鬥",
"儿": "兒",
"干": "乾",
"谷": "谷",
"柜": "櫃",
"合": "合",
"划": "劃",
"坏": "壞",
"几": "幾",
"系": "系",
"家": "家",
"价": "價",
"据": "據",
"卷": "捲",
"适": "適",
"蜡": "蠟",
"腊": "臘",
"了": "了",
"累": "累",
"么": "麽",
"蒙": "蒙",
"万": "萬",
"宁": "寧",
"朴": "樸",
"苹": "蘋",
"仆": "僕",
"曲": "曲",
"确": "確",
"舍": "舍",
"胜": "勝",
"术": "術",
"台": "台",
"体": "體",
"涂": "塗",
"叶": "葉",
"吁": "吁",
"旋": "旋",
"佣": "傭",
"与": "與",
"折": "折",
"征": "徵",
"症": "症",
"恶": "惡",
"发": "發",
"复": "復",
"汇": "匯",
"获": "獲",
"饥": "飢",
"尽": "盡",
"历": "歷",
"卤": "滷",
"弥": "彌",
"签": "簽",
"纤": "纖",
"苏": "蘇",
"坛": "壇",
"团": "團",
"须": "須",
"脏": "臟",
"只": "只",
"钟": "鐘",
"药": "藥",
"同": "同",
"志": "志",
"杯": "杯",
"岳": "岳",
"布": "布",
"当": "當",
"吊": "弔",
"仇": "仇",
"蕴": "蘊",
"线": "線",
"为": "為",
"产": "產",
"众": "眾",
"伪": "偽",
"凫": "鳧",
"厕": "廁",
"启": "啟",
"墙": "牆",
"壳": "殼",
"奖": "獎",
"妫": "媯",
"并": "並",
"录": "錄",
"悫": "愨",
"极": "極",
"沩": "溈",
"瘘": "瘺",
"硷": "鹼",
"竖": "豎",
"绝": "絕",
"绣": "繡",
"绦": "絛",
"绱": "緔",
"绷": "綳",
"绿": "綠",
"缰": "韁",
"苧": "苎",
"莼": "蒓",
"说": "說",
"谣": "謠",
"谫": "譾",
"赃": "贓",
"赍": "齎",
"赝": "贗",
"酝": "醞",
"采": "採",
"钩": "鉤",
"钵": "缽",
"锈": "銹",
"锐": "銳",
"锨": "杴",
"镌": "鐫",
"镢": "钁",
"阅": "閱",
"颓": "頹",
"颜": "顏",
"骂": "罵",
"鲇": "鯰",
"鲞": "鯗",
"鳄": "鱷",
"鸡": "雞",
"鹚": "鶿",
"荡": "盪",
"锤": "錘",
"㟆": "㠏",
"㛟": "
python实现中文繁体字与简体字转换相关推荐
- 中文繁体字与简体字转换
/* * $Id: ChangeCode.java$ * * 来自:http://thorlst.blog.163.com/blog/static/59275749201122402041317 ...
- 【python】epub的电子书繁体字与简体字转换
在网上找到了一本非常喜欢的电子书,但是它是繁体字.在网上找了很多转换工具不是收费就是转换后格式出现问题,后面又找到了一段python代码,使用起来非常简单,而且格式不会出现问题,非常好用! impor ...
- python繁体中文到简体中文的转换
处理中文字符串遇到了繁体和简体中文的转换,python版: 1.下载zh_wiki.py及langconv zh_wiki.py:https://github.com/skydark/nstools/ ...
- python实现中文的繁简转换
我们在对中文语料作预处理时,往往会遇到繁简体转换的问题,这也是预处理环节的重要一环. python支持繁简体转换的库有许多,常用是opencc,但是由于直接安装opencc或是opencc-pytho ...
- 【Python】用OpenCC将繁体字转为简体字
官方GitHub:BYVoid/OpenCC 相关文章:python实现中文的繁简转换 一.安装 pip install opencc-python-reimplemented 二.使用 1. 繁体字 ...
- 学点实用工作小技巧【Python】汉字转拼音、繁体字和简体字互转、提取字符串中的中文（英文）、判断是否纯中文（英文）
大家早上好,本人姓吴,如果觉得文章写得还行的话也可以叫我吴老师.欢迎大家跟我一起走进数据分析的世界,一起学习! 感兴趣的朋友可以关注我或者我的数据分析专栏,里面有许多优质的文章跟大家分享哦. 前言 又 ...
- python获取中文字体点阵坐标_Python实现点阵字体读取与转换的方法
点阵字体是指根据文字的像素点来显示的字体,效果如下: 使用Python读取并显示的过程如下: 根据中文字符获取GB2312编码 通过GB2312编码计算该汉字在点阵字库中的区位和码位 通过区位和码位计 ...
- python url中文转码_python实现中文转换url编码的方法
本文实例讲述了python实现中文转换url编码的方法.分享给大家供大家参考,具体如下: 今天要处理百度贴吧的东西.想要做一个关键词的list,每次需要时,直接添加 到list里面就可以了.但是添加到 ...
- python的chr可以转换中文吗,chr()在python中怎么实现编码的转换
chr()在python中怎么实现编码的转换 发布时间:2020-12-14 09:36:42 来源:亿速云 阅读:95 作者:小新 这篇文章主要介绍chr()在python中怎么实现编码的转换,文中 ...
- python-10-爬虫selenium繁体字转换为简体字和NCR字符处理
1 将NCR字符转换成真实字符 以 &# 或 &#x 开头的字符串叫做 NCR 字符,在浏览器中查看会直接转换成中文. 一个Numeric Character Reference编码是 ...
最新文章
- Jasperreport导出pdf内容展示不完全处理
- 石川es6课程---5、函数-参数
- Python Train_出乎意料的简单！10分钟用python建立人工智能预测模型
- 用友U8固定资产总账重算语句
- hdu 1203 I NEED A OFFER!
- HTTPS是如何保证数据传输安全的？
- 世园会开幕式上的机器人_小胖机器人驻扎“2019北京世园会”媒体分会场
- 魔兽地图服务器修改,如何修改魔兽地图（傻瓜版）
- 单片机八灯交替闪烁c语言代码,单片机闪烁灯汇编语言源代码大全（四款闪烁灯的汇编语言源代码）...
- 小菜鸟的自我激励与不服输的心
- python中五个一行输出_python 如何将一系列数字十个一行输出
- 重启docker卡死问题解决systemctl start docker卡住不动
- 网络攻防环境的搭建--MacOs配置Vmnet1及Vmnet8
- 第三方支付平台结算流程是什么样的？
- link2SD的原理以及开启方法
- 『VQA认知』如何理解Bias？
- 云对象 - 重新定义前后端交互
- 配置计算机共享为免验证,网启局域网全网批量和单一机装系统工具winpe版
- Dockerfile的使用和实践（10）
- 《C语言编程魔法书：基于C11标准》——2.9 本章小结
热门文章