需求，一个csv文件中有很多行，每行是个id，字符串，每个字符串可能两两相似（是相似，不是相同），怎样去重，保留两两相似度小于0.8的id。

做法，用diff库计算两两相似度，每次计算结果，这里可以用（进程/线程/协程加速），然后将相似度大于0.8的结果放到set中，最后set中取任意一个加非集合中成员，可以返回去重结果。

import sys, os
from HTMLParser import HTMLParser
from multiprocessing import Queue
wait_set=Queue()class HTMLStripper(HTMLParser):def __init__(self):self.reset()self.fed = []def handle_data(self, d):self.fed.append(d)def get_data(self):return ''.join(self.fed)def strip_tags(html):s = HTMLStripper()s.feed(html.decode('UTF-8'))return s.get_data()def distance(s1, s2):import difflibreturn difflib.SequenceMatcher(None, s1, s2).ratio()class CSV():def __init__(self, csvfile, delimiter=',', quotechar='"'):self.csvfile = csvfileself.delimiter = delimiterself.quotechar = quotecharself.items = []import csvcsv.field_size_limit(sys.maxsize)#this line is limited in linux,if run in windows will rasise#csv.field_size_limit(131072)with open(csvfile, 'rU') as f:for item in csv.reader(f, delimiter=delimiter, quotechar=quotechar):self.items.append(item)f.close()def diff(self, threshold=None, strip=None, truncate=None, start=0, end=None):i = startfor item in self.items[start:end]:i = i + 1id = item[0]text = item[1]if strip:text = strip_tags(text)# Skip the items already diffed or itselffor c_item in self.items[i:]:c_id = c_item[0]c_text = c_item[1]if strip:c_text = strip_tags(c_text)if truncate:d = distance(text[0:truncate], c_text[0:truncate])else:d = distance(text, c_text)if (threshold and d < threshold):continuewait_set.put(id)# because the subprocesswait_set.put(c_id)def output(self, data):print ','.join(data)sys.stdout.flush()def analysis(self):print "%s items, %s bytes." % ( len(self.items), os.path.getsize(self.csvfile) )print '%6s %32s %32s %32s' % ('No', 'UUID', 'Length(after strip)', 'Length(before strip)')i = 0for item in self.items:print '%6s %32s %32s %32s' % (i, item[0], len(strip_tags(item[1])), len(item[1]))i = i + 1;def wrapper(args):CSV.diff(*args)def main():csvfile = sys.argv[1]threshold = 0.8threads = 4strip = Truetruncate = 256action = 'diff'if len(sys.argv) > 2:if sys.argv[2] == '-a':action = 'analysis'else:threshold = float(sys.argv[2])if len(sys.argv) > 3:threads = int(sys.argv[3])if len(sys.argv) > 4:strip = bool(int(sys.argv[4]))if len(sys.argv) > 5:truncate = int(sys.argv[5])c = CSV(csvfile)if action == 'analysis':c.analysis()elif action == 'diff':if threads > 1:batch = len(c.items) / threadstail = len(c.items) % threadsfrom multiprocessing import Poolpool = Pool()args = []i = 0while (i < threads):start = batch * iend = start + batch# Last loopif i == (threads - 1):end = end + tailargs.append( (c, threshold, strip, truncate, start, end) )i = i + 1pool.map(wrapper, args)else:c.diff(threshold, strip, truncate)wait_list=[x[0] for x in c.items[1:]]if wait_set.qsize:queue_list=list(set([wait_set.get() for x in range(wait_set.qsize())]))for x in range(len(queue_list)-1):#left one elementele = queue_list[x]if ele in wait_list:wait_list.remove(ele)print wait_listelse:return wait_listsys.stdout.flush()if __name__ == '__main__':main()