def process_file(path):try:with open(path, 'r') as file:text = file.read()except IOError:print("Read File Error!")return None
    return text

import re
word_list = re.sub('[^a-zA-Z0-9n]', ' ', textString).lower().split()

for word in word_list:if word in word_freq:word_freq[word] += 1else:word_freq[word] = 1

                "mother":{"Value":1}            

count = len(word_list)i = 0while i < count:  # 因为需要用到索引遍历列表，只能使用while来遍历列表    finish = i    start = i - num + 1  # num表示词组的单词个数限制，start表示以该单词作为词组结尾的第一个单词的索引    if start < 0:        start = 0  # 处理开始时索引前面没有单词的特殊情况    index = i    while index >= start:  # 做num次建立节点        if word_list[i] in get_dict_value(word_freq, word_list[index: finish]).keys():            get_dict_value(word_freq, word_list[index: finish])[word_list[i]]['Value'] += 1        else:            get_dict_value(word_freq, word_list[index: finish]).update({word_list[i]: {'Value': 1}})        index -= 1    i += 1

def get_dict_value(word_freq={}, keys=[]):"""如果keys为字符串,返回word_freq字典中以keys为键的值。如果keys为列表,则使用eval()函数进行字符串拼接，深度查找word_freq字典中以keys为键的值。"""if type(keys).__name__ == 'str':return word_freq[keys]else:count = len(keys)if count == 0:return word_freqelif count == 1:return word_freq[keys[0]]elif count == 2:return word_freq[keys[0]][keys[1]]elif count == 3:return word_freq[keys[0]][keys[1]][keys[2]] #对寻找三个单词以下的词组进行特化else:string = "word_freq['"string += "']['".join(keys)string += "']"return eval(string) #动态寻找字典的值的一般版本

def format_dict(word_freq={}):"""对统计短语的情况生成的复杂字典进行格式化，格式化后的形式为<str,int>"""formated_word_freq = {}phrases = []for word in word_freq.keys(): #将所有根节点放入队列中phrases.append(word)while len(phrases) > 0: #只要队列还有元素，就表明还没有遍历结束phrase = phrases[0]if len(get_dict_value(word_freq, phrase)) == 1 and type(phrase).__name__ == 'list':formated_word_freq[' '.join(phrase)] = get_dict_value(word_freq, phrase)['Value'] #搜索到叶子节点了，这个节点存入格式化好的字典中else:for next_word in get_dict_value(word_freq, phrase): #除"Value"键值对以外的键都入队temp = []if type(phrase).__name__ == 'str':temp.append(phrase)else:temp.extend(phrase)if next_word != 'Value':temp.append(next_word)phrases.append(temp)phrases.pop(0) #搜索完队列的第一个节点的子节点，这个节点出队# print(formated_word_freq)return formated_word_freq

def output_result(word_freq):if word_freq:sorted_word_freq = sorted(word_freq.items(), key=lambda v: v[1], reverse=True)for item in sorted_word_freq[:10]:print(item)

if __name__ == "__main__":import argparseparser = argparse.ArgumentParser()parser.add_argument('path') #路径parser.add_argument('num') #词组单词个数args = parser.parse_args()path = args.pathnum = int(args.num)buffer = process_file(path)if buffer:word_freq = process_buffer(buffer, num)if num != 1:word_freq = format_dict(word_freq)output_result(word_freq)

def get_dict_value(word_freq={}, keys=[]):"""如果keys为字符串,返回word_freq字典中以keys为键的值。如果keys为列表,则使用eval()函数进行字符串拼接，深度查找word_freq字典中以keys为键的值。"""if type(keys).__name__ == 'str':return word_freq[keys]else:count = len(keys)if count == 0:return word_freqelif count == 1:return word_freq[keys[0]]elif count == 2:return word_freq[keys[0]][keys[1]]elif count == 3:return word_freq[keys[0]][keys[1]][keys[2]] #对寻找三个单词以下的词组进行特化else:string = "word_freq['"string += "']['".join(keys)string += "']"return eval(string) #动态寻找字典的值的一般版本

python statistic.py Gone_with_the_wind.txt 1

python statistic.py Gone_with_the_wind.txt 2

python -m cProfile -o result.out -s cumulative statistic.py Gone_with_the_wind.txt 2

python gprof2dot.py -f pstats result.out | dot -Tpng -o result.png