MMR 排序多样化重排序算法

参考：https://zhuanlan.zhihu.com/p/102285855
https://blog.csdn.net/Mercedes_wwz/article/details/109028124

[公式] : 用户;
[公式] : 推荐结果集合;
[公式] : [公式] 中已被选中集合; R\S: [公式] 中未被选中集合;
[公式] : 权重系数，调节推荐结果相关性与多样性

sim1是query与doc的相关权重；sim2是docs之间的相关权重


def MMR(itemScoreDict, similarityMatrix, lambdaConstant=0.5, topN=20):s, r = [], list(itemScoreDict.keys())while len(r) > 0:score = 0selectOne = Nonefor i in r:firstPart = itemScoreDict[i]secondPart = 0for j in s:sim2 = similarityMatrix[i][j]if sim2 > secondPart:secondPart = sim2equationScore = lambdaConstant * (firstPart - (1 - lambdaConstant) * secondPart)if equationScore > score:score = equationScoreselectOne = iif selectOne == None:selectOne = ir.remove(selectOne)s.append(selectOne)return (s, s[:topN])[topN > len(s)]

或


import numpy as npclass MMRModel(object):# def __init__(self,item_score_dict,similarity_matrix,lambda_constant,topN):#     self.item_score_dict = item_score_dict#     self.similarity_matrix = similarity_matrix#     self.lambda_constant = lambda_constant#     self.topN = topNdef __init__(self, **kwargs):self.lambda_constant = kwargs['lambda_constant']self.topN = kwargs['topN']def build_data(self):sorce = np.random.random(size=(self.topN))item = np.random.randint(1, 1000, size=self.topN)self.item_score_dict = dict()for i in range(len(item)):self.item_score_dict[i] = sorce[i]item_embedding = np.random.randn(self.topN, self.topN)  # item的embeddingitem_embedding = item_embedding / np.linalg.norm(item_embedding, axis=1, keepdims=True)sim_matrix = np.dot(item_embedding, item_embedding.T)  # item之间的相似度矩阵self.similarity_matrix = sorce.reshape((self.topN, 1)) * sim_matrix * sorce.reshape((1, self.topN))def mmr(self):s, r = [], list(self.item_score_dict.keys())while len(r) > 0:score = 0select_item = Nonefor i in r:sim1 = self.item_score_dict[i]sim2 = 0for j in s:if self.similarity_matrix[i][j] > sim2:sim2 = self.similarity_matrix[i][j]equation_score = self.lambda_constant * sim1 - (1 - self.lambda_constant) * sim2if equation_score > score:score = equation_scoreselect_item = iif select_item == None:select_item = ir.remove(select_item)s.append(select_item)return (s, s[:self.topN])[self.topN > len(s)]if __name__ == "__main__":kwargs = {'lambda_constant': 0.5,'topN': 5,}dpp_model = MMRModel(**kwargs)dpp_model.build_data()print(dpp_model.mmr())

使用tfidf得分来测试

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import jieba
import jieba.analyse
import numpy as npdef stopwordslist(path):stopwords = [line.strip() for line in open(path, encoding='UTF-8').readlines()]return stopwordsdef split_word(sentence_depart):# 输出结果为outstroutstr = ''# 去停用词for word in sentence_depart:if word not in stopwords:if word != '\t':outstr += wordoutstr += " "return outstrdef cut_(words):all_querys = []# qq1 = datas_all["{}".format(word)].unique()for ii in list(words):ex_ = split_word(jieba.cut(str(ii)))#     print("111",ii,ex_)all_querys.append(ex_)return all_querys# 进行分词
path = r'D:\LTR_REC\stopwords.txt'jieba.analyse.set_stop_words(path)stopwords = stopwordslist(path)  # 创建一个停用词列表query = ["我要看免费电影"]lists=["电影频道","免费电影","我要看免费电影","西游记","我要"]
query = cut_(query)
lists = cut_(lists)tfidf_vec = TfidfVectorizer(min_df=1, max_df=1.0, token_pattern='\\b\\w+\\b')
def count_sim(x, y):x_len = len(x)y_len = len(y)coss = np.zeros((x_len,y_len))# print(coss)for i in range(x_len):for j in range(y_len):corpus = [x[i], y[j]]# print(corpus)vectoerizer = tfidf_vec.fit_transform(corpus)# print(vectoerizer)cos1 = cosine_similarity(vectoerizer)[0]# print(cos1)coss[i][j] = cos1[1]return cossprint(query, lists)itemScoreDict = { num:i for num,i in enumerate(count_sim(query, lists)[0])} ##query与docs tfidf相关度
similarityMatrix = count_sim(lists, lists)   ## docs内容间tfidf相似度
print(itemScoreDict)
print(similarityMatrix)def MMR(itemScoreDict, similarityMatrix, lambdaConstant=0.5, topN=20):s, r = [], list(itemScoreDict.keys())while len(r) > 0:score = 0selectOne = Nonefor i in r:firstPart = itemScoreDict[i]secondPart = 0for j in s:sim2 = similarityMatrix[i][j]if sim2 > secondPart:secondPart = sim2equationScore = lambdaConstant * (firstPart - (1 - lambdaConstant) * secondPart)if equationScore > score:score = equationScoreselectOne = iif selectOne == None:selectOne = ir.remove(selectOne)s.append(selectOne)return (s, s[:topN])[topN > len(s)]print(MMR(itemScoreDict, similarityMatrix))

****优化版，只算一半的矩阵，docs间矩阵计算

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import jieba
import jieba.analyse
import numpy as npdef stopwordslist(path):stopwords = [line.strip() for line in open(path, encoding='UTF-8').readlines()]return stopwordsdef split_word(sentence_depart):# 输出结果为outstroutstr = ''# 去停用词for word in sentence_depart:if word not in stopwords:if word != '\t':outstr += wordoutstr += " "return outstrdef cut_(words):all_querys = []# qq1 = datas_all["{}".format(word)].unique()for ii in list(words):ex_ = split_word(jieba.cut(str(ii)))#     print("111",ii,ex_)all_querys.append(ex_)return all_querys# 进行分词
path = r'D:\LTR_REC\stopwords.txt'jieba.analyse.set_stop_words(path)stopwords = stopwordslist(path)  # 创建一个停用词列表query = ["西游记"]lists=['西游记','西游记','西游记','西游记','西游记','新西游记','西游记续集','西游记续集','西游记后传','新西游记','西游记手绘版','西游记少儿版','西游记红孩儿','西游记比丘国','天真派西游记','西游记之沙僧','西游记之唐僧','西游记的故事','西游记：女儿国','凯叔·西游记【贝塔】','西游记之白龙马','西游记之猪八戒','西游记之孙悟空','不一样的西游记','西游记故事儿歌','西游记之红孩儿','西游记 张掖寻踪','水木剧场西游记','西游记里的故事','西游记：张掖寻踪','西游记之再世妖王','名著导读之西游记','眼镜叔叔讲西游记','西游记之西梁女国','西游记之大圣归来','西游记之三件宝贝','西游记里的那些事','西游记之大闹天宫','你不知道的西游记','凯叔西游记全集音频','小戏骨西游记红孩儿','西游记之锁妖封魔塔','西游记精彩片段集锦','西游记中的那些事儿','百家讲坛 玄奘西游记','七彩童书坊：西游记音频','西游记之大闹天宫粤语','西游记篇500个汉字轻松学','紫微斗数之西游记—白龙马','紫微斗数评说西游记系列','西游记之再世妖王 动漫版','韩田鹿讲给青少年的西游记','紫微斗数评说西游记特辑1','西游记师徒四人是什么星座','西游记之孙悟空三打白骨精','西游记之大闹天宫环绕声版','西游记中不为人知的奇葩事','弹词选曲西游记·白虎岭遇妖','妖怪密码：解密神魔巅峰西游记','精编趣讲西游记让孩子开心学名著','百万孩子都在看的西游记精选故事集','西游记女儿情-李萍丨炫舞未来广场舞蹈','西游记之动物世界：妖怪原型竟然是这些动物']
query = cut_(query)
lists = cut_(lists)tfidf_vec = TfidfVectorizer(min_df=1, max_df=1.0, token_pattern='\\b\\w+\\b')
def count_sim(x, y):x_len = len(x)y_len = len(y)coss = np.zeros((x_len,y_len))# print(coss)for i in range(x_len):for j in range(i,y_len): ## 这样只算一个三角，上三角矩阵corpus = [x[i], y[j]]# print(corpus)vectoerizer = tfidf_vec.fit_transform(corpus)# print(vectoerizer)cos1 = cosine_similarity(vectoerizer)[0]# print(cos1)coss[i][j] = cos1[1]return cossprint(query, lists)itemScoreDict = { num:i for num,i in enumerate(count_sim(query, lists)[0])} ##query与docs tfidf相关度
similarityMatrix = count_sim(lists, lists)   ## docs内容间tfidf相似度
print(itemScoreDict)
# print(count_sim(query, lists))
print(similarityMatrix)
# assert similarityMatrix[2][3]==similarityMatrix[3][2]def MMR(itemScoreDict, similarityMatrix, lambdaConstant=0.5, topN=20):s, r = [], list(itemScoreDict.keys())while len(r) > 0:score = 0selectOne = Nonefor i in r:firstPart = itemScoreDict[i]secondPart = 0for j in s:### 三角矩阵,对称矩阵的最大特点就是a[m][n] = a[n][m]if j<i:sim2 = similarityMatrix[j][i]else:sim2 = similarityMatrix[i][j]if sim2 > secondPart:secondPart = sim2equationScore = lambdaConstant * (firstPart - (1 - lambdaConstant) * secondPart)if equationScore > score:score = equationScoreselectOne = iif selectOne == None:selectOne = ir.remove(selectOne)s.append(selectOne)return (s, s[:topN])[topN > len(s)]print(MMR(itemScoreDict, similarityMatrix))

MMR 排序多样化重排序算法相关推荐

Java并发编程的艺术(二)——重排序
当我们写一个单线程程序时,总以为计算机会一行行地运行代码,然而事实并非如此. 什么是重排序? 重排序指的是编译器.处理器在不改变程序执行结果的前提下,重新排列指令的执行顺序,以达到最佳的运行效率. 重 ...
Java内存模型深度解析：重排序 --转
原文地址:http://www.codeceo.com/article/java-memeory-2.html 数据依赖性如果两个操作访问同一个变量,且这两个操作中有一个为写操作,此时这两个操作之间 ...
java并发排序_Java并发（三）：重排序
在执行程序时为了提升性能,提升并行度,编译器和处理器经常会对指令作重排序.重排序分三种类型:java 编译器优化的重排序.编译器在不改变单线程程序语义的前提下,能够从新安排语句的执行顺序. 指令级并行 ...
java面试-Java并发编程(二)——重排序
当我们写一个单线程程序时,总以为计算机会一行行地运行代码,然而事实并非如此. 什么是重排序? 重排序指的是编译器.处理器在不改变程序执行结果的前提下,重新排列指令的执行顺序,以达到最佳的运行效率. 重 ...
java多线程并行执行命令_深入理解Java多线程与并发框（第④篇）——重排序、屏障指令、as-if-serial规则...
![](http://img.blog.itpub.net/blog/2020/03/24/906435fda570a5e3.png?x-oss-process=style/bb) # 一.重排序前 ...
什么是指令重排序？为什么要重排序？
什么是重排序假设我们写了一个 Java 程序,包含一系列的语句,我们会默认期望这些语句的实际运行顺序和写的代码顺序一致. 但实际上,编译器.JVM 或者 CPU 都有可能出于优化等目的,对于实际指令 ...
Java指令屏障_指令重排序和内存屏障
sap hana计算技术项目实战指南内存 61元 (需用券) 去购买 > 一.指令重排序指令重排序分为三种,分别为编译器优化重排序.指令级并行重排序.内存系统重排序.如图所示,后面两种为处理器 ...
【实践】端智能在大众点评搜索推荐重排序的应用实践
猜你喜欢 0.2022年人才市场洞察及薪酬指南1.如何搭建一套个性化推荐系统?2.[免费下载]2022年1月份热门报告3.全民K歌推荐系统算法.架构及后台实现4.微博推荐算法实践与机器学习平台演进5. ...
端智能在大众点评搜索重排序的应用实践
端智能,是指在移动端设备运行人工智能(AI)应用的技术.本文主要讲述大众点评搜索场景下,在端侧部署大规模深度学习模型进行搜索重排序任务的实践方案,包括端上特征工程.模型迭代思路,以及具体部署优化的过程 ...
Java之内存模型的基础、重排序、顺序一致性、volatile、锁、final
为什么80%的码农都做不了架构师?>>> 深入理解Java内存模型(一)--基础深入理解Java内存模型(二)--重排序深入理解Java内存模型(三)--顺序一致性深入理 ...

MMR 排序多样化重排序算法

使用tfidf得分来测试

MMR 排序多样化重排序算法相关推荐

最新文章

热门文章