统计学习方法第六章作业：逻辑斯谛梯度下降法、最大熵模型 IIS / DFP 算法代码实现

逻辑斯谛梯度下降法

import numpy as np
import matplotlib.pyplot as pltclass logist:def __init__(self,a=1,c=None,max_iter=9999):self.w = Noneself.a = aself.c = cself.max_iter = max_iterdef sigmoid(self,x):x = np.array(x)return np.exp(x.dot(self.w))/(1+np.exp(x.dot(self.w)))def expand_b_for_x(self,x):x = np.array(x)if len(x.shape)==1:x = np.concatenate((x,np.array([1])))else:x = np.concatenate((x,np.ones(len(x)).reshape(len(x), 1)),axis=1)return xdef fit(self,x,y):x = self.expand_b_for_x(x)self.w = np.zeros(len(x[0]))grade = [999,999,999]iter = 0while (np.abs(grade) * self.a).any() > self.c and iter <= self.max_iter:grade = np.sum(x * (y - self.sigmoid(x)).reshape(len(y), 1), axis=0)deta_w = grade * self.aself.w += deta_witer += 1returndef predict(self,x):x = self.expand_b_for_x(x)return self.sigmoid(x)def main():x = [[3,3],[4,3],[1,1],[1,2],[6,2]]y = [1,1,0,1,0]logist_classer = logist(a=1,c=0.6,max_iter=20)logist_classer.fit(x,y)print(logist_classer.w)print(logist_classer.predict(x))positive_ =  np.array(x)[np.array(y) == 1]negetive_ = np.array(x)[np.array(y) == 0]plt.scatter([k[0] for k in positive_],[k[1] for k in positive_],c='r',label='1')plt.scatter([k[0] for k in negetive_], [k[1] for k in negetive_],c='b',label='0')x_ = np.arange(0, 10, 0.1)y_ = -(logist_classer.w[0] * x_ + logist_classer.w[-1]) / logist_classer.w[1]plt.plot(x_, y_)plt.show()if __name__ == '__main__':main()########result#####################
/usr/bin/python3 /Users/zhengyanzhao/PycharmProjects/tongjixuexi/logist_Gradient_Descent.py
[-6.48376131 11.44111606 -6.43331211]
[9.99783727e-01 8.75989820e-01 1.86038812e-01 9.99952993e-011.77216694e-10]

最大熵模型 IIS / DFP 算法代码实现

import numpy as np
import pandas as pd
import math
from collections import defaultdict
from sklearn.metrics import accuracy_scoredef load_data(path):def transform(row):return list((row[1:] > 128).apply(int))data = pd.read_csv(path,header=None)x_list = list(data.apply(transform,axis=1))y_list = list(data[0])return x_list,y_listclass Maximum_Entropy:def __init__(self,X,Y, way='IIS'):self.X = Xself.Y = Yself.feature_num = np.array(X).shape[-1]self.M = 10000self.sample_num = len(X)self.f_num = 0self.label_list = list(set(Y))self.label_num = len(set(Y))self.w = Noneself.create_fixy()self.create_index()self.EP_fi()self.way = waydef create_fixy(self):self.fi_dict = [defaultdict(int) for i in range(self.feature_num)]for i in range(self.feature_num):for j in range(self.sample_num):x = self.X[j][i]y = self.Y[j]self.fi_dict[i][(x,y)] += 1def create_index(self):self.fixy_id = [{} for i in range(self.feature_num)]self.id_xy = {}index = 0for i in range(self.feature_num):for x,y in self.fi_dict[i]:self.fixy_id[i][(x,y)] = indexself.id_xy[index] = (x,y)index += 1self.f_num = indexself.w = [0] * self.f_numdef EP_fi(self):self.EP_fi_list = [0] * self.f_numfor i in range(self.feature_num):for x, y in self.fi_dict[i]:self.EP_fi_list[self.fixy_id[i][(x,y)]] = self.fi_dict[i][(x, y)]/self.sample_numdef EPfi(self):self.EPfi_list = [0] * self.f_numfor i in range(self.sample_num):x = self.X[i]for j in self.label_list:P_Y_X = self.caculate_P_Y_X(x,j)for m in range(self.feature_num):if (x[m],j) in self.fixy_id[m]:index = self.fixy_id[m][(x[m],j)]self.EPfi_list[index] += P_Y_X * 1/self.sample_numdef caculate_P_Y_X(self,X,y):fenzi = 0fenmu = 0for j in self.label_list:sum_1 = 0for i in range(self.feature_num):x = X[i]if (x, j) in self.fi_dict[i]:index = self.fixy_id[i][(x, j)]sum_1 += self.w[index]if j == y:fenzi = math.exp(sum_1)fenmu += math.exp(sum_1)return fenzi/fenmudef updata_w(self,max_iter):for i in range(max_iter):print(i)self.EPfi()self.w += [1/self.M * math.log(self.EP_fi_list[i]/self.EPfi_list[i]) for i in range(self.f_num)]#-----------DFP-------------------------------def compute_gk(self):self.EPfi()return np.array(self.EPfi_list) - np.array(self.EP_fi_list)def compute_fw(self):left_part = 0for i in range(self.sample_num):fenmu = 0x_ = self.X[i]for j in self.label_list:sum_1 = 0for i in range(self.feature_num):x = x_[i]if (x, j) in self.fi_dict[i]:index = self.fixy_id[i][(x, j)]sum_1 += self.w[index]fenmu += math.exp(sum_1)left_part += math.log(fenmu) * 1/self.sample_numright_part = np.sum(np.array(self.EP_fi_list) * np.array(self.w))return left_part - right_partdef DFP(self,eplison=1,max_iter=20):self.GK = np.array(np.eye(self.f_num),dtype=float)gk = self.compute_gk()if np.linalg.norm(gk,ord=2) < eplison:print(self.w)returnfor i in range(max_iter):pk = -self.GK.dot(gk)temp_w = self.wfw_list = []rate = 0.005for k in range(20):#线性搜索20次self.w = pk * rate * k + temp_wfw = self.compute_fw()fw_list.append(fw)min_index = fw_list.index(min(fw_list))print('min_index is '+ str(min_index))deta_w = pk * rate * min_indexself.w = deta_w + temp_wgk_ = self.compute_gk()if np.linalg.norm(gk_,ord=2) < eplison:print(self.w)breakyk = (gk_ - gk).reshape(1,self.f_num)si_k = deta_w.reshape(1,self.f_num)gk = gk_PK = si_k.dot(si_k.T) / si_k.T.dot(yk)QK = self.GK.dot(yk).dot(yk.T).dot(self.GK)/yk.T.dot(self.GK).dot(yk)self.GK = self.GK + PK -QK#-------------------------------------def model_fit(self,max_iter):if self.way == 'IIS':self.updata_w(max_iter=max_iter)if self.way == 'DFP':self.DFP(max_iter=max_iter)def predict(self, X):result = []for x in X:result_x = []for y in self.label_list:p_y = self.caculate_P_Y_X(x, y)result_x.append(p_y)result.append(result_x)result_y = np.argmax(result, -1)return result_ydef main():path_train = './mnist_train.csv'path_test = './mnist_test.csv'x_train,y_train = load_data(path_train)x_test, y_test = load_data(path_test)Maximum_Entropy_ = Maximum_Entropy(x_train[:1000],y_train[:1000],way='DFP')Maximum_Entropy_.model_fit(10)y_pred = Maximum_Entropy_.predict(x_test[:10])print(accuracy_score(y_test[:10], y_pred))if __name__ == '__main__':main()#########result######################
#略

IIS 算法思路参考 https://www.pkudodo.com/2018/12/05/1-7/

统计学习方法第六章作业：逻辑斯谛梯度下降法、最大熵模型 IIS / DFP 算法代码实现相关推荐

统计学习方法第七章作业：SVM非线性支持向量机之SMO序列最小优化算法代码实现
SMO序列最小优化算法 import numpy as np import math from sklearn.metrics import accuracy_score from sklearn.m ...
学习笔记——逻辑斯蒂回归与最大熵模型
逻辑斯蒂回归(logistic regression)是统计学习中的经典分类方法.最大熵是概率模型学习的一个准则,将其推广到分类问题得到最大熵模型(maximum entropy).都属于对数线性模型 ...
统计学习方法读书笔记（六）-逻辑斯蒂回归与最大熵模型（迭代尺度法（IIS））
全部笔记的汇总贴:统计学习方法读书笔记汇总贴逻辑斯谛回归 (logistic regression )是统计学习中的经典分类方法.最大熵是概率模型学习的一个准则,将其推广到分类问题得到最大熵模型(m ...
最大熵阈值python_第六章-逻辑斯蒂回归和最大熵模型
逻辑斯谛回归是统计学习中的经典分类方法,和最大熵模型相比,具有以下的共同点和区别:共同点都属于概率模型,该模型要寻找的是给定一个x,得到输出变量Y的概率分布P(Y|x),如果是二分类,Y取值为0或1, ...
统计学习方法第四章朴素贝叶斯法-李航
第4章朴素贝叶斯法朴素贝叶斯 (naive Bayes) 法是基于贝叶斯定理与特征条件独立假设的分类方法.对于给定的训练数据集,首先基于特征条件独立假设学习输入/输出的联合概率分布:然后基于此模型 ...
统计学习方法第十一章作业：随机条件场—概率计算问题、IIS/GD学习算法、维特比预测算法代码实现
随机条件场-概率计算问题.IIS/GD学习算法.维特比预测算法这一章的算法不是很好写,整整研究了好几天,代码还是有点小问题,仅供参考. 用的是书上定义的特征函数. import numpy as n ...
统计学习方法第二十一章作业：PageRank迭代算法、幂法、代数算法代码实现
PageRank迭代算法.幂法.代数算法 import numpy as npclass PageRank:def __init__(self,M,D=0.85):self.M = np.array( ...
统计学习方法第十七章作业：LSA潜在语义分析算法代码实现
LSA潜在语义分析算法 import numpy as np import jieba import collectionsclass LSA:def __init__(self,text_list) ...
统计学习方法第五章作业：ID3/C4.5算法分类决策树、平方误差二叉回归树代码实现
ID3/C4.5算法分类决策树 import numpy as np import math class Node:def __init__(self,feature_index=None,value ...

统计学习方法第六章作业：逻辑斯谛梯度下降法、最大熵模型 IIS / DFP 算法代码实现

逻辑斯谛梯度下降法

最大熵模型 IIS / DFP 算法代码实现

统计学习方法第六章作业：逻辑斯谛梯度下降法、最大熵模型 IIS / DFP 算法代码实现相关推荐

最新文章

热门文章