#机器学习模型选择与参数调优

#三种集成学习算法-GBDT/XGBoost/lightGBM

#1-1 GBDT算法：梯度决策树，加强型模型，构建多个决策树进行合并

import numpy as np

import matplotlib.pyplot as plt

from sklearn import ensemble

from sklearn import datasets

from sklearn.model_selection import train_test_split

from sklearn.metrics import mean_squared_error

'''

#导入数据集

bost=datasets.load_boston()

x,y=bost.data,bost.target

x_train,x_test,y_train,y_test=train_test_split(x,y,random_state=666)

#构建模型

params={"n_estimators":800,"max_depth":4,"min_samples_split":2,"learning_rate":0.01,"loss":"ls"}

clf=ensemble.GradientBoostingRegressor(**params)

clf.fit(x_train,y_train)

y_pre=clf.predict(x_test)

mse=mean_squared_error(y_test,y_pre)

print("MSE:",mse)

#绘制测试误差-每一次所输出的迭代误差

test_score=np.zeros((params["n_estimators"],),dtype=np.float64)

for i ,y_pre in enumerate(clf.staged_predict(x_test)):

test_score[i]=clf.loss_(y_test,y_pre)

plt.figure(figsize=(12,6))

plt.subplot(1,2,1)

plt.title("Deviance")

plt.plot(np.arange(params["n_estimators"])+1,clf.train_score_,"b-",label="Training set deviance")

plt.plot(np.arange(params["n_estimators"])+1,test_score,"r-",label="Test set deviance")

plt.legend(loc="upper right")

plt.xlabel("Boostng Iterations")

plt.ylabel("Deviance")

#绘制特征重要性图

feature_importance=clf.feature_importances_

feature_importance=100.0*(feature_importance/feature_importance.max())

sort_index=np.argsort(feature_importance)

pos=np.arange(sort_index.shape[0])+0.5

plt.subplot(1,2,2)

plt.barh(pos,feature_importance[sort_index],align="center")

plt.yticks(pos,bost.feature_names[sort_index])

plt.xlabel("Relative Importance")

plt.title("Veriable Importance")

plt.show()

#1-2 XGboost扩展的GBDT

import xgboost as xgb

data=np.random.rand(100000,10)

label=np.random.randint(2,size=100000)

dtrain=xgb.DMatrix(data,label=label,missing=-999.0)

data2=np.random.rand(5000,10)

label2=np.random.randint(2,size=5000)

dtest=xgb.DMatrix(data2,label=label2,missing=-999.0)

params={"bst:max_depth":2,"bst:eta":1,"silent":1,"objective":"binary:logistic"}

params["nthread"]=4

params["eval_metric"]="auc"

evallist=[(dtrain,"train"),(dtest,"eval")] #监控效果

num_round=10 #训练迭代的次数

bst=xgb.train(params,dtrain,num_round,evallist)

#设置一种早停，输出最好的效果

bst=xgb.train(params,dtrain,num_round,evallist,early_stopping_rounds=10)

#1-3 lightGBM 轻量级梯度提升机

import lightgbm as lgb

data=np.random.rand(100000,10)

label=np.random.randint(2,size=100000)

train=lgb.Dataset(data,label=label)

data2=np.random.rand(5000,10)

label2=np.random.randint(2,size=5000)

test=lgb.Dataset(data2,label=label2)

params={"num_leaves":31,"num_trees":100,"objective":"binary","metrics":"binary_error"}

num_round=10 #10论训练

bst=lgb.train(params,train,num_round,valid_sets=[test])

#交叉验证

num_round=10

params={"num_leaves":50,"num_trees":100,"objective":"binary"}

print(lgb.cv(params,train,num_round,nfold=5))

bst=lgb.train(params,train,20,valid_sets=test,early_stopping_rounds=10)

'''

#招聘数据的实际项目需要

#2-1 GBDT算法进行训练和预测

import pandas as pd

import numpy as np

df=pd.read_csv("D:\Byrbt2018\Study\Python机器学习全流程项目实战精讲\配套课件\第七讲 机器学习建模\lagou_featured.csv",encoding="gbk")

print(df.shape)

pd.options.display.max_columns=999 #设置最大的展示列数为999

print(df.head())

#首先对salary的目标回归变量进行直方图展示

import matplotlib.pyplot as plt

plt.hist(df["salary"])

plt.show()

#将表格数据转化为目标向量和训练特征矩阵

x=df.drop(["salary"],axis=1).values

y=df["salary"].values.reshape(-1,1)

print(x.shape,y.shape)

from sklearn.model_selection import train_test_split

x_train,x_test,y_train,y_test=train_test_split(x,y,random_state=666)

print(x_train.shape,x_test.shape,y_train.shape,y_test.shape)

from sklearn.ensemble import GradientBoostingRegressor

clf=GradientBoostingRegressor(n_estimators=100,max_depth=5)

clf.fit(x_train,y_train)

from sklearn.metrics import mean_squared_error,max_error,mean_absolute_error

y_pre=clf.predict(x_test)

print(np.sqrt(mean_squared_error(y_test,y_pre)))

print(np.sqrt(mean_squared_error(y_test,y_pre)))

print(np.sqrt(mean_absolute_error(y_test,y_pre)))

print(np.sqrt(max_error(y_test,y_pre)))

print(clf.score(x_test,y_test))

plt.plot(y_pre)

plt.plot(y_test)

plt.legend(["y_pre","y_test"])

plt.show()

#对目标回归变量进行对数化处理，消除有偏数据的影响

from sklearn.model_selection import train_test_split

x_train,x_test,y_train,y_test=train_test_split(x,np.log(y),random_state=666)

print(x_train.shape,x_test.shape,y_train.shape,y_test.shape)

from sklearn.ensemble import GradientBoostingRegressor

clf=GradientBoostingRegressor(n_estimators=100,max_depth=5)

clf.fit(x_train,y_train)

from sklearn.metrics import mean_squared_error,max_error,mean_absolute_error

y_pre=clf.predict(x_test)

print(np.sqrt(mean_squared_error(np.exp(y_test),np.exp(y_pre))))

print(np.sqrt(mean_absolute_error(np.exp(y_test),np.exp(y_pre))))

print(np.sqrt(max_error(np.exp(y_test),np.exp(y_pre))))

print(clf.score(x_test,y_test))

print()

plt.plot(np.exp(y_pre))

plt.plot(np.exp(y_test))

plt.legend(["y_pre","y_test"])

plt.show()

#2-2 XGBoost模型训练和预测

from sklearn.model_selection import KFold #五划分的交叉验证的方式模块

import xgboost as xgb

from sklearn.metrics import mean_squared_error

import time

kf=KFold(n_splits=5,random_state=123,shuffle=True)

def evalereor(pre,train):

labels=train.getlabel()

return "mse",mean_squared_error(np.exp(pre),np.exp(labels))

y=np.log(y)

valid_pre=np.zeros((330,5))

time_start=time.time()

for i,(train_ind,valid_ind) in enumerate(kf.split(x)):

print("FOLD",i+1,"out of",5)

x_train,y_train=x[train_ind],y[train_ind]

x_valid,y_valid=x[valid_ind],y[valid_ind]

xgb_params={"eta":0.01,"max_depth":6,"subsample":0.9,"colsample_bytree":0.9,"objective":"reg:linear","seed":99,"eval_metgric":"rmse","silent":True}

d_train=xgb.DMatrix(x_train,y_train)

d_valid=xgb.DMatrix(x_valid,y_valid)

watchlist=[(d_train,"train"),(d_valid,"valid")] #监控效果

model=xgb.train(

xgb_params,

d_train,

2000,

watchlist,

verbose_eval=100,

#feval=evalereor,

early_stopping_rounds=1000

)

# valid_pre[:,i]=np.exp(model.predict(d_valid))

print("cv training time{} second".format(time.time()-time_start))

#网格搜索的方式

import xgboost as xgb

xg_train=xgb.DMatrix(x,y)

params={"eta": 0.01, "max_depth": 6, "subsample": 0.9, "colsample_bytree": 0.9, "objective": "reg:linear",

"seed": 99, "eval_metgric": "rmse", "silent": True}

cv=xgb.cv(params,xg_train,1000,nfold=5,early_stopping_rounds=800,verbose_eval=100)

print(cv)

#2-3 lightGBM算法模型

import time

import lightgbm as lgb

from sklearn.model_selection import KFold

from sklearn.metrics import mean_squared_error

from lightgbm import LGBMRegressor

x=df.drop(["salary"],axis=1).values

y=df["salary"].values

y=np.log(y)

def evalerror(pre,train):

labels=train.getlabel()

return "mse",mean_squared_error(np.exp(pre),np.exp(labels))

params={

"learning_rate":0.01,

"boosting_type":"gbdt",

"objective": "regression",

"metric":"mse",

"sub_feature":0.7,

"num_leaves":17,

"colsample_bytree":0.7,

"feature_fraction":0.7,

"min_data":100,

"min_hessian":1,

"verbose":-1

}

print("begin cv 5-fold training...")

scores=[]

time_start=time.time()

kf=KFold(n_splits=5,shuffle=True,random_state=27)

for i,(train_ind,valid_ind) in enumerate(kf.split(x)):

print("FOLD",i+1,"out of",5)

x_train,y_train=x[train_ind],y[train_ind]

x_valid,y_valid=x[valid_ind],y[valid_ind]

d_train=lgb.Dataset(x_train,y_train)

d_valid=lgb.Dataset(x_valid,y_valid)

model=lgb.train(

params,

d_train,

num_boost_round=2000,

valid_sets=d_valid,

verbose_eval=200,

#feval=evalerror(),

early_stopping_rounds=1000)

#valid_pre[:,i]=np.exp(model.predict(d_valid))

print("cv training time{} second".format(time.time()-time_start))