机器学习项目-金融反欺诈模型

#coding:utf-8
#先读取数据，用pandas 读入表，再进行分析和处理import pandas as pd
import numpy as npdf = pd.read_csv('../data/LoanStats3a.csv', skiprows = 1, low_memory = False)#......................
# print(df.info())
#大约是145列，对其进行特征处理，凭借经验可以降维操作
#删除id 和 member_id
df.drop('id', 1, inplace = True)
df.drop('member_id', 1, inplace = True)
#删除term这一列的字符串
df.term.replace(to_replace= '[^0-9]+', value = '', inplace = True, regex = True)# 60 months......................
df.term.astype(float)
df.int_rate.replace('%', '', inplace = True)#10.65%
df.drop(['sub_grade','emp_title'], 1, inplace = True)
#替换掉工作年限中的n/a,并进行正则匹配，仅保留年份
df.emp_length.replace('n/a', np.nan , inplace = True)
df.emp_length.replace(to_replace='[^0-9]+', value = '',\inplace = True, regex = True)
#删除空值的，全是0的，全是同一个数字的
df.dropna(1, how = 'all', inplace = True) #按照列删除
df.dropna(0, how = 'all', inplace = True) #按照行删除
df.drop(['mths_since_last_record','next_pymnt_d','debt_settlement_flag_date','settlement_status',\'settlement_date','settlement_amount','settlement_percentage','settlement_term',],\1, inplace = True)
#统计并删除列分类很少，占比很少的列
#先统计float类型
for col in df.select_dtypes(include = ['float']).columns:# print('col {} has {}'.format(col, len(df[col].unique())))
#删除float中分类与实例个数不在一个数量级上的
df.drop(['collections_12_mths_ex_med','policy_code','acc_now_delinq','chargeoff_within_12_mths',\'delinq_amnt','pub_rec_bankruptcies','tax_liens','total_acc','out_prncp','out_prncp_inv',\'delinq_2yrs','inq_last_6mths','mths_since_last_delinq','open_acc','pub_rec',],\1, inplace = True)
#查看object类型中占比很少的数据
print('===========================================================')
for col in df.select_dtypes(include = ['object']).columns:# print('col {} has {}'.format(col, len(df[col].unique())))
#删除object类型数据
df.drop(['term','int_rate','grade','emp_length','home_ownership','verification_status','issue_d','desc',\'pymnt_plan','purpose','zip_code','addr_state','earliest_cr_line','initial_list_status',\'last_pymnt_d','last_credit_pull_d','application_type','hardship_flag','disbursement_method','debt_settlement_flag',],\1, inplace = True)
#查看y状态，并删除y为空的实例(按行删除)
df.loan_status.replace('Fully Paid', int(1), inplace = True)
df.loan_status.replace('Charged Off', int(0), inplace = True)
df.loan_status.replace('Does not meet the credit policy. Status:Fully Paid', np.nan, inplace = True)
df.loan_status.replace('Does not meet the credit policy. Status:Charged Off', np.nan, inplace = True)
df.dropna(subset = ['loan_status'],inplace = True)
df.drop('title', 1, inplace = True)
df.fillna(0, inplace = True)
df.fillna(0.0, inplace = True)
#删除相关性太高的列
df.drop(['loan_amnt','funded_amnt','total_pymnt'], 1 ,inplace = True)
#正交化处理，删除相关性高的列
cor = df.corr()#pandas 求协方差矩阵公式
cor.loc[:,:] = np.tril(cor, k=-1)
cor = cor.stack() #结构重建
print(cor[(cor>0.55) | (cor < -0.55)])
# 进行哑变量处理
df = pd.get_dummies(df)
df.to_csv('../data/feature02.csv')

#coding:utf-8
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model.logistic import LogisticRegression
from sklearn import metrics
from sklearn.ensemble.forest import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn import svm
import time
from sklearn.model_selection import GridSearchCV
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np#先用logistic进行训练和预测
df = pd.read_csv('../data/feature02.csv')
# print(df.head(5))
Y = df.loan_status
X = df.drop('loan_status', 1, inplace = False)
# print(X.shape)
# print(X.columns)
x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size = 0.3, random_state = 0)
#=========================================================================================
lr = LogisticRegression()
start= time.time()
lr.fit(x_train, y_train)
train_predict = lr.predict(x_train)
train_f1 = metrics.f1_score(train_predict, y_train)
train_acc = metrics.accuracy_score(train_predict, y_train)
train_rec = metrics.recall_score(train_predict, y_train)
print("逻辑回归模型上的效果入下：")
print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % train_rec)
test_predict = lr.predict(x_test)
test_f1 = metrics.f1_score(test_predict, y_test)
test_acc = metrics.accuracy_score(test_predict, y_test)
test_rec = metrics.recall_score(test_predict, y_test)
print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % test_rec)
end = time.time()
print(end-start)
#=================================================================
print("随机森林效果如下" + "=" * 30)
rf = RandomForestClassifier()
start = time.time()
rf.fit(x_train, y_train)
train_predict = rf.predict(x_train)
train_f1 = metrics.f1_score(train_predict, y_train)
train_acc = metrics.accuracy_score(train_predict, y_train)
train_rec = metrics.recall_score(train_predict, y_train)
print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % train_rec)
test_predict = rf.predict(x_test)
test_f1 = metrics.f1_score(test_predict, y_test)
test_acc = metrics.accuracy_score(test_predict, y_test)
test_rec = metrics.recall_score(test_predict, y_test)
print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % test_rec)
end = time.time()
print(end - start)
#======================================================================
print("GBDT上效果如下" + "=" * 30)
gb = GradientBoostingClassifier()
start = time.time()
gb.fit(x_train, y_train)
train_predict = gb.predict(x_train)
train_f1 = metrics.f1_score(train_predict, y_train)
train_acc = metrics.accuracy_score(train_predict, y_train)
train_rec = metrics.recall_score(train_predict, y_train)
print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % train_rec)
test_predict = gb.predict(x_test)
test_f1 = metrics.f1_score(test_predict, y_test)
test_acc = metrics.accuracy_score(test_predict, y_test)
test_rec = metrics.recall_score(test_predict, y_test)
print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % test_rec)
end = time.time()
print(end-start)# print("支持向量机的效果如下" + "=" * 30)
# sv = svm.SVC(kernel = 'linear') #C = 1, probability = True, decision_function_shape = 'ovo', random_state = 0
# start = time.time()
# sv.fit(x_train, y_train)
# train_predict = sv.predict(x_train)
# train_f1 = metrics.f1_score(train_predict, y_train)
# train_acc = metrics.accuracy_score(train_predict, y_train)
# train_rec = metrics.recall_score(train_predict, y_train)
# print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
# print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
# print("在训练集上的查全率的值为%.4f" % train_rec)
# test_predict = sv.predict(x_test)
# test_f1 = metrics.f1_score(test_predict, y_test)
# test_acc = metrics.accuracy_score(test_predict, y_test)
# test_rec = metrics.recall_score(test_predict, y_test)
# print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
# print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
# print("在训练集上的查全率的值为%.4f" % test_rec)
# end = time.time()
# print(end - start)
#
# start = time.time()
# parameters ={#     'kernel':['linear','sigmoid','poly'],
#     'C':[0.01, 1],
#     'probability':[True, False]
#     }
# clf = GridSearchCV(svm.SVC(random_state = 0), param_grid = parameters, cv = 5)
# clf.fit(x_train, y_train)
# print('最优参数是：',end=' ')
# print(clf.best_params_)
# print('最优模型准确率是：', end = ' ')
# print(clf.best_score_)
# end = time.time()
# print(end-start)feature_importance = rf.feature_importances_#度量特征权重的接口
# print(feature_importance)
# print(feature_importance.max())
feature_importance = 100.0*(feature_importance/feature_importance.max())
index = np.argsort(feature_importance)[-10:]
plt.barh(np.arange(10), feature_importance[index], color = 'dodgerblue', alpha = 0.4)
print(np.array(X.columns)[index])
plt.yticks(np.arange(10+0.25), np.array(X.columns)[index])
plt.xlabel('Relative importance')
plt.title('Top 10 Importance Variable')
plt.show()

机器学习项目-金融反欺诈模型相关推荐

金融反欺诈模型----项目实战--机器学习
机器学习:从源数据清洗到特征工程建立谈金融反欺诈模型训练本文旨在通过一个完整的实战例子,演示从源数据清洗到特征工程建立,再到模型训练,以及模型验证和评估的一个机器学习的完整流程.由于初识机器学习,会 ...
项目：金融行业反欺诈模型
当今以互联网.移动终端等为代表的技术力量正深刻地影响着金融支付市场,信息化.网络化.无线终端等技术的应用,使金融机构特别是银行业的经营发生了天翻地覆的变化,传统的银行柜台和网点业务,正渐渐被电子化交易 ...
AI：人工智能实践六大场景(金融信用违约、反欺诈模型、客户偏好洞察、智能推荐、精准营销、客户流失管理)及其对应常用机器学习算法经验总结(不断更新)
AI:人工智能实践六大场景(金融信用违约.反欺诈模型.客户偏好洞察.智能推荐.精准营销.客户流失管理)及其对应常用机器学习算法经验总结(不断更新) 目录
【待继续研究】解析机器学习技术在反欺诈领域的应用
反欺诈简单说,就是:根据借款人提供的信息,查找多方面资料,进行不同属性的比对,从而发现"羊群中的狼".这种工作复杂而枯燥,为了识别团伙欺诈,往往需要收集.整理.分析各种维度的数据, ...
宜人贷何林海：社交图谱在金融反欺诈产品的应用
在今年的「QingCloud Insight 2017」上,麦思博(msup)组织了主题为"AI落地实践"的专场论坛.宜人贷数据产品经理何林海在本场带来了题为<构建基于社 ...
金融反欺诈和金融构建信用评分或者金融预测特征抽取案例
金融反欺诈和金融构建信用评分或者金融预测特征抽取案例之前在微信朋友圈读到过这样一篇文章,施一公:"我国的最大危机,是所有精英都想干金融!",当然,笔者是非常赞同的,因为和钱距离越 ...
顶象深度画像亮相GMIC，用AI提升金融反欺诈
顶象深度画像亮相GMIC,用AI提升金融反欺诈凤凰新闻04-2819:30 原标题:顶象深度画像亮相GMIC,用AI提升金融反欺诈 "最近Facebook信息泄露问题,再次让隐私保护成为焦 ...
金融反欺诈-交易基础介绍 - 阿里巴巴云誉【转载】
转载{http://www.vuln.cn/7136} 0×00 前言一.简介如今,互联网金融比较火热,金融欺诈也变得非常普遍,金融反欺诈也应运而生.本文将主要介绍下金融交易中的一些基本内容,并简 ...
【采用】互联网金融反欺诈体系构建及典型应用案例
一.互联网反欺诈体系的构建存在着以下三个原则: (准)实时性:考虑到用户体验,互联网反欺诈体系必须能够在非常短的时间内对欺诈行为进行认定,并给出判断.对于注册.登陆.支付等一些场景,必须能够在用户无感 ...
原理+代码｜手把手教你使用Python实战反欺诈模型
三本点击上方"早起Python",关注并"星标" 每日接收Python干货! 本文含 6192 字,15 图表截屏建议阅读 20分钟本文将基于不平衡数据,使 ...

机器学习项目-金融反欺诈模型

机器学习项目-金融反欺诈模型相关推荐

最新文章

热门文章