一. 数据集的准备与预处理

1 . 收集dataset

(大量用户名--包含正常用户名与非法用户名)

包含两个txt文件 legal_name.txt ilegal_name.txt. 如下图所示

2. 用文件进行预处理

# Data sets
import os
import pandas as pdDATAPATH = "../dataset"POS = os.path.join(DATAPATH, "legal_name.txt")
POS_OUTPUT = os.path.join(DATAPATH, "legal_name.csv")NEG = os.path.join(DATAPATH, "ilegal_name.txt")
NEG_OUTPUT = os.path.join(DATAPATH, "ilegal_name.csv")def process_org_data(input_data, output_data, lable):reader = pd.read_csv(input_data, iterator=True)while True:try:train = reader.get_chunk(10000)train['username'] = train['username'].astype(str)train['username'] = map(lambda x: x.strip(), train['username'])train['length'] = train['username'].apply(len)... ...train['label'] = map(lambda x: lable, train['username'])train.to_csv(output_data, encoding='utf-8', mode='a', index=False, header=False)except StopIteration:print "Iteration is stopped."breakif __name__ == '__main__':process_org_data(POS, POS_OUTPUT, 1)process_org_data(NEG, NEG_OUTPUT, 0)

根据需求提取相应的特征, 输出成 csv 格式，包含特征列与label列

把合法用户dataset与非法用户dataset,合并打乱，切割成 train.csv 和 test.csv

pos_dataset = read_dataset(POS)
neg_dataset = read_dataset(NEG)
dataset = pd.concat([pos_dataset, neg_dataset])
dataset = dataset.sample(frac=1).reset_index(drop=True)train_data = dataset.loc[:200000, :]
test_data = dataset.loc[200000:, :]train_data.to_csv(os.path.join(DataPath, "train.csv"), index=False)
test_data.to_csv(os.path.join(DataPath, "test.csv"), index=False)

二. Keras 构建DNN模型进行训练与模型保存

import pandas as pd
import os
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import numpy as nptf.logging.set_verbosity(tf.logging.INFO)DataPath = "../dataset"TRAIN = os.path.join(DataPath, "train.csv")
TEST = os.path.join(DataPath, "test.csv")COLUMNS = ["username", ... , "label"]train_dataset = pd.read_csv(TRAIN, skipinitialspace=True, skiprows=1, names=COLUMNS)
test_dataset = pd.read_csv(TEST, skipinitialspace=True, skiprows=1, names=COLUMNS)for col in train_dataset.columns[1:]:train_dataset[col] = pd.to_numeric(train_dataset[col], errors='coerce')for col in test_dataset.columns[1:]:test_dataset[col] = pd.to_numeric(test_dataset[col], errors='coerce')X_train = train_dataset.iloc[:, range(1, 19)].values
y_train = train_dataset.iloc[:, 19].valuesX_test = test_dataset.iloc[:, range(1, 19)].values
y_test = test_dataset.iloc[:, 19].valuesdef build_model():############model = tf.keras.models.Sequential()model.add(tf.keras.layers.Dense(64, input_dim=18))# model.add(tf.keras.layers.BatchNormalization())model.add(tf.keras.layers.Activation('relu'))model.add(tf.keras.layers.Dense(32))# model.add(tf.keras.layers.BatchNormalization())model.add(tf.keras.layers.Activation('relu'))model.add(tf.keras.layers.Dense(16))# model.add(tf.keras.layers.BatchNormalization())model.add(tf.keras.layers.Activation('relu'))model.add(tf.keras.layers.Dense(1, activation='sigmoid'))return if __name__ == '__main__':model_file = './my_model.h5'if (os.path.isfile(model_file)):print('model file detected. Loading.')model = tf.keras.models.load_model(model_file)else:print('No model file detected.  Starting from scratch.')model = build_model()model.compile(loss='binary_crossentropy', optimizer="adam", metrics=['accuracy'])model.fit(X_train, y_train, batch_size=100, epochs=1, validation_data=(X_test, y_test))

模型输出为 my_model.h5 , 准确率百分之90%

三. 导出tensorflow-serving 模型, 运行在线预测服务

def save_model_for_production(model, version, path='prod_models'):tf.keras.backend.set_learning_phase(1)if not os.path.exists(path):os.mkdir(path)export_path = os.path.join(tf.compat.as_bytes(path),tf.compat.as_bytes(version))builder = tf.saved_model.builder.SavedModelBuilder(export_path)model_input = tf.saved_model.utils.build_tensor_info(model.input)model_output = tf.saved_model.utils.build_tensor_info(model.output)prediction_signature = (tf.saved_model.signature_def_utils.build_signature_def(inputs={'inputs': model_input},outputs={'output': model_output},method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME))with tf.keras.backend.get_session() as sess:builder.add_meta_graph_and_variables(sess=sess, tags=[tf.saved_model.tag_constants.SERVING],signature_def_map={'predict':prediction_signature,})builder.save()

导出为 tensorflow serving 模型

export_path = "tf-model"
save_model_for_production(model, "7", export_path)

运行在线预测服务（tensorflow 官方方法）

/serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=username --model_base_path=/data/model/tf-model

四. client通过grpc 调用预测服务

#!/usr/bin/env python
# encoding: utf-8  """
@version: v1.0
@author: zwqjoy
@contact: zwqjoy@163.com
@site: https://blog.csdn.net/zwqjoy
@file: client
@time: 2018/6/29 15:02
"""from grpc.beta import implementations
import tensorflow as tffrom tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2
import numpy as nptf.app.flags.DEFINE_string('server', '172.xxx.xxx.xxx:9000','PredictionService host:port')
tf.app.flags.DEFINE_string('username', 'demo_user','传入一个username')
FLAGS = tf.app.flags.FLAGSdef nametovec(username):username = username.astype(str)length = len(username)... ...return np.array([length, ...])def main(_):host, port = FLAGS.server.split(':')channel = implementations.insecure_channel(host, int(port))stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)# Send request# See prediction_service.proto for gRPC request/response details.data = nametovec(FLAGS.username)data = data.astype(np.float32)request = predict_pb2.PredictRequest()request.model_spec.name = 'username'  # 这个name跟tensorflow_model_server  --model_name="username" 对应request.model_spec.signature_name = 'predict'  # 这个signature_name  跟signature_def_map 对应request.inputs['inputs'].CopyFrom(tf.contrib.util.make_tensor_proto(data, shape=(1, 18)))result = stub.Predict(request, 10.0)  # 10 secs timeoutprint(result)if __name__ == '__main__':tf.app.run()

Keras 构建DNN 对用户名检测判断是否为非法用户名（从数据预处理到模型在线预测）相关推荐

Keras 构建DNN 对用户名检测判断是否为非法用户名（从数据预处理到模型在线预测）...
一. 数据集的准备与预处理 1 . 收集dataset (大量用户名--包含正常用户名与非法用户名) 包含两个txt文件 legal_name.txt ilegal_name.txt. 如下图所 ...
R语言机器学习Caret包（Caret包是分类和回归训练的简称）、数据划分、数据预处理、模型构建、模型调优、模型评估、多模型对比、模型预测推理
R语言机器学习Caret包(Caret包是分类和回归训练的简称).数据划分.数据预处理.模型构建.模型调优.模型评估.多模型对比.模型预测推理目录
基于Django构建在线文本分类预测系统：SVM模型在线预测与部署基于 Django 3.2 框架
文章大纲算法端与后端环境搭建前端构建 Ajax XHR 创建网页与 Django项目创建网页前端 -- django 项目创建与启动前端 -- 创建 django app 应用 postma ...
TensorFlow2.0(四)--Keras构建深度神经网络(DNN)
Keras构建深度神经网络(DNN) 1. 深度神经网络简介 2. Kerase搭建DNN模型 2.1 导入相应的库 2.2 数据加载与归一化 2.3 网络模型的构建 2.4 批归一化,dropout ...
[Python图像识别] 五十.Keras构建AlexNet和CNN实现自定义数据集分类详解
该系列文章是讲解Python OpenCV图像处理知识,前期主要讲解图像入门.OpenCV基础用法,中期讲解图像处理的各种算法,包括图像锐化算子.图像增强技术.图像分割等,后期结合深度学习研究图像识别 ...
网络安全公司奇安信集团是如何基于 Flink 构建 CEP 引擎实时检测网络攻击【未来不可忽视的网络安全】
摘要: 奇安信集团作为一家网络安全公司是如何基于 Flink 构建 CEP 引擎实时检测网络攻击?其中面临的挑战以及宝贵的实践经验有哪些?本文主要内容分为以下四个方面: 背景及现状技术架构产品及运 ...
keras构建卷积神经网络_在Keras中构建，加载和保存卷积神经网络
keras构建卷积神经网络 This article is aimed at people who want to learn or review how to build a basic Convo ...
keras构建卷积神经网络_通过此简单教程学习在网络上构建卷积神经网络
keras构建卷积神经网络 by John David Chibuk 约翰·大卫·奇布克(John David Chibuk) 通过此简单教程学习在网络上构建卷积神经网络 (Learn to buil ...
TensorFlow2.0(五)--Keras构建Wide Deep模型
Keras构建Wide & Deep模型 1. Wide & Deep模型简介 2. Keras实现Wide & Deep模型 2.1 导入相应的库 2.2 数据集加载与处理 ...

Keras 构建DNN 对用户名检测判断是否为非法用户名（从数据预处理到模型在线预测）