1、环境说明

服务器角色：

192.168.50.211         kafka+zookeeper

192.168.50.212          kafka+zookeeper

192.168.50.213         kafka+zookeeper

192.168.50.214         nginx filebeat logstash

192.168.50.215        elasticsearch kibana logstash

软件版本：

kafka_2.12-0.10.2.1.tgz

zookeeper-3.4.10.tar.gz

filebeat-5.3.2-linux-x86_64.tar.gz

kibana-5.3.2-linux-x86_64.tar.gz

logstash-5.3.2.tar.gz

elasticsearch-5.3.2.tar.gz

x-pack-5.3.2.zip

安装es的系统的系统版本：

# uname -a

Linux ansibleer 2.6.32-431.el6.x86_64 #1 SMP Fri Nov 22 03:15:09 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux

# cat /etc/issue

CentOS release 6.5 (Final)

Kernel \r on an \m

# uname -r

2.6.32-431.el6.x86_64

2、es安装配置

2.1、

内核升级

5以上版本的es需要部署在linux内核的版本3.0以上的系统上

内核下载：

https://www.kernel.org/

longterm:3.10.1052017-02-10[tar.xz]

安装依赖：

# yum groupinstall "Development Tools"

# yum install ncurses-devel

# yum install qt-devel

# yum install hmaccalc zlib-devel binutils-devel elfutils-libelf-devel

升级内核：

编译内核配置环境

# tar -xf linux-3.10.105.tar.xz -C /usr/local

# cd /usr/local/linux-3.10.105

# cp /boot/config-2.6.32-431.el6.x86_64 .config

# sh -c 'yes "" | make oldconfig'

HOSTCC  scripts/basic/fixdep

HOSTCC  scripts/kconfig/conf.o

SHIPPED scripts/kconfig/zconf.tab.c

SHIPPED scripts/kconfig/zconf.lex.c

SHIPPED scripts/kconfig/zconf.hash.c

HOSTCC  scripts/kconfig/zconf.tab.o

HOSTLD  scripts/kconfig/conf

scripts/kconfig/conf --oldconfig Kconfig

............

CRC8 function (CRC8) [M/y/?] (NEW)

XZ decompression support (XZ_DEC) [Y/?] (NEW) y

x86 BCJ filter decoder (XZ_DEC_X86) [Y/n] (NEW)

PowerPC BCJ filter decoder (XZ_DEC_POWERPC) [N/y] (NEW)

IA-64 BCJ filter decoder (XZ_DEC_IA64) [N/y] (NEW)

ARM BCJ filter decoder (XZ_DEC_ARM) [N/y] (NEW)

ARM-Thumb BCJ filter decoder (XZ_DEC_ARMTHUMB) [N/y] (NEW)

SPARC BCJ filter decoder (XZ_DEC_SPARC) [N/y] (NEW)

XZ decompressor tester (XZ_DEC_TEST) [N/m/y/?] (NEW)

Averaging functions (AVERAGE) [Y/?] y

CORDIC algorithm (CORDIC) [M/y/?] m

JEDEC DDR data (DDR) [N/y/?] (NEW)

#

# configuration written to .config

#

使用cpu的个数作为参数去编译内核

# grep pro /proc/cpuinfo | wc -l

2

# make -j2 bzImage

# make -j2 modules

# make -j2 modules_install

# make install

sh /usr/local/linux-3.10.105/arch/x86/boot/install.sh 3.10.105 arch/x86/boot/bzImage \

System.map "/boot"

ERROR: modinfo: could not find module vsock

ERROR: modinfo: could not find module vmci

ERROR: modinfo: could not find module vmware_balloon

# vi /etc/grub.conf

default=1

timeout=5

splashimage=(hd0,0)/grub/splash.xpm.gz

hiddenmenu

title CentOS (3.10.105)

root (hd0,0)

kernel /vmlinuz-3.10.105 ro root=/dev/mapper/vg_ansibleer-lv_root rd_LVM_LV=vg_ansibleer/lv_swap rd_NO_LUKS rd_LVM_LV=vg_ansibleer/lv_root rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=128M  KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM LANG=en_US.UTF-8 rhgb quiet

initrd /initramfs-3.10.105.img

title CentOS (2.6.32-431.el6.x86_64)

root (hd0,0)

kernel /vmlinuz-2.6.32-431.el6.x86_64 ro root=/dev/mapper/vg_ansibleer-lv_root rd_LVM_LV=vg_ansibleer/lv_swap rd_NO_LUKS rd_LVM_LV=vg_ansibleer/lv_root rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=128M  KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM LANG=en_US.UTF-8 rhgb quiet

initrd /initramfs-2.6.32-431.el6.x86_64.img

修改default=1 to default=0

# reboot

# uname -r

3.10.105

2.2、es编译安装

环境配置：

# vi /etc/security/limits.conf

添加

*               soft    nproc           65536

*               hard    nproc           65536

*               soft    nofile          65536

*               hard    nofile          65536

# vi /etc/sysctl.conf

添加

fs.file-max=65536

vm.max_map_count=262144

# sysctl -p

# vi /etc/security/limits.d/90-nproc.conf

*          soft    nproc     1024

修改为

*          soft    nproc     2048

配置java环境：

# tar -zxf jdk-8u101-linux-x64.tar.gz -C /usr/local

# vi /etc/profile

添加

export JAVA_HOME=/usr/local/jdk1.8.0_101

export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export PATH=$PATH:$JAVA_HOME/bin

# source /etc/profile

编译安装及配置es

# groupadd esuser

# useradd -g esuser -d /home/esuser -m esuser

# passwd esuser

# tar -zxf elasticsearch-5.3.2.tar.gz -C /usr/local

# cd /usr/local

# ln -sv elasticsearch-5.3.2 elasticsearch

`elasticsearch' -> `elasticsearch-5.3.2'

# mkdir -pv /data/elasticsearch/{data,logs}

mkdir: created directory `/data'

mkdir: created directory `/data/elasticsearch'

mkdir: created directory `/data/elasticsearch/data'

mkdir: created directory `/data/elasticsearch/logs'

# chown -R esuser:esuser /data/

# ll -d /data

drwxr-xr-x. 3 esuser esuser 4096 May  5 13:21 /data

# vi /usr/local/elasticsearch/config/elasticsearch.yml

cluster.name: es-cluster

node.name: es-node

path.data: /data/elasticsearch/data

path.logs: /data/elasticsearch/logs

network.host: 182.180.117.200

http.port: 9200

bootstrap.memory_lock: false

bootstrap.system_call_filter: false

启动服务

# su - esuser -c "/usr/local/elasticsearch/bin/elasticsearch &"

这个时候需要注意/usr/local/elsticsearch目录以及目录下的文件的权限，所属者和所属组均是esuser

开通防火墙相应端口

# vi /etc/sysconfig/iptables

在80的规则下添加9200的规则

-A INPUT -p tcp --dport 80 -j ACCEPT

-A INPUT -p tcp --dport 9200 -j ACCEPT

浏览器输入http://192.168.50.215:9200/

得到结果

{

"name" : "es-node",

"cluster_name" : "es-cluster",

"cluster_uuid" : "F3vEJMuvTxmwrT5j049GPA",

"version" : {

"number" : "5.3.2",

"build_hash" : "3068195",

"build_date" : "2017-04-24T16:15:59.481Z",

"build_snapshot" : false,

"lucene_version" : "6.4.2"

},

"tagline" : "You Know, for Search"

}

2.3、215 logstash环境配置

# tar -zxf logstash-5.3.2.tar.gz -C /usr/local

# cd /usr/local

# ln -sv logstash-5.3.2 logstash

`logstash' -> `logstash-5.3.2'

# cd /usr/local/logstash/config

可以使用下面的简单测试系统之间的连通性

# vi logstash-simple.conf

input { stdin { } }

output {

elasticsearch {

hosts => ["182.180.17.200:9200"]

user => elastic

password => changeme

}

stdout { codec => rubydebug }

}

3、kibana环境配置及插件安装

# tar -zxf kibana-5.3.2-linux-x86_64.tar.gz -C /usr/local

# cd /usr/local

# ln -sv kibana-5.3.2-linux-x86_64 kibana

`kibana' -> `kibana-5.3.2-linux-x86_64'

# cd kibana/bin

# ./kibana-plugin install file:///root/x-pack-5.3.2.zip

es插件安装

# cd /usr/local/elasticsearch/bin

# su - esuser -c "/usr/local/elasticsearch/bin/elasticsearch-plugin install file:///root/x-pack-5.3.2.zip"

-> Downloading file:///root/x-pack-5.3.2.zip

[=================================================] 100%??

@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

@     WARNING: plugin requires additional permissions     @

@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

* java.lang.RuntimePermission accessClassInPackage.com.sun.activation.registries

* java.lang.RuntimePermission getClassLoader

* java.lang.RuntimePermission setContextClassLoader

* java.lang.RuntimePermission setFactory

* java.security.SecurityPermission createPolicy.JavaPolicy

* java.security.SecurityPermission getPolicy

* java.security.SecurityPermission putProviderProperty.BC

* java.security.SecurityPermission setPolicy

* java.util.PropertyPermission * read,write

* java.util.PropertyPermission sun.nio.ch.bugLevel write

* javax.net.ssl.SSLPermission setHostnameVerifier

See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html

for descriptions of what these permissions allow and the associated risks.

Continue with installation? [y/N]y

-> Installed x-pack

# cd /usr/local/elasticsearch/config

# vi elasticsearch.yml

添加

#x-pack authc

xpack.security.authc

anonymous:

username: guest

roles: superuser

authz_exception: true

# cd /usr/local/kibana/config

# vi kibana.yml

server.host: "182.180.117.200"

elasticsearch.url: "http://182.180.117.200:9200"

elasticsearch.username: "elastic"

elasticsearch.password: "changeme"

pid.file: /var/run/kibana.pid

# ps -ef | grep elasticsearsh

kill掉es进程

再次启动

# su - esuser -c "/usr/local/elasticsearch/bin/elasticsearch &"

当可以看到端口打开，如下所示，说明启动完成

# netstat -an | grep 9200

tcp        0      0 ::ffff:182.180.117.200:9200 :::*                        LISTEN

# cd /usr/local/logstash/bin

# ./logstash -f ../config/logstash-simple.conf

# cd bin

# ./kibana

# vi /etc/sysconfig/iptables

在9200规则下添加

-A INPUT -p tcp --dport 5601 -j ACCEPT

浏览器输入http://192.168.50.215:5601

出现图示：kibana-1

输入用户名和密码： elastic changeme

4、kafka和zookeeper集群搭建配置

kafka和zookeeper集群的搭建参考我之前es2版本的部署配置文章

链接:

http://xiaoxiaozhou.blog.51cto.com/4681537/1854684

5、nginx日志处理

214上搭建有nginx的应用作为多个服务的反向代理

nginx配置：

# cd /usr/local/nginx

# vi conf/nginx.conf

log_format  main  '$remote_addr - $upstream_addr - $remote_user [$time_local] "$request" '

'$status $upstream_status $body_bytes_sent "$http_referer" '

'"$http_user_agent" "$http_x_forwarded_for"'

'$request_time - $upstream_cache_status' ;

nginx access日志示例：

IP1 - IP2:port - - [11/May/2017:14:18:31 +0800] "GET /content/dam/phone/emv/index.html HTTP/1.1" 304 304 0 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4" "IP3, IP4, IP5, IP6"0.007 - -

$remote_addr客户端地址

$upstream_addr后台upstream的地址，即真正提供服务的主机地址

$remote_user客户端用户名称

$time_local访问时间和时区

$request请求的URI和HTTP协议

$statusHTTP请求状态

$upstream_statusupstream状态

$body_bytes_sent发送给客户端文件内容大小

$http_refererurl跳转来源

$http_user_agent用户终端浏览器等信息

$http_x_forwarded_for记录客户端的ip地址,客户端IP,Nginx负载均衡服务器IP

$request_time  整个请求的总时间

$upstream_cache_status  缓存的状态

%{IPORHOST:client_ip}

(%{URIHOST:upstream_host}|-)

%{USER:ident} %{USER:auth}

\[%{HTTPDATE:timestamp}\]

\"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version})?|-)"

%{HOST:domain} %{NUMBER:response}

(?:%{NUMBER:bytes}|-)

%{QS:referrer}

%{QS:agent}

"(%{WORD:x_forword}|-)"

(%{BASE16FLOAT:request_time})

0\"-\"

# tar -zxf logstash-5.3.2.tar.gz -C /usr/local

# cd /usr/local

# ln -sv logstash-5.3.2 logstash

`logstash' -> `logstash-5.3.2'

# cd logstash

# mkdir patterns

# vi patterns/nginx

NGINXACCESS %{IPORHOST:client_ip} (%{URIHOST:upstream_host}|-) %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version})?|-)" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} "(%{WORD:x_forword}|-)" (%{BASE16FLOAT:request_time}) 0\"-\"

注意：logstash也需要jdk1.8的支持

214的logstash管道文件

# vi /usr/local/logstash/config/logstash_in_nginx.conf

input {

file {

type => "nginx-access"

path => "/usr/local/nginx/logs/access.log"

tags => [ "nginx","access" ]

}

file {

type => "nginx-error"

path => "/usr/local/nginx/logs/error.log"

tags => [ "nginx","error" ]

}

}

output {

stdout { codec => rubydebug }

kafka {

bootstrap_servers => "192.168.50.211:9092,192.168.50.212:9092,192.168.50.213:9092"

topic_id => "access-nginx-messages"

}

}

215的logstash管道文件

# vi /usr/local/logstash/config/logstash_nginx_indexer.conf

input {

kafka {

bootstrap_servers => "192.168.50.211:9092,192.168.50.212:9092,192.168.50.213:9092"

topics => ["access-nginx-messages"]

}

}

filter {

if [type] == "nginx-access" {

grok{

patterns_dir => "/usr/local/logstash/patterns"

match => {

"message" => "%{NGINXACCESS}"

}

}

date{

match=>["time","dd/MMM/yyyy:HH:mm:ss"]

target=>"logdate"

}

ruby{

code => "event.set('logdateunix',event.get('logdate').to_i)"

}

}

else if [type] == "nginx-error" {

grok {

match => [

"message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}(%{NUMBER:pid:int}#%{NUMBER}:\s{1,}\*%{NUMBER}|\*%{NUMBER}) %{DATA:err_message}(?:,\s{1,}client:\s{1,}(?<client_ip>%{IP}|%{HOSTNAME}))(?:,\s{1,}server:\s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, host: %{QS:client_ip})?(?:, referrer: \"%{URI:referrer})?",

"message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}%{GREEDYDATA:err_message}"]

}

date{

match=>["time","yyyy/MM/dd HH:mm:ss"]

target=>"logdate"

}

ruby{

code => "event.set('logdateunix',event.get('logdate').to_i)"

}

}

}

output {

stdout { codec => rubydebug }

elasticsearch {

hosts => ["192.168.50.215:9200"]

index => "access-nginx-messages-%{+YYYY.MM.dd}"

flush_size => 20000

idle_flush_time => 10

template_overwrite => true

}

}

6、es索引操作

查询索引

# curl 192.168.50.215:9200/_search?pretty=true  | grep _index

删除nginx-access-messages开头和access-nginx-messages开头的索引

# curl -XDELETE 'http://192.168.50.215:9200/nginx-access-messages*'

{"acknowledged":true}[root@ansibleer config]#

# curl -XDELETE 'http://192.168.50.215:9200/access-nginx-messages*'

创建索引

# curl -XPUT '192.168.50.215:9200/customer?pretty'

{

"acknowledged" : true,

"shards_acknowledged" : true

}

查询所有索引

# curl '192.168.50.215:9200/_cat/indices?v'

7、214服务器filebeat、logstash配置

# tar -zxf filebeat-5.3.2-linux-x86_64.tar.gz -C /usr/local

# cd /usr/local

# ln -s filebeat-5.3.2 filebeat

# cd filebeat

# egrep -v "#|^$" filebeat.yml

filebeat.prospectors:

- input_type: log

paths:

- /usr/local/nginx/logs/access.log

output.logstash:

hosts: ["192.168.50.214:5043"]

# cd /usr/local/logstash

# cat config/logstash_in_nginx.conf

input {

beats {

port => "5043"

}

}

filter {

grok {

patterns_dir => "/usr/local/logstash/patterns"

match => {

"message" => "%{NGINXACCESS}"

}

}

}

output {

stdout { codec => rubydebug }

kafka {

bootstrap_servers => "192.168.50.211:9092,192.168.50.212:9092,192.168.50.213:9092"

topic_id => "access-nginx-messages"

}

}

215服务器filebeat配置

# tar -zxf filebeat-5.3.2-linux-x86_64.tar.gz -C /usr/local

# cd /usr/local

# ln -s filebeat-5.3.2-linux-x86_64 filebeat

# cd filebeat

# egrep -v "#|^$" filebeat.yml

filebeat.prospectors:

- input_type: log

paths:

- /usr/local/nginx/logs/access.log

output.logstash:

hosts: ["192.168.50.215:5043"]

# nohup ./filebeat -c filebeat.yml &

215服务器的logstash管道文件

# cat /usr/local/logstash/config/logstash_in_nginx.conf

input {

beats {

port => "5043"

}

}

filter {

grok {

patterns_dir => "/usr/local/logstash/patterns"

match => {

"message" => "%{NGINXACCESS}"

}

}

}

output {

stdout { codec => rubydebug }

kafka {

bootstrap_servers => "192.168.50.211:9092,192.168.50.212:9092,192.168.50.213:9092"

topic_id => "access-nginx-messages"

}

}