一个日志服务架构主要包括3个部分：日志采集agent，日志存储，及日志浏览

本日志服务方案采用logstash+elasticsearch+kibana的组合搭建，其中logstash负责日志的采集和入库，elasticsearch负责日志的存储和索引，kubana负责日志的搜索和前端展示

在实际部署过程中，为保证可用和节省资源，主要注意以下几点：

1.logstash支持很多socket远程input和ouput插件，但为不影响主干业务流，尽量采用日志文件异步读出写入es的方式，如log4j支持socketappend，但需要修改既有应用的配置，而且一旦server端出错，应用日志没有本地文件，容易丢失

2.logstash agent可以直接写入es，但为减小es的压力，采用redis和logstash

indexer的较色来延长消息传递流程，降低各环节的压力

3.logstash基于java实现，为节约系统资源，不论agent还是indexer，单台物理机只配置一个文件，采用一个守护进程启动服务

cat

/usr/local/logserver/logstash/conf/allinone_indexer.conf

input

{

redis

{

host =>

"10.241.223.112"

type =>

"log4j-account"

data_type =>

"list"

key =>

"log4j-account"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-activity"

data_type =>

"list"

key =>

"log4j-activity"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-databag"

data_type =>

"list"

key =>

"log4j-databag"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-document"

data_type =>

"list"

key =>

"log4j-document"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-job"

data_type =>

"list"

key =>

"log4j-job"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-monitor"

data_type =>

"list"

key =>

"log4j-monitor"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-notification"

data_type =>

"list"

key =>

"log4j-notification"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-qto"

data_type =>

"list"

key =>

"log4j-qto"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-search"

data_type =>

"list"

key =>

"log4j-search"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-social"

data_type =>

"list"

key =>

"log4j-social"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"log4j-storage"

data_type =>

"list"

key =>

"log4j-storage"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"tomcat-account"

data_type =>

"list"

key =>

"tomcat-account"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"tomcat-databag"

data_type =>

"list"

key =>

"tomcat-databag"

format =>

"json_event"

}

redis

{

host =>

"10.241.223.112"

type =>

"tomcat-monitor"

data_type =>

"list"

key =>

"tomcat-monitor"

format =>

"json_event"

}

}

filter {

multiline

{

type =>

"log4j-account"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-activity"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-databag"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-document"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-job"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-monitor"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-notification"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-qto"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-search"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-social"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"log4j-storage"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"tomcat-account"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"tomcat-databag"

pattern =>

"^\s"

what =>

"previous"

}

multiline

{

type =>

"tomcat-monitor"

pattern =>

"^\s"

what =>

"previous"

}

}

output {

elasticsearch {

host

=> "10.241.223.112"

cluster

=> "logstashelasticsearch"

port

=> 9300

}

}

cat

/usr/local/logserver/logstash/conf/nginx_agent.conf

input {

file {

type => "nginx"

path => ["/data/logs/nginx/*.log"

]

}

}

output {

redis {

host => "10.241.223.112"

data_type => "list"

key => "nginx"

type => "nginx"

}

}

cat

/usr/local/logserver/logstash/startup.sh

#!/bin/bash

workdir=$(cd $(dirname $0);pwd)

#echo $workdir

usage(){

cat <

This script start, stop and retart logstash agent and indexer

of apps

Usage: $(basename $0) [ agent | indexer | all ] app1 app2 app3

...

EOF1

}

[ $# -lt 2 ] && usage

&& exit

start_mode=$1

for appname in "$@";do

case "$appname" in

"agent" )

continue

;;

"indexer" )

continue

;;

"all" )

continue

;;

*)

if [ "$start_mode" =

"all" ]; then

java

-jar $workdir/logstash.jar agent -f

$workdir/conf/"$appname"_indexer.conf &

java

-jar $workdir/logstash.jar agent -f

$workdir/conf/"$appname"_agent.conf &

else

java

-jar $workdir/logstash.jar agent -f

$workdir/conf/"$appname"_"$start_mode".conf &

fi

;;

esac

echo

done

cat /usr/local/logserver/logstash/indexer_startup.sh

#!/bin/bash

#/usr/local/logserver/logstash/startup.sh indexer account activity

databag

/usr/local/logserver/logstash/startup.sh indexer allinone