1、logstash 收集系统日志

1.1 logstash配置文件

root@logstash-node1:~# vim /etc/logstash/conf.d/syslog-to-es.conf input {file {path =>"/var/log/syslog"#第一次从头收集，之后从新添加的日志收集start_position => "beginning"#日志收集的间隔时间stat_interval =>"3"type=>"syslog"}
}
output {if [type] == "syslog" {elasticsearch {hosts => ["10.10.100.120:9200"]index => "syslog-100.105-%{+YYYY.MM.dd}"}}
}

1.2 启动logstash

#添加syslog文件读权限，或者直接使用root用户启动logstash
root@logstash-node1:~# chmod o+r /var/log/syslog#启动前可以通过命令测试配置文件
root@logstash-node1:~# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/syslog-to-es.conf -t#测试正常后重启logstash
root@logstash-node1:~# systemctl restart logstash

1.3 查看es索引

root@es-node1:~# curl 'localhost:9200/_cat/indices?v'
health status index                          uuid                   pri rep docs.count docs.deleted store.size
green  open   syslog-100.105-2022.04.12      2sKdW70BSz2ob8Jfxb6jXw   1   1      13062            0      4.8mb          2.4mb

1.4 kibana 查看日志

1.4.1 创建索引模式

Stack Management >> 索引模式 >> 创建索引模式

1.4.2 查看日志

Discover

2、tomcat日志收集

2.1 tomcat 日志转 json

root@logstash-node1:/apps/tomcat# vim conf/server.xml
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"prefix="localhost_access_log" suffix=".log"pattern="{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l&quot;,&quot;authenticated&quot;:&quot;%u&quot;,&quot;AccessTime&quot;:&quot;%t&quot;,&quot;method&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Query?string&quot;:&quot;%q&quot;,&quot;partner&quot;:&quot;%{Referer}i&quot;,&quot;AgentVersion&quot;:&quot;%{User-Agent}i&quot;}"/>#验证查看日志
root@logstash-node1:/apps/tomcat# tail -f logs/localhost_access_log.2022-04-13.log
{"clientip":"10.10.100.1","ClientUser":"-","authenticated":"-","AccessTime":"[13/Apr/2022:11:32:18 +0800]","method":"GET / HTTP/1.1","status":"200","SendBytes":"14","Query?string":"","partner":"-","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36"}
{"clientip":"10.10.100.1","ClientUser":"-","authenticated":"-","AccessTime":"[13/Apr/2022:11:32:18 +0800]","method":"GET /favicon.ico HTTP/1.1","status":"200","SendBytes":"21630","Query?string":"","partner":"http://10.10.100.115:8080/","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36"}

2.2 配置logstash

root@logstash-node1:~# vim /etc/logstash/conf.d/tomcat_access_log.confinput {file {path => "/apps/tomcat/logs/localhost_access_log.2022-04-13.log"start_position => "beginning"stat_interval =>"3"type=>"tomcat_access_log"codec => "json"}
}
output {if [type] == "tomcat_access_log" {elasticsearch {hosts => ["10.10.100.120:9200"]index => "tomcat-accesslog-100.105-%{+YYYY.MM.dd}"}}
}#添加日志文件权限
root@logstash-node1:~# chmod o+r /apps/tomcat/logs/localhost_access_log.2022-04-13.log#重启logstash
root@logstash-node1:~# systemctl restart logstash

2.3 验证查看

查看索引

root@es-node1:~# curl '10.10.100.120:9200/_cat/indices?v'
health status index                               uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   tomcat-accesslog-100.105-2022.04.13 nXmwge_JQo6oTDx5BZK_3g   1   1         23            0    124.9kb         73.3kb
green  open   syslog-100.105-2022.04.13           8sm8Asb1RSyZZQvVSvsjhw   1   1         42            0     90.8kb         57.9kb

kibana查看日志

3、nginx日志收集

3.1 配置nginx json格式日志

root@logstash-node1:/apps/nginx# vim conf/nginx.conflog_format access_json '{"@timestamp":"$time_iso8601",''"host":"$server_addr",''"clientip":"$remote_addr",''"size":$body_bytes_sent,''"responsetime":$request_time,''"upstreamtime":"$upstream_response_time",''"upstreamhost":"$upstream_addr",''"http_host":"$host",''"url":"$uri",''"domain":"$host",''"xff":"$http_x_forwarded_for",''"referer":"$http_referer",''"status":"$status"}';access_log /apps/nginx/logs/access.log access_json;
#添加权限
root@logstash-node1:/apps/nginx# chmod o+r /apps/nginx/logs/access.log

3.2 配置logstash

root@logstash-node1:~# vim /etc/logstash/conf.d/nginx-access.log input {file {path => "/apps/nginx/logs/access.log"start_position => "beginning"stat_interval => "3"type => "nginx-access-log"codec => json}
}
output {if [type] == "nginx-access-log" {elasticsearch {hosts => ["10.10.100.120:9200"]index => "nginx-accesslog-100.105-%{+YYYY.MM.dd}"}}
}#重启logstash
root@logstash-node1:~# systemctl restart logstash

3.3 验证查看

4、java 日志搜集

使用 codec 的 multiline 插件实现多行匹配，这是一个可以将多行进行合并的插件，而且可以使用 what 指定将匹配到的行与前面的行合并还是和后面的行合并，
https://www.elastic.co/guide/en/logstash/current/plugins-codecs-multiline.html

4.1 配置logstash

root@logstash-node1:~# vim /etc/logstash/conf.d/java_log.conf
input {file {path = "/tmp/java.log"type = "java-log"start_position => "beginning"stat_interval => "3"codec => multiline {#匹配年月日开头行时进行多行合并pattern => "^([0-9]{4}-[0-9]{2}-[0-9]{2}"#true 为匹配成功进行操作，false 为不成功进行操作negate => true#与之前的行合并，如果是下面的行合并就是 nextwhat =>"previous"}}
}
output {if [type] == "syslog" {elasticsearch {hosts => ["10.10.100.120:9200"]index => "javalog-100.105-%{+YYYY.MM.dd}"}}
}

4.2 kibana 查看日志

logstash 日志收集相关推荐

logstash日志收集走过的坑
问题1:OOM内存溢出背景介绍,通常对logstash的应用,是logstash日志收集日志然后写入到kafka,因为logstash本身是插件化,所以就会应用到kafka-output-plugi ...
Logstash日志收集实践
一.安装logstach [root@linux-node2 ~]# tar xf /usr/local/src/logstash-5.2.2.tar.gz -C /usr/local/ [root@ ...
SpringBoot继承LogStash实现日志收集
一.环境准备安装Elasticsearch.kibana.logstash,教程链接安装教程二.配置SpringBoot 依赖在springBoot 项目下pom文件增加logStash 依赖 ...
性能优越的轻量级日志收集工具，微软、亚马逊都在用！
ELK日志收集系统大家都知道,但是还有一种日志收集系统EFK,肯定有很多朋友不知道!这里的F指的是Fluentd,它具有Logstash类似的日志收集功能,但是内存占用连Logstash的十分之一都不 ...
项目实战｜史上最简单的springboot 整合elk教程，实现日志收集（带视频哦）
配套视频教程已经上传整合ELK-实现日志收集(知乎) 整合ELK-实现日志收集(CSDN) 项目源码已上传至 https://gitee.com/yangleliu/learning.git,免费索 ...
ELK 日志收集系统方案
文章目录背景 ELK使用组件简介方案1 ELK 方案二 EFK 方案三: FELK 方案四:个性化框架总结: 日志展示及查询环境背景在项目初期的时候,大家都是赶着上线,一般来说对日志没有过 ...
ELK分布式日志收集搭建和使用
大型系统分布式日志采集系统ELK 全框架 SpringBootSecurity 1.传统系统日志收集的问题 2.Logstash操作工作原理 3.分布式日志收集ELK原理 4.Elasticsearc ...
Logback+ELK+SpringBoot搭建日志收集服务器
前言本文重点介绍Logback和ELK和SpringBoot是怎么整合收集日志的关于ELK的说明和安装可以点击查看ELK(Elasticsearch.Logstash.Kibana)安装, htt ...
微服务架构-链路追踪、日志收集篇
一.链路追踪 1.涉及的插件elasticsearch7.0.0.skywalking-oap-server:8.9.1.skywalking-ui:8.9.1.skywalking-agent 8. ...
ELKstack日志收集系统
简介 ELKstack是由Elasticsearch.Logstash.Kibana三个开源软件组合而成的一款集分析.搜索.图形展示于一身的实施日志收集系统. 各个组件的功能如下: Elastic ...

logstash 日志收集