分布式架构技术
ElasticSearch

ELK统一日志管理

ElasticSearch部署

下载解压改配置文件/config/elasticsearch.yml

1
2
3
4
5
cluster.name: my-application
node.name: node-1
network.host: 0.0.0.0
http.port: 9200
cluster.initial_master_nodes: ["node-1"]

启动es命令：

1
2
3
cd /usr/app/elasticSearch
#后台启动
./bin/elasticksearch -d

Kibana部署

下载解压改配置文件/config/kibana.yml：

1
2
3
port: 5601
server.host: 0.0.0.0
elasticsearch.hosts: ["ip:port","ip:port"]

启动kibana命令：

1
2
3
cd /usr/app/kibana
#后台启动
nohup ./bin/kibana &

logstash部署

启动logstash

1
2
3
4
5
6
7
cd /usr/app/logstash/bin
#测试
logstash --path.settings ../config/ -f ../config/logstash.conf --config.test_and_exit
#启动
logstash -f ../config/logstash-es.conf
#查看端口监听状态以及pid
netstat -lntp |grep 10514

logstash-es.conf 文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
input {
    tcp {
        port => 10514
        codec => json
    }
}
filter {
  grok {
    match=>{"message"=> "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
  }
}

output {
	stdout{
		codec=>rubydebug #美化输出
	}
    elasticsearch {
        action => "index"
        hosts => ["10.0.12.72:9200"]
        index => "dmp_audit_logs_%{[+YYYY-MM-dd]}"
    }
}

logback.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="false">    
    <appender name="console"
              class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%d{MM-dd HH:mm:ss.SSS} %-5level [%logger{50}] - %msg%n
            </pattern>
        </encoder>
    </appender>

    <appender name="logstash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <param name="Encoding" value="UTF-8"/>
        <destination>10.0.12.72:10514</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
            <!--%{appName}中的appName需要在属性中配置，作为字段写入到doc中-->
            <customFields>{"appname":"dmp"}</customFields>
        </encoder>
        <connectionStrategy>
            <roundRobin>
                <connectionTTL>5 minutes</connectionTTL>
            </roundRobin>
        </connectionStrategy>
    </appender>

    <root level="info">
        <appender-ref ref="console"/>
    </root>

    <logger name="com.cetiti.es.controller" level="INFO" addtivity="false">
        <appender-ref ref="logstash"/>
    </logger>
</configuration>

参考文档：

ELK-概念

logback+ELK日志搭建