Docker配備ELK

ダウンロード

#    ELK + Filebeat
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.6.1
docker pull docker.elastic.co/kibana/kibana:7.6.1
docker pull docker.elastic.co/logstash/logstash:7.6.1
docker pull docker.elastic.co/beats/filebeat:7.6.1

#     
docker images
#   
REPOSITORY                                      TAG                 IMAGE ID            CREATED             SIZE
docker.elastic.co/logstash/logstash             7.6.1               d6d66afe6805        10 days ago         813MB
docker.elastic.co/kibana/kibana                 7.6.1               f9ca33465ce3        10 days ago         1.01GB
docker.elastic.co/elasticsearch/elasticsearch   7.6.1               41072cdeebc5        10 days ago         790MB
docker.elastic.co/beats/filebeat                7.6.1               cd244d9a74c9        10 days ago         364MB

ESオン

#   ： https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html
docker run -d -p 9201:9200 -p 9301:9300 -e "discovery.type=single-node" \
docker.elastic.co/elasticsearch/elasticsearch:7.6.1

#       （   ）
docker exec -it [CONTAINER ID] /bin/bash

Kibanaを開く

#   ：https://www.elastic.co/guide/en/kibana/current/docker.html
docker run --link [ES_CONTAINER ID]:elasticsearch -p 5602:5601 -d \
docker.elastic.co/kibana/kibana:7.6.1

Logstashを開く

配置logstash.yml

#             
mkdir -p /usr/share/logstash/config/
cd /usr/share/logstash/config/
#     
vim logstash.yml
#   
http.host: "0.0.0.0"
xpack.management.pipeline.id: ["main"]
#   

配置pipeline.yml

#        
vim pipeline.yml
#   （    ）
 - pipeline.id: main
  path.config: "/usr/share/logstash/pipeline/logstash.conf"
#   

配置logstash.conf

#             
mkdir -p /usr/share/logstash/pipeline/
cd /usr/share/logstash/pipeline/
#     
vim logstash.conf
#   
input {
    beats { port => 5044 }
}

filter {
	 if [project] == "Nginx" {
                grok {
                        match => {
                                "message" =>[
                                        "%{IPORHOST:client_ip}\s{1,}\-\s\-\s\[%{HTTPDATE:time}\]\s{1,}\"(?:%{WORD:verb}\s{1,}%{NOTSPACE:request}(?:\s{1,}HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response}\s{1,}(?:%{NUMBER:bytes}|-)\s{1,}%{QS:referrer}\s{1,}%{QS:agent}"                
                                ]
                        }
                }
                date {
                         match => ["time","dd/MMM/yyyy:HH:mm:ss Z"]
                         target => "logdate"
                }

                mutate {
                        lowercase => ["company", "project", "server", "application"]
                        remove_field => ['cloud', 'tags', 'host', 'agent', 'log', 'ecs', '@version', 'message']
                }

                ruby {
                    code => "event.set('logdate', event.get('logdate').time.localtime)"
                }
                geoip {
                        source => "client_ip"
                }

        }
}

output {
    elasticsearch {
        hosts => ["192.168.60.221:9201"]
        index => "%{[company]}_%{[project]}_%{[server]}_%{[application]}_%{+YYYY}"
    }
}
#   

Logstash

を開く

#   ：https://www.elastic.co/guide/en/logstash/current/docker.html
docker run -d -p 5046:5044 -p 9601:9600 --rm -it -v /usr/share/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml \
-v /usr/share/logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml \
-v /usr/share/logstash/pipeline/logstash.conf:/usr/share/logstash/pipeline/logstash.conf \
docker.elastic.co/logstash/logstash:7.6.1

Filebeatを開く

#   ：https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html
#             
mkdir -p /usr/share/filebeat/
cd /usr/share/filebeat/
#      filebeat.yml 
#     
chmod go-w /usr/share/filebeat/filebeat.yml
#    Filebeat（  ，   ）
docker run -v /usr/share/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml \
-v /usr/local/nginx/logs/access.log:/var/lib/docker/containers/access.log \
docker.elastic.co/beats/filebeat:7.6.1