ELK多服务器环境部署实践

ELK kafka
笔记
发布日期:   2022-05-28
更新日期:   2022-12-04
阅读时长:   10 分

本次实践过程全部在Docker环境下完成。
操作系统环境为centos7，dockerce,docker-compose,xshell、xftp 教育版等。

前提

需要提前准备三台独立服务器(虚拟机),每台服务器的内存至少需要4G,实践过程会将ELK三大件分别部署在这三台服务器上，以最接近生产环境的部署方式进行学习实践。

docker-compose

docker-compose环境变量

文件命名为 .env


ES_VERSION=8.2.2
ZOO=172.17.30.158:12181
KAFKA_CFG_MAX_REQUEST_SIZE=11534336
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=36000
KAFKA1_HOST=172.17.30.158
KAFKA1_PORT=19092
KAFKA2_HOST=172.17.30.160
KAFKA2_PORT=19093
KAFKA3_HOST=172.17.30.160
KAFKA3_PORT=19094

elasticsearch.yml

version: '3.4'

services:
    elasticsearch:
        image: "elasticsearch:${ES_VERSION}"
        container_name: es
        restart: always
        environment:
            - discovery.type=single-node
            - ES_JAVA_OPTS=-Xms3g -Xmx3g
        volumes:
            - /etc/localtime:/etc/localtime
            - /srv/elk/data:/usr/share/elasticsearch/data
            - /root/elk/esconfig/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
        ports:
            - "19200:9200"
            - "19300:9300"
        deploy:
          resources:
            limits:
              memory: 3G
        networks:
          - app-network
networks:
  app-network:
    driver: bridge

elasticsearch 主体配置文件

映射路径为：

elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml

配置内容为：

cluster.name: "docker-cluster"
network.host: 0.0.0.0
xpack.security.enabled: false
xpack.security.authc.api_key.enabled: false
#discovery.type: single-node
logger.level: error
ingest.geoip.downloader.enabled: false
#index.number_of_replicas: 0
xpack.security.enrollment.enabled: true

kibana & logstash

version: '3.4'

services:
    logstash:
        image: "logstash:${ES_VERSION}"
        container_name: logstash
        restart: always
        environment:
          #LOG_LEVEL: error
          ES_JAVA_OPTS: -Xms1g -Xmx1g
        volumes:
            - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
            - ./logstash.yml:/usr/share/logstash/config/logstash.yml
            - /srv/logstash/logs:/usr/share/logstash/logs
        ports:
            - "5044:5044"
        deploy:
          resources:
            limits:
              memory: 1G
        networks:
          - app-network
    kibana:
        image: "kibana:${ES_VERSION}"
        container_name: kibana
        restart: always
        environment:
            - ELASTICSEARCH_URL=http://172.17.30.193:19200
        volumes:
            - /etc/localtime:/etc/localtime
            - ./kibana.yml:/usr/share/kibana/config/kibana.yml
        ports:
            - "5601:5601"
        networks:
          - app-network
    metricbeat:
        image: "elastic/metricbeat:${ES_VERSION}"
        container_name: mb
        restart: always
        environment:
            - setup.kibana.host=kibana:5601
            - output.elasticsearch.hosts=["172.17.30.193:19200"]
        depends_on:
            - kibana
            - logstash
        volumes:
            - /etc/localtime:/etc/localtime
            - /root/elk/mbconfig/logstash.yml:/usr/share/metricbeat/modules.d/logstash.yml
            - /root/elk/mbconfig/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml
        links:
            - kibana
            - logstash
        networks:
          - app-network
networks:
  app-network:
    driver: bridge

kibana 配置文件 kibana.yml

映射路径为：

./kibana.yml:/usr/share/kibana/config/kibana.yml

配置文件内容为：

#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.host: "0.0.0.0"
server.shutdownTimeout: "5s"
elasticsearch.hosts: [ "http://172.17.30.193:19200" ]
monitoring.ui.container.elasticsearch.enabled: true
server.publicBaseUrl: "https://kibana.eo.demo.ehi.com.cn"
i18n.locale: "zh-CN"
xpack.reporting.encryptionKey: f7df822ad927a287fb4579831c16350f
xpack.security.encryptionKey: 7c98394b79e706d8f48deab2635d0c3f
xpack.encryptedSavedObjects:
  encryptionKey: "ae0c2bce5918f313ba192162e944d48b" 
  keyRotation:
    decryptionOnlyKeys: ["wy80Kj77iJZlFg65nz4jolf2IKoKlymo", "6oAOsRhu86PVgdaMILLNvjh4oWulx8EG"]
xpack.reporting.kibanaServer.hostname: localhost
xpack.reporting.roles.enabled: false

logstash 配置文件 logstash.yml

映射路径为：

- ./logstash.yml:/usr/share/logstash/config/logstash.yml

配置内容为：

http.host: 0.0.0.0
log.level: warn
monitoring.enabled: false
xpack.monitoring.elasticsearch.hosts:
- http://172.17.30.193:19200

logstash.conf

映射路径为：

- ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf

配置文件为：

input {
  kafka {
      bootstrap_servers => "172.17.30.158:19092,172.17.30.160:19093,172.17.30.160:19094"
      consumer_threads => 10
      group_id => "eolog"
      auto_commit_interval_ms => 1000
      topics => ["csenterprisecarrentalapilog","csenterprisecarrentalapiaudit","csenterprisecarrentalapirequest"]
      decorate_events => "basic"
     # auto_offset_reset => "latest"
      auto_offset_reset => "earliest"
      #auto_offset_reset => "none"
      max_partition_fetch_bytes => 104857600
  }

}
filter {
      json {
           source => "message"             #将message字段进行解析
           skip_on_invalid_json => true    #无效的json则跳过
           remove_field => ["message"]     #删除message字段
      }
      
      # json {
      #      source => "[message][responseData]"             #将responseData字段进行解析
      #      target => "[message][responseData]"
      #      skip_on_invalid_json => true    #无效的json则跳过
      #     #  remove_field => ["responseData"]     #删除message字段
      # }
    #增加一个字段，计算 timestamp 8小时
    ruby {
        code => "event.set('timestamp', event.get('@timestamp').time.utc+8*60*60)"
    }

    #用 mutate 插件先转换为 string 类型，gsub只处理string类型的数据
    #再用正则匹配，最终得到想要的日期
    mutate {
        convert => ["timestamp", "string"]
        gsub => ["timestamp", "T([\S\s]*?)Z", ""]
        gsub => ["timestamp", "-", "."]
   }
}
 

output {
  
  if [@metadata][kafka][topic] == "csenterprisecarrentalapirequest" {
    elasticsearch {
    hosts => ["172.17.30.193:19200"]
    index => "csenterprisecarrentalapirequest-%{timestamp}"
    }
  }
  else if [@metadata][kafka][topic] == "csenterprisecarrentalapiaudit" {
    elasticsearch {
    hosts => ["172.17.30.193:19200"]
    index => "csenterprisecarrentalapiaudit-%{timestamp}"
    }
  }
  else if [@metadata][kafka][topic] == "csenterprisecarrentalapilog" {
    elasticsearch {
    hosts => ["172.17.30.193:19200"]
    index => "csenterprisecarrentalapilog-%{timestamp}"
    }
  }
  
 #stdout { codec => rubydebug }
  
}

metricbeat 监控

配置文件名为：metricbeat.yml
映射路径为：

metricbeat.yml:/usr/share/metricbeat/metricbeat.yml

配置内容为：

metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

processors:
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

output.elasticsearch:
  hosts: ["https://172.17.30.193:19200"]
  username: '${ELASTICSEARCH_USERNAME:}'
  password: '${ELASTICSEARCH_PASSWORD:}'

kafka-master

version: '3.4'

services:
#  kafka-ui
    kafka-ui:
      container_name: kafka-ui
      hostname: kafka-ui
      restart: always
      image: provectuslabs/kafka-ui:latest
      ports:
        - 8080:8080
      depends_on:
        - zookeeper
        - kafka
      environment:
        KAFKA_CLUSTERS_0_NAME: kafka1
        KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: ${KAFKA1_HOST}:${KAFKA1_PORT}
        KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
        KAFKA_CLUSTERS_1_NAME: kafka2
        KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS: ${KAFKA2_HOST}:${KAFKA2_PORT}
        KAFKA_CLUSTERS_1_ZOOKEEPER: zookeeper:2181
        KAFKA_CLUSTERS_2_NAME: kafka3
        KAFKA_CLUSTERS_2_BOOTSTRAPSERVERS: ${KAFKA3_HOST}:${KAFKA3_PORT}
        KAFKA_CLUSTERS_2_ZOOKEEPER: zookeeper:2181
      links:
        - zookeeper
        - kafka
      networks:
        - app-network
#   zookeeper
    zookeeper:
      image: wurstmeister/zookeeper
      container_name: zoo
      restart: always
      volumes:
        - /srv/zookeeper/data:/data
      ports:
        - 12181:2181
      networks:
        - app-network
#   zoo-ui
    zoo-ui:
      image: elkozmon/zoonavigator
      container_name: zoo-ui
      restart: always
      environment:
        HTTP_PORT: 9000
      ports:
        - 19000:9000
      networks:
        - app-network
# kafka-master   
    kafka:
      image: wurstmeister/kafka
      container_name: kafka
      restart: always
      ports:
        - ${KAFKA1_PORT}:${KAFKA1_PORT}
      environment:
        KAFKA_BROKER_ID: 1
        KAFKA_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://:${KAFKA1_PORT}
        KAFKA_ADVERTISED_HOST_NAME: ${KAFKA1_HOST}
        KAFKA_ADVERTISED_PORT: ${KAFKA1_PORT}
        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://${KAFKA1_HOST}:${KAFKA1_PORT}
        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_DOCKER:PLAINTEXT
        KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT_DOCKER
        KAFKA_LOG4J_ROOT_LOGLEVEL: ERROR
        KAFKA_TOOLS_LOG4J_LOGLEVEL: ERROR
        KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
        KAFKA_ZOOKEEPER_CONNECT: ${ZOO}
        KAFKA_CFG_MAX_REQUEST_SIZE: ${KAFKA_CFG_MAX_REQUEST_SIZE}
        KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: ${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}
        KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: ${KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS}
      volumes:
        - /srv/kafka:/kafka
      deploy:
        resources:
          limits:
            memory: 800M 
      depends_on:
        - zookeeper
      links:
        - zookeeper
      networks:
        - app-network
networks:
  app-network:
    driver: bridge

kafka-node

version: '3.4'

services:   
    kafka2:
      image: wurstmeister/kafka
      container_name: kafka2
      restart: always
      ports:
        - ${KAFKA2_PORT}:${KAFKA2_PORT}
      environment:
        KAFKA_BROKER_ID: 2
        KAFKA_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://:${KAFKA2_PORT}
        KAFKA_ADVERTISED_HOST_NAME: ${KAFKA2_HOST}
        KAFKA_ADVERTISED_PORT: ${KAFKA2_PORT}
        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://${KAFKA2_HOST}:${KAFKA2_PORT}
        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_DOCKER:PLAINTEXT
        KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT_DOCKER
        KAFKA_LOG4J_ROOT_LOGLEVEL: ERROR
        KAFKA_TOOLS_LOG4J_LOGLEVEL: ERROR
        KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
        KAFKA_ZOOKEEPER_CONNECT: ${ZOO}
        KAFKA_CFG_MAX_REQUEST_SIZE: ${KAFKA_CFG_MAX_REQUEST_SIZE}
        KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: ${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}
        KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: ${KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS}
      volumes:
        - /srv/kafka/2:/kafka
      deploy:
        resources:
          limits:
            memory: 800M
      networks:
        - app-network
    kafka3:
      image: wurstmeister/kafka
      container_name: kafka3
      restart: always
      ports:
        - ${KAFKA3_PORT}:${KAFKA3_PORT}
      environment:
        KAFKA_BROKER_ID: 3
        KAFKA_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://:${KAFKA3_PORT}
        KAFKA_ADVERTISED_HOST_NAME: ${KAFKA3_HOST}
        KAFKA_ADVERTISED_PORT: ${KAFKA3_PORT}
        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://${KAFKA3_HOST}:${KAFKA3_PORT}
        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_DOCKER:PLAINTEXT
        KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT_DOCKER
        KAFKA_LOG4J_ROOT_LOGLEVEL: ERROR
        KAFKA_TOOLS_LOG4J_LOGLEVEL: ERROR
        KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
        KAFKA_ZOOKEEPER_CONNECT: ${ZOO}
        KAFKA_CFG_MAX_REQUEST_SIZE: ${KAFKA_CFG_MAX_REQUEST_SIZE}
        KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: ${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}
        KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: ${KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS}
      volumes:
        - /srv/kafka/3:/kafka
      deploy:
        resources:
          limits:
            memory: 800M
      networks:
        - app-network
networks:
  app-network:
    driver: bridge

注意

kafka集群与单体实例

kafka集群与单体实例在docker环节下的访问与注册比较大的差异。

  • 单体情况下,各服务直接允许直接使用links方式直接使用service别称访问kafka服务。
  • 在集群环境下，需要特别关注kafka的几个配置,
    • KAFKA_LISTENERS,kafka容器内部监听的地址集合，通常情况下只需要监听默认的9092即可，但集群条件下，特别是kafka的节点分布在不同主键的情况下，还需要监听一个专门用来提供对外服务的端口，该端口会直接与宿主机端口进行映射，对外提供服务,其中 PLAINTEXT是协议类型，像docker集群环境下一般会出现至少2个监听端口，一个对内一个对外，此时如果协议相同的情况，还需要对协议取别名，利用的是 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP 环境变量。需要特别注意的是KAFKA_LISTENERS仅需要监听端口，一般不需要定义host部分
    • KAFKA_ADVERTISED_LISTENERS 这个环境变量表示的是外部kafka客户端访问服务器端时可以使用的endpoint集合，单机情况下与KAFKA_LISTENERS相同。集群条件下需要同时声明docker网络环节下的别名访问地址(//kafka:9200),还有外部客户端访问宿主机端口的地址定义(172.17.30.158:19200),其中 19200 其实是提前约定好的宿主机端口，为了统一一般将kafka容器内的自定义监听端口也设置成这个，方便管理和区分
    • KAFKA_ADVERTISED_HOST_NAME,kafka对外部主机提供服务的地址(一般就是宿主机的ip地址)
    • KAFKA_ADVERTISED_PORT,kafka对外部主机提供服务的端口,就是上一条说过的 19200
KAFKA_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://:${KAFKA3_PORT}
KAFKA_ADVERTISED_HOST_NAME: ${KAFKA3_HOST}
KAFKA_ADVERTISED_PORT: ${KAFKA3_PORT}
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,PLAINTEXT_DOCKER://${KAFKA3_HOST}:${KAFKA3_PORT}
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_DOCKER:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT_DOCKER

kibana与es主机分离

kibana与es主机分离后再需要连接回去，只能通过es的宿主机映射地址访问，并且需要在es启动完成后，重启kibana服务，才会生效，否则每次kibana容器重启后，都会要求手动填token或者输入kibana验证码。

补充

es定时清理脚本

#######################################
#删除早于十天的ES集群的索引
#######################################

#!/bin/bash

range=5

if [ -n "$1" ]; then 
  range=$1
fi
indexs=` curl -X GET 'http://localhost:19200/_cat/indices?v' | awk '{print $3}' | grep -vE '(kibana|index|watcher|monitoring)'`

for index in $indexs
  do
     echo "启动清理流程-$index"
     curl  -X POST "localhost:19200/$index/_delete_by_query?pretty" -H 'Content-Type:application/json' -d '
      {
         "query": { 
              "bool": {
                   "must": [
                      {
                        "range": {
                           "@timestamp": {
                               "lte": "now-'${range}'d/d"
                              }
                         }
                     }
                   ],
                   "must_not": []
              }  
         }  
     }'
     echo "已清除$index 索引内$range 天前数据~"
   done
文章作者: rokey
文章链接: https://www.rokeyyan.com/2022/05/28/note/elk-cluster-deployment/
版权声明: 本博客所有文章除特別声明外，均采用 CC BY 4.0 许可协议。转载请注明来源 rokey !
ELK kafka
赏
评论
 上一篇
ACME脚本自动续期HTTPS证书 ACME脚本自动续期HTTPS证书
泛域名证书太贵，单域名证书一年一续，域名多了也很麻烦，幸好有github可以抱大佬大腿，ACME自动化续期HTTPS证书ACME.SH 资源地址acme.sh 主要步骤 安装 acme.sh 生成证书 copy 证书到 nginx/
2022-06-03 笔记
HTTPS NGINX ACME
下一篇 
MacOS 硬盘启动盘制作 MacOS 硬盘启动盘制作
1.下载MacOS到mac上2.启动终端，输入命令sudo /Applications/Install macOS Big Sur.app/Contents/Resources/createinstallmedia --volume /Vo
2021-03-21 笔记
MacOS
  目录