DockerはZookeeperとKafkaクラスタを構築する

33869 ワード

Docker-composeのインストール


テープネットワークの独自のネットワークを作成


zoo_という名前の作成kafkaのネットワーク、サブネットセグメント172.23.0.10/20
root@localhost ]# docker network create --subnet=172.23.0.10/20 zoo_kafka
root@localhost ]# docker network ls
NETWORK ID          NAME                DRIVER              SCOPE
4fad358ef523        bridge              bridge              local
2ea1ad136fa2        host                host                local
a66aec056df4        kafka_default       bridge              local
9a4fbf8d993d        none                null                local
d584dd2084c7        zoo_kafka           bridge              local

Zookeeperクラスタの作成


フォルダの作成

root@localhost ]# mkdir zookeeper
root@localhost ]# cd zookeeper

docker-composeを作成します。ymlファイル

root@localhost ]# touch docker-compose.yml
root@localhost ]# vim docker-compose.yml

内容は次のとおりです.
version: '2'
services:
  zoo1:
    image: zookeeper:3.4 #     
    restart: always #           
    hostname: zoo1
    container_name: zoo1
    privileged: true
    ports: #   
      - 2181:2181
    volumes: #      
      - ./zoo1/data:/data
      - ./zoo1/datalog:/datalog 
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 1 #   ID
      ZOO_PORT: 2181 # zookeeper   
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 # zookeeper    
    networks:
      default:
        ipv4_address: 172.23.0.11

  zoo2:
    image: zookeeper:3.4
    restart: always
    hostname: zoo2
    container_name: zoo2
    privileged: true
    ports:
      - 2182:2181
    volumes:
      - ./zoo2/data:/data
      - ./zoo2/datalog:/datalog
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 2
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.12

  zoo3:
    image: zookeeper:3.4
    restart: always
    hostname: zoo3
    container_name: zoo3
    privileged: true
    ports:
      - 2183:2181
    volumes:
      - ./zoo3/data:/data
      - ./zoo3/datalog:/datalog
    environment:
      TZ: Asia/Shanghai
      ZOO_MY_ID: 3
      ZOO_PORT: 2181
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      default:
        ipv4_address: 172.23.0.13

networks:
  default:
    external:
      name: zoo_kafka

起動と検証

  • バックグラウンドで起動し、コンテナ
  • を表示します.
    root@localhost zookeeper]# docker-compose up -d
    Starting zoo3 ... done
    Starting zoo2 ... done
    Starting zoo1 ... done
    [root@localhost zookeeper]# docker ps
    CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                        NAMES
    125a78286d6b        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp   zoo1
    4998e54b7140        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zoo2
    5eec0034d73b        zookeeper:3.4       "/docker-entrypoint.…"   22 minutes ago      Up 12 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2183->2181/tcp   zoo3
    
  • zookeeperクラスタを検証zoo 3はleader、zoo 1とzoo 2はfollower
  • [root@localhost zookeeper]# docker exec zoo1 zkServer.sh status
    ZooKeeper JMX enabled by default
    Using config: /conf/zoo.cfg
    Mode: follower
    [root@localhost zookeeper]# docker exec zoo2 zkServer.sh status
    ZooKeeper JMX enabled by default
    Using config: /conf/zoo.cfg
    Mode: follower
    [root@localhost zookeeper]# docker exec zoo3 zkServer.sh status
    ZooKeeper JMX enabled by default
    Using config: /conf/zoo.cfg
    Mode: leader
    
    

    Kafkaクラスタの作成


    フォルダの作成


    フォルダの作成

    root@localhost ]# mkdir kafka
    root@localhost ]# cd kafka
    

    docker-composeを作成します。ymlファイル

    root@localhost ]# touch docker-compose.yml
    root@localhost ]# vim docker-compose.yml
    

    内容は次のとおりです.
    version: '2'
    
    services:
      broker1:
        image: wurstmeister/kafka:2.11-0.11.0.3
        restart: always
        hostname: broker1
        container_name: broker1
        privileged: true
        ports:
          - "9091:9092"
        environment:
          KAFKA_BROKER_ID: 1
          KAFKA_LISTENERS: PLAINTEXT://broker1:9092
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:9092
          KAFKA_ADVERTISED_HOST_NAME: broker1
          KAFKA_ADVERTISED_PORT: 9092
          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
          JMX_PORT: 9988
        volumes:
          - /var/run/docker.sock:/var/run/docker.sock
          - ./broker1:/kafka/kafka\-logs\-broker1
        external_links:
        - zoo1
        - zoo2
        - zoo3
        networks:
          default:
            ipv4_address: 172.23.0.14
    
      broker2:
        image: wurstmeister/kafka:2.11-0.11.0.3
        restart: always
        hostname: broker2
        container_name: broker2
        privileged: true
        ports:
          - "9092:9092"
        environment:
          KAFKA_BROKER_ID: 2
          KAFKA_LISTENERS: PLAINTEXT://broker2:9092
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2:9092
          KAFKA_ADVERTISED_HOST_NAME: broker2
          KAFKA_ADVERTISED_PORT: 9092
          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
          JMX_PORT: 9988
        volumes:
          - /var/run/docker.sock:/var/run/docker.sock
          - ./broker2:/kafka/kafka\-logs\-broker2
        external_links:  #    compose     container
        - zoo1
        - zoo2
        - zoo3
        networks:
          default:
            ipv4_address: 172.23.0.15
    
      broker3:
        image: wurstmeister/kafka:2.11-0.11.0.3
        restart: always
        hostname: broker3
        container_name: broker3
        privileged: true
        ports:
          - "9093:9092"
        environment:
          KAFKA_BROKER_ID: 3
          KAFKA_LISTENERS: PLAINTEXT://broker3:9092
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3:9092
          KAFKA_ADVERTISED_HOST_NAME: broker3
          KAFKA_ADVERTISED_PORT: 9092
          KAFKA_ZOOKEEPER_CONNECT: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
          JMX_PORT: 9988
        volumes:
          - /var/run/docker.sock:/var/run/docker.sock
          - ./broker3:/kafka/kafka\-logs\-broker3
        external_links:  #    compose     container
        - zoo1
        - zoo2
        - zoo3
        networks:
          default:
            ipv4_address: 172.23.0.16
    
      kafka-manager:
        image: sheepkiller/kafka-manager:latest
        restart: always
        container_name: kafka-manager
        hostname: kafka-manager
        ports:
          - "9000:9000"
        links:            #    compose     container
          - broker1
          - broker2
          - broker3
        external_links:   #    compose     container
          - zoo1
          - zoo2
          - zoo3
        environment:
          ZK_HOSTS: zoo1:2181/kafka1,zoo2:2181/kafka1,zoo3:2181/kafka1
          KAFKA_BROKERS: broker1:9092,broker2:9092,broker3:9092
          APPLICATION_SECRET: letmein
          KM_ARGS: -Djava.net.preferIPv4Stack=true
        networks:
          default:
            ipv4_address: 172.23.0.10
    
    networks:
      default:
        external:   #         
          name: zoo_kafka
    
    

    起動と検証

    [root@localhost kafka]# docker-compose up -d
    Creating broker1 ... done
    Creating broker3 ... done
    Creating broker2 ... done
    Creating kafka-manager ... done
    [root@localhost kafka]# docker ps
    CONTAINER ID        IMAGE                              COMMAND                  CREATED             STATUS              PORTS                                        NAMES
    8140b9c9eb07        sheepkiller/kafka-manager:latest   "./start-kafka-manag…"   4 minutes ago       Up 4 minutes        0.0.0.0:9000->9000/tcp                       kafka-manager
    5df60f08f5aa        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9093->9092/tcp                       broker3
    4d17d490faa7        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9092->9092/tcp                       broker2
    07206f603d20        wurstmeister/kafka:2.11-0.11.0.3   "start-kafka.sh"         4 minutes ago       Up 4 minutes        0.0.0.0:9091->9092/tcp                       broker1
    125a78286d6b        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp   zoo1
    4998e54b7140        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2182->2181/tcp   zoo2
    5eec0034d73b        zookeeper:3.4                      "/docker-entrypoint.…"   37 minutes ago      Up 27 minutes       2888/tcp, 3888/tcp, 0.0.0.0:2183->2181/tcp   zoo3