Kubernetes v1.8.4クラスタの各コンポーネントの暗号化認証機能の構成参考

19937 ワード
クラウドコンピューティング kubernetes
記録v 1.8.4のコンポーネント構成は以下の通りで、テストによって完璧に動作する.
ここでkube-apiserverはHAをしていません.まずこのテストに合格して、後でします.ハ側は、haproxy+keepalivedを加えて構成し、:443でアクセスさせるだけでよい.haproxyは負荷転送を担当し、keepalivedはhaproxyのhaを監視する.haproxyとkeepalivedは、3台のmasterのうち2台に配置できます.これにより、masterのapi-serverで競合するポートを使用しないで、ポートの問題に注意してください.
もう言わないで、コンポーネントの暗号化認証機能をどのように構成するかを直接見て、具体的にコメントを追加しました.
その中のca証明書は、まずmasterでmaster_に従う必要があります.ssl.cnfはroot証明書、すなわちca.crt,ca.keyを生成し、serverを生成するように構成する.crt, server.key.一番後ろにca.crt,ca.keyをnodeノードに配置し、kubeletの証明書の生成を続行します.

master_ssl.cnf


マスターの編集ssl.cnf、関連する構成を追加
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.k8s.xxx.com # specify domain name.
DNS.5 = CDM1B12-209202200.wdds.com # apiserver hostname
IP.1 = 10.0.0.1 # kubernetes cluster ip
IP.2 = 10.209.202.200  # kubernetes apiserver ip

masterで証明書を作成するスクリプト

function create_master_ca() {
    echo "start create master ca ........."

    openssl genrsa -out ca.key 2048
    openssl req -x509 -new -nodes -key ca.key -subj "/CN=xxx.com" -days 5000 -out ca.crt
    openssl genrsa -out server.key 2048

    HN=`hostname`
    echo "hostname is :$HN"
    openssl req -new -key server.key -subj "/CN=$HN" -config master_ssl.cnf -out server.csr
    openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 5000 -extensions v3_req -extfile master_ssl.cnf -out server.crt


    openssl genrsa -out cs_client.key 2048
    openssl req -new -key cs_client.key -subj "/CN=$HN" -out cs_client.csr
    openssl x509 -req -in cs_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out cs_client.crt -days 5000

    cp -f ca.* server.crt server.key cs_client.crt cs_client.key /etc/kubernetes/pki
    cp -f /root/kube_package/ssl/kubeconfig /etc/kubernetes/kubeconfig
    echo "end create master ca ........."
}

create_master_ca

nodeで証明書を作成するスクリプト


#!/bin/bash

cd `dirname $0`
HN=`hostname`
openssl genrsa -out kubelet_client.key 2048
openssl req -new -key kubelet_client.key -subj "/CN=$HN" -out kubelet_client.csr
openssl x509 -req -in kubelet_client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out kubelet_client.crt -days 5000

kube-apiserver

apiVersion: v1
kind: Pod
metadata:
  name: kube-apiserver
spec:
  hostNetwork: true
  containers:
  - name: kube-apiserver
    image: 10.213.42.254:10500/root/hyperkube:v1.8.4-ceph
    command:
    - /bin/sh
    - -c
    - /hyperkube apiserver
      --apiserver-count=1 # ： ha
      --allow-privileged=true
      --etcd-prefix=/cd-dev02
      --etcd-servers=http://10.209.202.200:2379,http://10.209.204.167:2379,http://10.209.204.199:2379
      --admission-control=SecurityContextDeny,ServiceAccount,NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota
      --insecure-bind-address=0.0.0.0
      --insecure-port=11080 #  。
      --secure-port=443 #  。
      --advertise-address=10.209.202.200
      --service-cluster-ip-range=10.0.0.0/18
      --tls-cert-file=/etc/kubernetes/pki/server.crt #  ： ，  VolumeMounts  。
      --tls-private-key-file=/etc/kubernetes/pki/server.key
      --client-ca-file=/etc/kubernetes/pki/ca.crt
      --alsologtostderr=false
      --logtostderr=true
      --v=0
      --log-dir=/var/log/kubernetes
      --service-node-port-range=10000-12000
      --storage-backend=etcd3
      --storage-media-type=application/vnd.kubernetes.protobuf
      --runtime-config=v1,extensions/v1beta1=true,extensions/v1beta1/ingress=true >> /var/log/kubernetes/kube-apiserver.log 2>&1
    ports:
        - containerPort: 443
          hostPort: 443
          name: https
        - containerPort: 7080
          hostPort: 7080
          name: http
        - containerPort: 11080
          hostPort: 11080
          name: local
        - containerPort: 6443
          hostPort: 6443
          name: seport
        volumeMounts:
        - mountPath: /etc/kubernetes
          name: pki
          readOnly: true
        - mountPath: /var/log
          name: logpath
        - mountPath: /etc/localtime
          name: localtime
      volumes:
      - hostPath:
          path: /etc/kubernetes
        name: pki
      - hostPath:
          path: /var/log
        name: logpath
      - hostPath:
          path: /etc/localtime
        name: localtime

kube-controller-manager


apiVersion: v1
kind: Pod
metadata:
  name: kube-controller-manager
spec:
  hostNetwork: true
  containers:
  - name: kube-controller-manager
    image: 10.213.42.254:10500/root/hyperkube:v1.8.4-ceph
    command:
    - /bin/sh
    - -c
    - /hyperkube controller-manager
      --v=0
      --logtostderr=true
      --log-dir=/var/log/kubernetes
      --alsologtostderr=false
      --root-ca-file=/etc/kubernetes/pki/ca.crt
      --service-account-private-key-file=/etc/kubernetes/pki/server.key
      --kubeconfig=/etc/kubernetes/kubeconfig # yaml  --master=https://vip:443  ， ， kubeconfig 
      --leader-elect=true >> /var/log/kubernetes/kube-controller-manager.log 2>&1
    ports:
    - containerPort: 10252
      hostPort: 10252
      name: local
    volumeMounts:
    - mountPath: /etc/kubernetes
      name: pki
      readOnly: true
    - mountPath: /var/log
      name: logpath
    - mountPath: /sbin/modprobe
      name: modprobe
      readOnly: true
    - mountPath: /lib/modules
      name: modules
      readOnly: true
    - mountPath: /dev
      name: devices
  volumes:
  - hostPath:
      path: /etc/kubernetes
    name: pki
  - hostPath:
      path: /var/log
    name: logpath
  - hostPath:
      path: /sbin/modprobe
    name: modprobe
  - hostPath:
      path: /lib/modules
    name: modules
  - hostPath:
      path: /dev
    name: devices

kube-scheduler


apiVersion: v1
kind: Pod
metadata:
  name: kube-scheduler
spec:
  hostNetwork: true
  containers:
  - name: kube-scheduler
    image: 10.213.42.254:10500/root/hyperkube:v1.8.4-ceph
    command:
    - /bin/sh
    - -c
    - /hyperkube scheduler
      --kubeconfig=/etc/kubernetes/kubeconfig # yaml  --master=https://vip:443  ， ， kubeconfig 
      --v=0
      --logtostderr=true
      --alsologtostderr=false
      --log-dir=/var/log/kubenetes
      --leader-elect=true >> /var/log/kubernetes/kube-scheduler.log 2>&1
    ports:
    - containerPort: 10251
      hostPort: 10251
      name: local
    volumeMounts:
    - mountPath: /etc/kubernetes
      name: pki
      readOnly: true
    - mountPath: /var/log
      name: logpath
    - mountPath: /etc/localtime
      name: localtime
  volumes:
    - hostPath:
        path: /etc/kubernetes
      name: pki
    - hostPath:
        path: /var/log
      name: logpath
    - hostPath:
        path: /etc/localtime
      name: localtime

masterの/etc/kubernetes/kubeconfig

apiVersion: v1
kind: Config
users:
- name: controllermanager
  user:
    client-certificate: /etc/kubernetes/pki/cs_client.crt
    client-key: /etc/kubernetes/pki/cs_client.key
clusters:
- name: local
  cluster:
    server: https://10.209.202.200:443 # controller-manager scheduler yaml  --master=https://vip:443  ， ， ，  https://  。
    certificate-authority: /etc/kubernetes/pki/ca.crt
contexts:
- context:
    cluster: local
    user: controllermanager
  name: my-context
current-context: my-context

kubelet.service

[Unit]
Description=Kubernetes Kubelet Server
Documentation=http://kubernetes.io/docs/admin/kubelet/
After=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
EnvironmentFile=-/etc/default/kube-default
EnvironmentFile=-/etc/default/kubelet
ExecStart=/bin/sh -c '/usr/local/bin/kubelet \
        --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin \
        --hostname-override=10.209.228.18 \
        --kubeconfig=/etc/kubernetes/kubeconfig \ #  --master=https://vip:443  ， ， kubeconfig 
        --pod-manifest-path=/etc/kubernetes/manifests \
        --require-kubeconfig=true \
        --logtostderr=true \
        --pod-infra-container-image=10.213.42.254:10500/pause:3.0 \
        --cluster-dns=10.0.0.10 \
        --cluster-domain=k8s.wanda.com \
        --max-pods=110 \
        --cgroup-driver=cgroupfs \
        --fail-swap-on=false \
        --runtime-cgroups=/systemd/system.slice \
        --kubelet-cgroups=/systemd/system.slice \
        --allow-privileged=true -v=0 >> /var/log/kubernetes/kubelet.log 2>&1'
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

Nodeノードの/etc/kubernetes/kubeconfig

apiVersion: v1
kind: Config
users:
- name: kubelet
  user:
    client-certificate: /etc/kubernetes/ssl/kubelet_client.crt
    client-key: /etc/kubernetes/ssl/kubelet_client.key
clusters:
- name: local
  cluster:
    server: https://10.209.202.200:443 # kubelet.service  --api_servers=https://vip:443  ， ， ， --api-server   server ，  https://  。
    certificate-authority: /etc/kubernetes/ssl/ca.crt
contexts:
- context:
    cluster: local
    user: kubelet
  name: my-context
current-context: my-context

kube-proxy.yaml

apiVersion: v1
kind: Pod
metadata:
  name: kube-proxy
spec:
  hostNetwork: true
  containers:
  - name: kube-proxy
    image: 10.213.42.254:10500/root/hyperkube:v1.8.4-ceph
    command:
    - /bin/sh
    - -c
    - /hyperkube proxy
      --logtostderr=true
      --proxy-mode=iptables
      --master=https://10.209.202.200:443 # ， kubeconfig server ， server --api-servers。 server --master？？ 。
      --kubeconfig=/etc/kubernetes/kubeconfig
      -v=4
      --conntrack-tcp-timeout-established=1200s  >> /var/log/kubernetes/kube-proxy.log 2>&1
    securityContext:
      privileged: true
    volumeMounts:
    - mountPath: /etc/kubernetes
      name: pki
      readOnly: true
    - mountPath: /var/log
      name: logpath
    - mountPath: /etc/localtime
      name: localtime
  volumes:
  - hostPath:
      path: /etc/kubernetes
    name: pki
  - hostPath:
      path: /var/log
    name: logpath
  - hostPath:
      path: /etc/localtime
    name: localtime
一般的なテクニック-テスト
pytestの件01_実行結果内容改造