kubernetes 二进制安装(v1.20.16)（二）部署 etcd 集群

文章目录
生成etcd证书
自签CA申请文件
生成自签CA证书
创建etcd证书申请文件
签发Etcd HTTPS证书
部署Etcd集群
生成etcd管理文件
分发文件
核对文件
启动etcd集群
查看etcd集群状态

从这里开始就要跟各种各样的证书打交道了，等写完这个部署系列我专门整理一篇写证书的。

那么，打起十二分精神，一不小心就要重头再来。不过也有一个好的地方就是，安装完一块，就是一块。

生成etcd证书

自签CA申请文件

代码语言：javascript复制

cd /opt/TLS/etcd/ssl
cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat > ca-csr.json << EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

生成自签CA证书

代码语言：javascript复制

[root@k8s-master ssl]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
....
[root@k8s-master ssl]# ll
total 20
-rw-r--r-- 1 root root  287 Apr  4 08:51 ca-config.json
-rw-r--r-- 1 root root  956 Apr  4 08:51 ca.csr
-rw-r--r-- 1 root root  209 Apr  4 08:51 ca-csr.json
-rw------- 1 root root 1679 Apr  4 08:51 ca-key.pem
-rw-r--r-- 1 root root 1216 Apr  4 08:51 ca.pem
#上述操作，会生成ca.pem和ca-key.pem两个文件

创建etcd证书申请文件

代码语言：javascript复制

cat > server-csr.json << EOF
{
    "CN": "etcd",
    "hosts": [
    "192.168.190.147",
    "192.168.190.148"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF
#上述文件hosts字段中IP为所有etcd节点的集群内部通信IP，为了方便后期扩容可以多写几个预留的IP

签发Etcd HTTPS证书

代码语言：javascript复制

[root@k8s-master ssl]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
....
[root@k8s-master ssl]# ll
total 36
-rw-r--r-- 1 root root  287 Apr  4 08:51 ca-config.json
-rw-r--r-- 1 root root  956 Apr  4 08:51 ca.csr
-rw-r--r-- 1 root root  209 Apr  4 08:51 ca-csr.json
-rw------- 1 root root 1679 Apr  4 08:51 ca-key.pem
-rw-r--r-- 1 root root 1216 Apr  4 08:51 ca.pem
-rw-r--r-- 1 root root 1013 Apr  4 08:55 server.csr
-rw-r--r-- 1 root root  290 Apr  4 08:55 server-csr.json
-rw------- 1 root root 1675 Apr  4 08:55 server-key.pem
-rw-r--r-- 1 root root 1338 Apr  4 08:55 server.pem
#上述操作会生成server.pem和server-key.pem两个文件

部署Etcd集群

代码语言：javascript复制

#这里为了方便操作，同时生成了2个etcd虚拟机上的配置文件，然后将各自的配置文件分发至不同的虚拟机，减少了修改的操作。
cd /opt/TLS/etcd/cfg

#-------------------------------------
#生成k8s-master虚拟机上对应的配置文件
#-------------------------------------
cat > etcd_master.conf << EOF
#[Member]
ETCD_NAME="etcd-master"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.190.147:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.190.147:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.190.147:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.190.147:2379"
ETCD_INITIAL_CLUSTER="etcd-master=https://192.168.190.147:2380,etcd-node1=https://192.168.190.148:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

#-------------------------------------
#生成k8s-node1虚拟机上对应的配置文件
#-------------------------------------
cat > etcd_node1.conf << EOF
#[Member]
ETCD_NAME="etcd-node1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.190.148:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.190.148:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.190.148:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.190.148:2379"
ETCD_INITIAL_CLUSTER="etcd-master=https://192.168.190.147:2380,etcd-node1=https://192.168.190.148:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

代码语言：javascript复制

#查看已生成的配置文件清单列表
[root@k8s-master cfg]# ll
total 16
-rw-r--r-- 1 root root 509 Apr  4 09:05 etcd_master.conf
-rw-r--r-- 1 root root 509 Apr  4 09:05 etcd_node1conf

#---------------------------备注说明-------------------------------
# • ETCD_NAME：节点名称，集群中唯一
# • ETCD_DATA_DIR：数据目录
# • ETCD_LISTEN_PEER_URLS：集群通信监听地址
# • ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址
# • ETCD_INITIAL_ADVERTISE_PEERURLS：集群通告地址
# • ETCD_ADVERTISE_CLIENT_URLS：客户端通告地址
# • ETCD_INITIAL_CLUSTER：集群节点地址
# • ETCD_INITIALCLUSTER_TOKEN：集群Token
# • ETCD_INITIALCLUSTER_STATE：加入集群的当前状态，new是新集群，existing表示加入已有集群
#-----------------------------------------------------------------

生成etcd管理文件

代码语言：javascript复制

cd /opt/TLS/etcd/cfg

cat > etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd 
--cert-file=/opt/etcd/ssl/server.pem 
--key-file=/opt/etcd/ssl/server-key.pem 
--peer-cert-file=/opt/etcd/ssl/server.pem 
--peer-key-file=/opt/etcd/ssl/server-key.pem 
--trusted-ca-file=/opt/etcd/ssl/ca.pem 
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem 
--logger=zap
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#查看已生成的文件列表清单
[root@k8s-master cfg]# ll
total 16
-rw-r--r-- 1 root root 509 Apr  4 09:05 etcd_master.conf
-rw-r--r-- 1 root root 509 Apr  4 09:05 etcd_node1.conf
-rw-r--r-- 1 root root 535 Apr  4 09:05 etcd.service

分发文件

代码语言：javascript复制

#创建etcd运行时所需的目录
mkdir -p /var/lib/etcd/default.etcd
ssh k8s-node1 "mkdir -p /var/lib/etcd/default.etcd"

#创建ecd配置文件目录
mkdir -p /opt/etcd/{bin,cfg,ssl}
ssh k8s-node1 "mkdir -p /opt/etcd/{bin,cfg,ssl}"

#分发etcd可执行文件
scp -r /opt/TLS/download/etcd-v3.4.9-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/
scp -r /opt/TLS/download/etcd-v3.4.9-linux-amd64/{etcd,etcdctl} k8s-node1:/opt/etcd/bin/

#分发etcd配置文件
scp -r /opt/TLS/etcd/cfg/etcd01.conf /opt/etcd/cfg/etcd.conf
scp -r /opt/TLS/etcd/cfg/etcd02.conf k8s-node1:/opt/etcd/cfg/etcd.conf

#分发etcd管理文件
scp -r /opt/TLS/etcd/cfg/etcd.service /usr/lib/systemd/system/etcd.service
scp -r /opt/TLS/etcd/cfg/etcd.service k8s-node1:/usr/lib/systemd/system/etcd.service

#分发etcd证书文件
scp -r /opt/TLS/etcd/ssl/*pem /opt/etcd/ssl
scp -r /opt/TLS/etcd/ssl/*pem k8s-node1:/opt/etcd/ssl

核对文件

代码语言：javascript复制

#核对etcd可执行文件
[root@k8s-master cfg]# ls -l /opt/etcd/bin/
total 40472
-rwxr-xr-x 1 root root 23827424 Apr  3 12:38 etcd
-rwxr-xr-x 1 root root 17612384 Apr  3 12:38 etcdctl
[root@k8s-master cfg]# ssh k8s-node1 "ls -l /opt/etcd/bin/"
total 40472
-rwxr-xr-x 1 root root 23827424 Apr  3 12:38 etcd
-rwxr-xr-x 1 root root 17612384 Apr  3 12:38 etcdctl

#核对etcd配置文件
[root@k8s-master cfg]# ls -l /opt/etcd/cfg/
total 4
-rw-r--r-- 1 root root 509 Apr  3 12:38 etcd.conf
[root@k8s-master cfg]# ssh k8s-node1 "ls -l /opt/etcd/cfg/"
total 4
-rw-r--r-- 1 root root 509 Apr  3 12:38 etcd.conf

#核对etcd管理文件
[root@k8s-master cfg]# ls -l /usr/lib/systemd/system/etcd*
-rw-r--r-- 1 root root 535 Apr  3 12:39 /usr/lib/systemd/system/etcd.service
[root@k8s-master cfg]# ssh k8s-node1 "ls -l /usr/lib/systemd/system/etcd*"
-rw-r--r-- 1 root root 535 Apr  3 12:39 /usr/lib/systemd/system/etcd.service

#核对etcd证书文件
[root@k8s-master cfg]# ls -l /opt/etcd/ssl
total 16
-rw------- 1 root root 1679 Apr  3 12:39 ca-key.pem
-rw-r--r-- 1 root root 1216 Apr  3 12:39 ca.pem
-rw------- 1 root root 1675 Apr  3 12:39 server-key.pem
-rw-r--r-- 1 root root 1338 Apr  3 12:39 server.pem
[root@k8s-master cfg]# ssh k8s-node1 "ls -l /opt/etcd/ssl"
total 16
-rw------- 1 root root 1679 Apr  3 12:39 ca-key.pem
-rw-r--r-- 1 root root 1216 Apr  3 12:39 ca.pem
-rw------- 1 root root 1675 Apr  3 12:39 server-key.pem
-rw-r--r-- 1 root root 1338 Apr  3 12:39 server.pem

启动etcd集群

代码语言：javascript复制

#按顺序分别在k8s-master、k8s-node1上执行以下命令，其中在k8s-master上执行命令时会有等待现象，主要是等待其他机器的状态

#在k8s-master上执行启动命令，并设置开机启动，同时查看etcd状态
[root@k8s-master cfg]# systemctl daemon-reload && systemctl start etcd && systemctl enable etcd && systemctl status etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
● etcd.service - Etcd Server
   Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: active (running) since Sun 2022-04-03 12:52:39 CST; 83ms ago
 Main PID: 1281 (etcd)
   CGroup: /system.slice/etcd.service
           └─1281 /opt/etcd/bin/etcd --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem -...

Apr 03 12:52:39 k8s-master etcd[1281]: {"level":"info","ts":"2022-04-03T12:52:39.282 0800","caller":"raft/node.go:325","msg":"raft.node: 6571fb7574e87dba elected leader 6571fb7574e87dba at term 4"}
Apr 03 12:52:39 k8s-master etcd[1281]: {"level":"info","ts":"2022-04-03T12:52:39.290 0800","caller":"etcdserver/server.go:2036","msg":"published local member to cluster through raft","local-member-id":"6571fb...
....

#在k8s-node1上执行启动命令，并设置开机启动，同时查看etcd状态
[root@k8s-node1 ~]# systemctl daemon-reload && systemctl start etcd && systemctl enable etcd && systemctl status etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
● etcd.service - Etcd Server
   Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: active (running) since Sun 2022-04-03 12:52:41 CST; 76ms ago
 Main PID: 1188 (etcd)
   CGroup: /system.slice/etcd.service
           └─1188 /opt/etcd/bin/etcd --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem -...

Apr 03 12:52:41 k8s-node1 etcd[1188]: {"level":"info","ts":"2022-04-03T12:52:41.311 0800","caller":"raft/raft.go:811","msg":"9b449b0ff1d4c375 [logterm: 1, index: 3] sent MsgVote request to d1f...e5c at term 2"}
....
Hint: Some lines were ellipsized, use -l to show in full.

查看etcd集群状态

这一步很重要，前面如果哪里写错了，说不定也能运行的下去，但是如果前面真的哪里写错了，这一步是走不通的。我遇到的几个问题解决方案已经发在（问题解决）专栏里了。

代码语言：javascript复制

#在任意一台集群上执行以下命令：

ETCDCTL_API=3 /opt/etcd/bin/etcdctl 
--cacert=/opt/etcd/ssl/ca.pem 
--cert=/opt/etcd/ssl/server.pem 
--key=/opt/etcd/ssl/server-key.pem 
--write-out=table 
--endpoints="https://192.168.190.147:2379,https://192.168.190.148:2379" endpoint health

#返回结果
 ---------------------------- -------- ------------- ------- 
|          ENDPOINT          | HEALTH |    TOOK     | ERROR |
 ---------------------------- -------- ------------- ------- 
| https://192.168.190.147:2379 |   true | 10.702229ms |       |
| https://192.168.10.13:2379 |   true |  18.81801ms |       |
| https://192.168.190.148:2379 |   true | 18.017598ms |       |
 ---------------------------- -------- ------------- -------

etcd 数据分析 https 网络安全

0 人点赞

kubernetes 二进制安装(v1.20.16)（二）部署 etcd 集群

文章目录生成etcd证书自签CA申请文件生成自签CA证书创建etcd证书申请文件签发Etcd HTTPS证书部署Etcd集群生成etcd管理文件分发文件核对文件启动etcd集群查看etcd集群状态

生成etcd证书

自签CA申请文件

生成自签CA证书

创建etcd证书申请文件

签发Etcd HTTPS证书

部署Etcd集群

生成etcd管理文件

分发文件

核对文件

启动etcd集群

查看etcd集群状态

文章目录
生成etcd证书
自签CA申请文件
生成自签CA证书
创建etcd证书申请文件
签发Etcd HTTPS证书
部署Etcd集群
生成etcd管理文件
分发文件
核对文件
启动etcd集群
查看etcd集群状态