手动安装K8s第三节：etcd集群部署

jrgf

　　手动安装K8s第三节：etcd集群部署
　　准备安装包
https://github.com/coreos/etcd
版本：3.2.18
wget https://github.com/coreos/etcd/releases/download/v3.2.18/etcd-v3.2.18-linux-amd64.tar.gz
　　0、安装
[root@k8smaster src]# tar zxf etcd-v3.2.18-linux-amd64.tar.gz
[root@k8smaster src]# cd etcd-v3.2.18-linux-amd64
[root@k8smaster etcd-v3.2.18-linux-amd64]# ls
Documentation  etcd  etcdctl  README-etcdctl.md  README.md  READMEv2-etcdctl.md
[root@k8smaster etcd-v3.2.18-linux-amd64]#
　　[root@k8smaster etcd-v3.2.18-linux-amd64]# cp etcd etcdctl /opt/kubernetes/bin/
[root@k8smaster etcd-v3.2.18-linux-amd64]# scp etcd etcdctl k8snode1:/opt/kubernetes/bin/
[root@k8smaster etcd-v3.2.18-linux-amd64]# scp etcd etcdctl k8snode2:/opt/kubernetes/bin/
　　1、创建 etcd 证书签名请求：
[root@k8smaster ssl]# pwd
/usr/local/src/ssl
　　[root@k8smaster ssl]# vim etcd-csr.json
　　{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.137.171",
"192.168.137.201",
"192.168.137.198"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
　　2、生成 etcd 证书和私钥：
[root@k8smaster ssl]# cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem \
-ca-key=/opt/kubernetes/ssl/ca-key.pem \
-config=/opt/kubernetes/ssl/ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd
　　2018/06/12 17:49:55 [INFO] generate received request
2018/06/12 17:49:55 [INFO] received CSR
2018/06/12 17:49:55 [INFO] generating key: rsa-2048
2018/06/12 17:49:56 [INFO] encoded CSR
2018/06/12 17:49:56 [INFO] signed certificate with serial number 436752967180796317929734672346870544021189509062
2018/06/12 17:49:56 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
*注意：上面WARRING忽略
　　生成证书：etcd.per，etcd-key.pem
[root@k8smaster ssl]# ll
-rw-r--r-- 1 root root 1062 Jun 12 17:49 etcd.csr
-rw-r--r-- 1 root root  293 Jun 12 17:46 etcd-csr.json
-rw------- 1 root root 1679 Jun 12 17:49 etcd-key.pem
-rw-r--r-- 1 root root 1436 Jun 12 17:49 etcd.pem
　　3、将证书移动到/opt/kubernetes/ssl目录下
[root@k8smaster ssl]# cp etcd.pem /opt/kubernetes/ssl
[root@k8smaster ssl]# scp etcd.pem k8snode1:/opt/kubernetes/ssl   
[root@k8smaster ssl]# scp etcd*.pem k8snode2:/opt/kubernetes/ssl
　　[root@k8smaster ssl]# rm -f etcd.csr etcd-csr.json
　　4、设置ETCD配置文件
[root@k8smaster ~]# vim /opt/kubernetes/cfg/etcd.conf
　　#[member]
ETCD_NAME="etcd-node1"                                       #修改为不同的名字
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_SNAPSHOT_COUNTER="10000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="https://192.168.137.171:2380"                                              #修改为本机IP地址
ETCD_LISTEN_CLIENT_URLS="https://192.168.137.171:2379,https://127.0.0.1:2379"       #修改为本机IP地址
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
#ETCD_CORS=""
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.137.171:2380"        #修改为本机IP地址
if you use different ETCD_NAME (e.g. test),
set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
　　ETCD_INITIAL_CLUSTER="etcd-node1=https://192.168.137.171:2380,etcd-node2=https://192.168.137.201:2380,etcd-node3=https://192.168.137.198:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.137.171:2379"           #修改为本机IP地址
#[security]
CLIENT_CERT_AUTH="true"
ETCD_CA_FILE="/opt/kubernetes/ssl/ca.pem"
ETCD_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"
ETCD_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"
PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CA_FILE="/opt/kubernetes/ssl/ca.pem"
ETCD_PEER_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"
　　将配置文件复制到其它节点，按上面标注进行修改
[root@k8smaster ~]# scp /opt/kubernetes/cfg/etcd.conf k8snode1:/opt/kubernetes/cfg/etcd.conf
[root@k8smaster ~]# scp /opt/kubernetes/cfg/etcd.conf k8snode2:/opt/kubernetes/cfg/etcd.conf
　　创建etcd.service
[root@k8smaster ~]# vim /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
　　[Service]
Type=simple
WorkingDirectory=/var/lib/etcd
EnvironmentFile=-/opt/kubernetes/cfg/etcd.conf
set GOMAXPROCS to number of processors
　　ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /opt/kubernetes/bin/etcd"
Type=notify
　　[Install]
WantedBy=multi-user.target
　　6、重新加载系统服务
[root@k8smaster ~]# scp /etc/systemd/system/etcd.service k8snode1:/etc/systemd/system/
[root@k8smaster ~]# scp /etc/systemd/system/etcd.service k8snode2:/etc/systemd/system/
　　在所有节点上创建etcd存储目录并启动etcd
systemctl daemon-reload
mkdir /var/lib/etcd
systemctl start etcd
systemctl enable etcd
systemctl status etcd
　　7、验证集群
[root@k8smaster ~]# etcdctl --endpoints=https://192.168.137.171:2379 \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem cluster-health
　　member 3365586f3ad09ed8 is healthy: got healthy result from https://192.168.137.201:2379
member 80ecb1efc62e0556 is healthy: got healthy result from https://192.168.137.171:2379
member 851e1a89f7240e3a is healthy: got healthy result from https://192.168.137.198:2379
cluster is healthy
　　[root@k8smaster ~]# etcdctl --endpoints=https://192.168.137.171:2379 \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem member list
　　3365586f3ad09ed8: name=etcd-node2 peerURLs=https://192.168.137.201:2380 clientURLs=https://192.168.137.201:2379 isLeader=false
80ecb1efc62e0556: name=etcd-node1 peerURLs=https://192.168.137.171:2380 clientURLs=https://192.168.137.171:2379 isLeader=true
851e1a89f7240e3a: name=etcd-node3 peerURLs=https://192.168.137.198:2380 clientURLs=https://192.168.137.198:2379 isLeader=false
　　[root@k8smaster ~]# vim etcdctl.sh
etcdctl cluster-health
　　etcdctl --endpoints=https://192.168.137.171:2379 \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem cluster-health
etcdctl member list
　　etcdctl --endpoints=https://192.168.137.171:2379 \
--ca-file=/opt/kubernetes/ssl/ca.pem \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem member list

　　在node节点上查看端口状态
[root@k8smaster ~]# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name   
tcp        0      0 192.168.137.171:2379    0.0.0.0:               LISTEN      3955/etcd           
tcp        0      0 127.0.0.1:2379          0.0.0.0:               LISTEN      3955/etcd           
tcp        0      0 192.168.137.171:2380    0.0.0.0:               LISTEN      3955/etcd           
tcp        0      0 0.0.0.0:22              0.0.0.0:               LISTEN      1154/sshd           
tcp        0      0 127.0.0.1:25            0.0.0.0:               LISTEN      1239/master         
tcp6       0      0 :::22                   :::                    LISTEN      1154/sshd           
tcp6       0      0 ::1:25                  :::*                    LISTEN      1239/master