[root@docker1 ~]# etcd --version
etcd Version: 3.2.18
Git SHA: eddf599
Go Version: go1.9.4
Go OS/Arch: linux/amd64
[root@docker1 ~]#
[root@docker1 ~]# etcdctl set test "a"
a
[root@docker1 ~]# etcdctl get test
a
[root@docker1 ~]#
2.2
安装配置flannel
[root@docker1 ~]# yum -y install flannel
启动
[root@docker1 ~]# systemctl start flanneld
报错
[root@docker1 ~]# systemctl status flanneld -l
● flanneld.service - Flanneld overlay address etcd agent
Loaded: loaded (/usr/lib/systemd/system/flanneld.service; disabled; vendor preset: disabled)
Active: activating (start) since Thu 2018-06-14 02:22:26 EDT; 1min 1s ago
Main PID: 2950 (flanneld)
Memory: 16.6M
CGroup: /system.slice/flanneld.service
└─2950 /usr/bin/flanneld -etcd-endpoints=http://127.0.0.1:2379 -etcd-prefix=/atomic.io/network
Jun 14 02:23:18 docker1 flanneld-start[2950]: E0614 02:23:18.974351 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:19 docker1 flanneld-start[2950]: E0614 02:23:19.977497 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:20 docker1 flanneld-start[2950]: E0614 02:23:20.980721 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:21 docker1 flanneld-start[2950]: E0614 02:23:21.983553 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:22 docker1 flanneld-start[2950]: E0614 02:23:22.988446 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:23 docker1 flanneld-start[2950]: E0614 02:23:23.992106 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:24 docker1 flanneld-start[2950]: E0614 02:23:24.994719 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:25 docker1 flanneld-start[2950]: E0614 02:23:25.998629 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:27 docker1 flanneld-start[2950]: E0614 02:23:27.002486 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
Jun 14 02:23:28 docker1 flanneld-start[2950]: E0614 02:23:28.006185 2950 network.go:102] failed to retrieve network config: 100: Key not found (/atomic.io) [11]
注意-etcd-prefix=/automic.io/network
flanel读取的网络配置是这个文件,这个文件是在
[root@docker1 ~]# cat /usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/flanneld
EnvironmentFile=-/etc/sysconfig/docker-network
ExecStart=/usr/bin/flanneld-start $FLANNEL_OPTIONS
ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure
[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
[root@docker1 sysconfig]# cat flanneld
# Flanneld configuration options
# etcd url location. Point this to the server where etcd runs
FLANNEL_ETCD_ENDPOINTS="http://127.0.0.1:2379"
# etcd config key. This is the configuration key that flannel queries
# For address range assignment
FLANNEL_ETCD_PREFIX="/atomic.io/network"
# Any additional options that you want to pass
#FLANNEL_OPTIONS=""
注意:
FLANNEL_ETCD_PREFIX="/atomic.io/network"
这个FLANNEL_ETCD_PREFIX需要etcdctl手动去建立
[root@docker1 ~]# systemctl restart etcd
[root@docker1 ~]# systemctl status etcd
● etcd.service - Etcd Server
Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
Active: active (running) since Thu 2018-06-14 04:38:49 EDT; 1min 11s ago
Main PID: 3401 (etcd)
Memory: 21.5M
CGroup: /system.slice/etcd.service
└─3401 /usr/bin/etcd --name=default --data-dir=/var/lib/etcd/default.etcd --listen-client-urls=http://0.0.0.0:23...
Jun 14 04:38:48 docker1 etcd[3401]: enabled capabilities for version 3.2
Jun 14 04:38:49 docker1 etcd[3401]: 8e9e05c52164694d is starting a new election at term 9
Jun 14 04:38:49 docker1 etcd[3401]: 8e9e05c52164694d became candidate at term 10
Jun 14 04:38:49 docker1 etcd[3401]: 8e9e05c52164694d received MsgVoteResp from 8e9e05c52164694d at term 10
Jun 14 04:38:49 docker1 etcd[3401]: 8e9e05c52164694d became leader at term 10
Jun 14 04:38:49 docker1 etcd[3401]: raft.node: 8e9e05c52164694d elected leader 8e9e05c52164694d at term 10
Jun 14 04:38:49 docker1 etcd[3401]: published {Name:default ClientURLs:[http://192.168.211.140:2379]} to cluster cdf8...3a8c32
Jun 14 04:38:49 docker1 etcd[3401]: ready to serve client requests
Jun 14 04:38:49 docker1 systemd[1]: Started Etcd Server.
Jun 14 04:38:49 docker1 etcd[3401]: serving insecure client requests on [::]:2379, this is strongly discouraged!
Hint: Some lines were ellipsized, use -l to show in full.
[root@docker1 ~]#
再启动还是报错
[root@docker2 ~]# systemctl status flanneld -l
● flanneld.service - Flanneld overlay address etcd agent
Loaded: loaded (/usr/lib/systemd/system/flanneld.service; disabled; vendor preset: disabled)
Active: inactive (dead)
Jun 14 04:21:53 docker2 flanneld-start[2706]: E0614 04:21:53.879476 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:21:54 docker2 flanneld-start[2706]: E0614 04:21:54.880962 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:21:55 docker2 flanneld-start[2706]: E0614 04:21:55.882332 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:21:56 docker2 flanneld-start[2706]: E0614 04:21:56.887002 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:21:57 docker2 flanneld-start[2706]: E0614 04:21:57.888246 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:21:58 docker2 flanneld-start[2706]: E0614 04:21:58.889903 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:21:59 docker2 flanneld-start[2706]: E0614 04:21:59.891323 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:22:00 docker2 flanneld-start[2706]: E0614 04:22:00.892229 2706 network.go:102] failed to retrieve network config: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:2379: getsockopt: connection refused
Jun 14 04:22:01 docker2 systemd[1]: Stopped Flanneld overlay address etcd agent.
Jun 14 04:22:01 docker2 flanneld-start[2706]: I0614 04:22:01.105679 2706 main.go:172] Exiting...
[root@docker2 ~]#
拒绝连接,应该是防火墙的问题了
关闭docker1的防火墙
[root@docker2 ~]# flanneld -etcd-endpoints=http://192.168.211.140:2379 -iface=ens33 -etcd-prefix=/atomic.io/network &
[1] 2938
[root@docker2 ~]# I0614 04:44:03.522494 2938 main.go:132] Installing signal handlers
I0614 04:44:03.523151 2938 manager.go:149] Using interface with name ens33 and address 192.168.211.154
I0614 04:44:03.523174 2938 manager.go:166] Defaulting external address to interface address (192.168.211.154)
I0614 04:44:03.530498 2938 local_manager.go:134] Found lease (172.17.41.0/24) for current IP (192.168.211.154), reusing
I0614 04:44:03.546625 2938 manager.go:250] Lease acquired: 172.17.41.0/24
I0614 04:44:03.547228 2938 network.go:98] Watching for new subnet leases
I0614 04:44:03.558669 2938 network.go:191] Subnet added: 172.17.21.0/24
[root@docker2 ~]# ip a |grep flannel
8: flannel0: mtu 1472 qdisc pfifo_fast state UNKNOWN group default qlen 500
inet 172.17.41.0/16 scope global flannel0
[root@docker2 ~]#
启动了
有点奇怪,重启docker1后,可以进来了,不需要添加开放2379的规则
2.4
docker3做一样的操作
3.分析flannel网络
[root@docker2 ~]# ip r
default via 192.168.211.2 dev ens33 proto dhcp metric 100
172.17.0.0/16 dev flannel.1
172.17.65.0/24 dev docker0 proto kernel scope link src 172.17.65.1
172.18.0.0/16 dev docker_gwbridge proto kernel scope link src 172.18.0.1
192.168.211.0/24 dev ens33 proto kernel scope link src 192.168.211.154 metric 100
[root@docker2 ~]# brctl show
bridge name bridge id STP enabled interfaces
docker0 8000.02427f17e635 no veth7680282
docker_gwbridge 8000.024266f01344 no
[root@docker2 ~]#
没有生成新的网桥,使用默认的docker0
172.17.65.0 同一docker主机 容器通过docker0连接
172.17.0.0 不同docker主机 容器通过flannel.1转发
3.3
容器连接flannel网络
[root@docker2 ~]# docker run -itd --name bbox1 busybox
dc344e4b30e48bbc5b914edc3724e43d56fa0ee7abf97246dc35ce57b6cf872c
[root@docker2 ~]# docker exec bbox1 ip r
default via 172.17.65.1 dev eth0
172.17.65.0/24 dev eth0 scope link src 172.17.65.2
[root@docker2 ~]#
[root@docker3 ~]# docker run -itd --name bbox2 busybox
e9e824f93e8dedb498f436b169ed8dcb85bc951f4d05ae4f254aad1e3d538a8a
[root@docker3 ~]# docker exec bbox2 ip r
default via 172.17.16.1 dev eth0
172.17.16.0/24 dev eth0 scope link src 172.17.16.2
[root@docker3 ~]#
互ping
[root@docker2 ~]# docker exec bbox1 ping -c 5 172.17.16.2
PING 172.17.16.2 (172.17.16.2): 56 data bytes
64 bytes from 172.17.16.2: seq=0 ttl=60 time=3.274 ms
64 bytes from 172.17.16.2: seq=1 ttl=60 time=1.356 ms
^C
[root@docker2 ~]#
[root@docker3 ~]# docker exec bbox2 ping -c 5 172.17.65.2
PING 172.17.65.2 (172.17.65.2): 56 data bytes
64 bytes from 172.17.65.2: seq=0 ttl=60 time=2.995 ms
64 bytes from 172.17.65.2: seq=1 ttl=60 time=1.396 ms
64 bytes from 172.17.65.2: seq=2 ttl=60 time=1.650 ms
^C
[root@docker3 ~]#
分析数据流
从bbox1 ping 172.17.16.2
看看它的默认路由
[root@docker2 ~]# docker exec bbox1 ip r
default via 172.17.65.1 dev eth0
172.17.65.0/24 dev eth0 scope link src 172.17.65.2
[root@docker2 ~]#
目的地址 172.17.16.2不在直连网络,因此数据包从default路由出。default路由的地址时 172.17.65.1,这个地址就是docker0的地址
[root@docker2 ~]# ip a |grep docker0
4: docker0: mtu 1450 qdisc noqueue state UP group default
inet 172.17.65.1/24 brd 172.17.65.255 scope global docker0
24: veth63d6978@if23: mtu 1450 qdisc noqueue master docker0 state UP group default
[root@docker2 ~]#
数据到达docker0后,发现这个数据包的地址是172.17.16.2,并不是给自己,寻找下一跳
看看node上的路由表
[root@docker2 ~]# ip r
default via 192.168.211.2 dev ens33 proto dhcp metric 100
172.17.0.0/16 dev flannel.1
172.17.65.0/24 dev docker0 proto kernel scope link src 172.17.65.1
172.18.0.0/16 dev docker_gwbridge proto kernel scope link src 172.18.0.1
192.168.211.0/24 dev ens33 proto kernel scope link src 192.168.211.154 metric 100
[root@docker2 ~]#
匹配到172.17.0.0/16这条路由,这是直连路由,数据送到flannel.1上
flannel.1收到数据包,自己不是目的地,需要发数据发送出去,数据包沿着网络协议向下流动,在二层封装以太包,填写目的mac地址。
这时发出arp“who is 172.17.16.2"
flannel.1是vxlan设备,vxlan并不会在二层发arp包,而是由linux kernel的 “L3 MISS"事件,将arp发到用户空间和flanneld程序。
linux kernel的 “L3 MISS"事件参数由下面的参数设置
[root@docker2 ~]# cat /proc/sys/net/ipv4/neigh/flannel.1/app_solicit
3
[root@docker2 ~]#
flanneld程序收到“L3 MISS”内核事件以及ARP请求后,并不会向外网发送arp request,而是从etcd查找匹配该地址的子网的vtep信息。
[root@docker3 ~]# ip -d link show
11: flannel.1: mtu 1450 qdisc noqueue state UNKNOWN mode DEFAULT group default
link/ether d2:72:c5:90:ff:8c brd ff:ff:ff:ff:ff:ff promiscuity 0
vxlan id 1 local 192.168.211.153 dev ens33 srcport 0 0 dstport 8472 nolearning ageing 300 noudpcsum noudp6zerocsumtx noudp6zerocsumrx
可以看到docker3的flannel.1的mac地址就是
找到目的地后,flanneld将查询到的信息存入arp缓存
[root@docker2 ~]# ip n
192.168.211.254 dev ens33 lladdr 00:50:56:ea:e1:f4 STALE
172.17.16.2 dev flannel.1 lladdr d2:72:c5:90:ff:8c STALE
192.168.211.2 dev ens33 lladdr 00:50:56:fd:b3:89 STALE
192.168.211.153 dev ens33 lladdr 00:0c:29:93:2c:89 STALE
192.168.211.1 dev ens33 lladdr 00:50:56:c0:00:08 DELAY
172.17.65.2 dev docker0 lladdr 02:42:ac:11:41:02 STALE
192.168.211.140 dev ens33 lladdr 00:0c:29:f9:b7:d2 DELAY
[root@docker2 ~]#
最后封装vxlan包,发送到目的地
4.
flannel 为每个主机分配了独立的 subnet但 flannel.1 将这些 subnet 连接起来了相互之间可以路由。本质上flannel 将各主机上相互独立的 docker0 容器网络组成了一个互通的大网络实现了容器跨主机通信。flannel 没有提供隔离。
5.
host-gw
flannel支持多种backend,host-gw是flannel的另一种backend.
与vxlan不同,host-gw不会封装数据包.而是在主机路由表中创建到其他主机subnet的路由条目,实现容器跨主机通讯.
设置backend
修改下前面设置的,把backend改成host-gw
[root@docker2 ~]# ip r
default via 192.168.211.2 dev ens33 proto dhcp metric 100
169.254.0.0/16 dev ens33 scope link metric 1002
172.17.16.0/24 via 192.168.211.153 dev ens33
172.17.65.0/24 dev docker0 proto kernel scope link src 172.17.65.1
172.18.0.0/16 dev docker_gwbridge proto kernel scope link src 172.18.0.1
192.168.211.0/24 dev ens33 proto kernel scope link src 192.168.211.154 metric 100
[root@docker2 ~]#
[root@docker3 ~]# ip r
default via 192.168.211.2 dev ens33 proto dhcp metric 100
169.254.0.0/16 dev ens33 scope link metric 1002
172.17.16.0/24 dev docker0 proto kernel scope link src 172.17.16.1
172.17.65.0/24 via 192.168.211.154 dev ens33
172.18.0.0/16 dev docker_gwbridge proto kernel scope link src 172.18.0.1
192.168.211.0/24 dev ens33 proto kernel scope link src 192.168.211.153 metric 100
[root@docker3 ~]#
172.17.16.0/24 via 192.168.211.153 dev ens33 docker2
172.17.65.0/24 via 192.168.211.154 dev ens33 docker3
出现了相对应的路由条目
需要修改mtu并且重启docker
mtu值前面已经有过说明,根据下面的值来修改
[root@docker2 ~]# docker exec bbox1 ping 172.17.16.2
PING 172.17.16.2 (172.17.16.2): 56 data bytes
64 bytes from 172.17.16.2: seq=0 ttl=62 time=1.851 ms
64 bytes from 172.17.16.2: seq=1 ttl=62 time=0.946 ms
64 bytes from 172.17.16.2: seq=2 ttl=62 time=0.972 ms
^C
[root@docker2 ~]# docker exec bbox1 ip r
default via 172.17.65.1 dev eth0
172.17.65.0/24 dev eth0 scope link src 172.17.65.2
[root@docker2 ~]#
走默认路由172.17.65.1
[root@docker2 ~]# ip a |grep docker0
4: docker0: mtu 1500 qdisc noqueue state UP group default
inet 172.17.65.1/24 brd 172.17.65.255 scope global docker0
7: vetha9f972a@if6: mtu 1500 qdisc noqueue master docker0 state UP group default
[root@docker2 ~]#
数据到达docker0
[root@docker2 ~]# ip r
default via 192.168.211.2 dev ens33 proto dhcp metric 100
169.254.0.0/16 dev ens33 scope link metric 1002
172.17.16.0/24 via 192.168.211.153 dev ens33
172.17.65.0/24 dev docker0 proto kernel scope link src 172.17.65.1
172.18.0.0/16 dev docker_gwbridge proto kernel scope link src 172.18.0.1
192.168.211.0/24 dev ens33 proto kernel scope link src 192.168.211.154 metric 100
[root@docker2 ~]#
匹配到172.17.16.0/24 via 192.168.211.153 dev ens33
直接发送数据过去.
6.
vxlan和host-gw的简单比较
host-gw 把每个主机配置成网关,主机知道其他主机的subnet和转发地址.
vxlan是在主机间建立隧道.
不同的主机在一个大的网内.
vxlan需要对数据进行打包拆包,性能低于host-gw