master/slave的单主架构,同一时间只有一个Keepalived对外提供服务,此主机繁忙,而另一台主机却
很空闲,利用率低下,可以使用master/master的双主架构,解决此问题。
master/master 的双主架构:
即将两个或以上VIP分别运行在不同的keepalived服务器,以实现服务器并行提供web访问的目的,提高
服务器资源利用率
#ha1主机配置 [root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { root@qq.com } notification_email_from keepalived@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id ka1.longxuan.vip vrrp_mcast_group4 224.8.8.188 } vrrp_instance N520 { state MASTER #在另一个主机上为BACKUP interface eth0 virtual_router_id 66 #每个vrrp_instance唯一 priority 100 #在另一个主机上为80 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 #指定vrrp_instance各自的VIP } } vrrp_instance N521 { #添加 VI_2 实例 state BACKUP #在另一个主机上为MASTER interface eth0 virtual_router_id 88 #每个vrrp_instance唯一 priority 80 #在另一个主机上为100 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 #指定vrrp_instance各自的VIP } } #ka2主机配置,和ka1配置只需五行不同 [root@ka2-centos8 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { root@qq.com } notification_email_from keepalived@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id ka2.longxuan.vip #修改此行 vrrp_mcast_group4 224.8.8.100 } vrrp_instance N520 { state BACKUP #此修改行为BACKUP interface eth0 virtual_router_id 66 priority 80 #此修改行为80 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } } vrrp_instance N521 { state MASTER #修改此行为MASTER interface eth0 virtual_router_id 88 priority 100 #修改此行为100 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 } }
范例:ka1配置
[root@centos8 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { acassen } notification_email_from Alexandre.Cassen@firewall.loc smtp_server 192.168.200.1 smtp_connect_timeout 30 router_id LVS_DEVEL } include /etc/keepalived/conf.d/*.conf # 创建子目录 [root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p # 项目1-master [root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf vrrp_instance n520 { state MASTER interface eth0 virtual_router_id 66 priority 100 advert_int 1 #nopreempt #preempt_delay 10 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } unicast_src_ip 172.31.0.28 unicast_peer{ 172.31.0.48 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 项目2-backup [root@centos8 ~]# vim /etc/keepalived/conf.d/n521.conf vrrp_instance n521 { state BACKUP interface eth0 virtual_router_id 88 priority 80 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 } unicast_src_ip 172.31.0.28 unicast_peer{ 172.31.0.48 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 重启 [root@centos8 ~]# systemctl restart keepalived 检查ip [root@centos8 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 00:0c:29:ac:f5:a4 brd ff:ff:ff:ff:ff:ff inet 172.31.0.28/16 brd 172.31.255.255 scope global noprefixroute eth0 valid_lft forever preferred_lft forever inet 172.31.0.188/32 scope global eth0:1 valid_lft forever preferred_lft forever inet 172.31.0.200/32 scope global eth0:2
范例:ka2配置
[root@centos8 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { acassen } notification_email_from Alexandre.Cassen@firewall.loc smtp_server 192.168.200.1 smtp_connect_timeout 30 router_id LVS_DEVEL } include /etc/keepalived/conf.d/*.conf # 创建子目录 [root@centos8 ~]# mkdir /etc/keepalived/conf.d/ -p # 项目1-backup [root@centos8 ~]# cat /etc/keepalived/conf.d/n520.conf vrrp_instance n520 { state BACKUP interface eth0 virtual_router_id 66 priority 80 advert_int 1 #nopreempt #preempt_delay 10 #如果设置延迟抢占模式,需要都是BACKUP,不然会造成脑裂 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } unicast_src_ip 172.31.0.48 unicast_peer{ 172.31.0.28 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 项目2-master [root@centos8 ~]# cat //etc/keepalived/conf.d/n521.conf vrrp_instance n521 { state MASTER interface eth0 virtual_router_id 88 priority 80 advert_int 1 authentication { auth_type PASS auth_pass 12345678 } virtual_ipaddress { 172.31.0.200 dev eth0 label eth0:2 } unicast_src_ip 172.31.0.48 unicast_peer{ 172.31.0.28 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } # 重启keepalived [root@centos8 ~]# systemctl restart keepalived # 查看ip [root@centos8 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 00:0c:29:16:9a:81 brd ff:ff:ff:ff:ff:ff inet 172.31.0.48/16 brd 172.31.255.255 scope global noprefixroute eth0 valid_lft forever preferred_lft forever inet 172.31.0.200/32 scope global eth0:2 valid_lft forever preferred_lft forever
查看ip
#ka1 [root@centos8 ~]# hostname -I 172.31.0.28 172.31.0.188 # ka2 [root@centos8 ~]# hostname -I 172.31.0.48 172.31.0.200
ka1主机故障,测试VIP漂移至ka2主机
[root@centos8 ~]# systemctl stop keepalived [root@centos8 ~]# hostname -I 172.31.0.28 [root@centos8 ~]# hostname -I 172.31.0.48 172.31.0.188 172.31.0.200
恢复ka1主机
[root@centos8 ~]# hostname -I 172.31.0.28 172.31.0.188 [root@centos8 ~]# hostname -I 172.31.0.48 172.31.0.200
# ka1 [root@centos8 ~]# iptables -A INPUT -s 172.31.0.48 -j DROP [root@centos8 ~]# iptables -vnL Chain INPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination 12 480 DROP all -- * * 172.31.0.48 0.0.0.0/0 ... [root@centos8 ~]# hostname -I 172.31.0.28 172.31.0.188 172.31.0.200 [root@centos8 ~]# hostname -I 172.31.0.48 172.31.0.200 # ka2 [root@centos8 ~]# iptables -A INPUT -s 172.31.0.28 -j REJECT [root@centos8 ~]# iptables -vnL Chain INPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination 10 400 REJECT all -- * * 172.31.0.28 0.0.0.0/0 reject-with icmp-port-unreachable ... [root@centos8 ~]# hostname -I 172.31.0.48 172.31.0.200 172.31.0.188 [root@centos8 ~]# hostname -I 172.31.0.28 172.31.0.188 172.31.0.200
总结脑裂现象:当ka1添加错误的iptables规则后查询ip就会看到有三个ip地址;ka2查询还是原来的两个ip地址,同时如果ka2也添加了错误的iptables规则后,查询ip就会看到是三个ip地址,首先backup收不到master的vrrp通信心跳检测,backup就会把虚拟vip抢过来,但实际master还存在的,只是拒绝了backup请求响应从而造成脑裂现象
虚拟服务器配置结构
virtual_server IP port { ... real_server { ... } real_server { ... } … }
virtual server (虚拟服务器)的定义格式
virtual_server IP port #定义虚拟主机IP地址及其端口 virtual_server fwmark int #ipvs的防火墙打标,实现基于防火墙的负载均衡集群 virtual_server group string #使用虚拟服务器组
将多个虚拟服务器定义成一个组,统一对外服务,如:http和https定义成一个虚拟服务器组
#参考文档:/usr/share/doc/keepalived/keepalived.conf.virtual_server_group virtual_server_group <STRING> { # Virtual IP Address and Port <IPADDR> <PORT> <IPADDR> <PORT> ... # <IPADDR RANGE> has the form ...
virtual_server IP port { #VIP和PORT delay_loop <INT> #检查后端服务器的时间间隔 lb_algo rr|wrr|lc|wlc|lblc|sh|dh #定义调度方法 lb_kind NAT|DR|TUN #集群的类型,注意要大写 persistence_timeout <INT> #持久连接时长 protocol TCP|UDP|SCTP #指定服务协议,一般为TCP sorry_server <IPADDR> <PORT> #所有RS故障时,备用服务器地址 real_server <IPADDR> <PORT> { #RS的IP和PORT weight <INT> #RS权重 notify_up <STRING>|<QUOTED-STRING> #RS上线通知脚本 notify_down <STRING>|<QUOTED-STRING> #RS下线通知脚本 HTTP_GET|SSL_GET|TCP_CHECK|SMTP_CHECK|MISC_CHECK { ... } #定义当前主机健康状态检测方法 } } #注意:括号必须分行写,两个括号写在同一行,如: }} 会出错
应用层检测:HTTP_GET|SSL_GET
HTTP_GET|SSL_GET { url { path <URL_PATH> #定义要监控的URL status_code <INT> #判断上述检测机制为健康状态的响应码,一般为 200 } connect_timeout <INTEGER> #客户端请求的超时时长, 相当于haproxy的timeout server nb_get_retry <INT> #重试次数 delay_before_retry <INT> #重试之前的延迟时长 connect_ip <IP ADDRESS> #向当前RS哪个IP地址发起健康状态检测请求 connect_port <PORT> #向当前RS的哪个PORT发起健康状态检测请求 bindto <IP ADDRESS> #向当前RS发出健康状态检测请求时使用的源地址 bind_port <PORT> #向当前RS发出健康状态检测请求时使用的源端口 }
范例:
virtual_server 172.31.0.188 80 { delay_loop 3 lb_algo rr lb_kind DR protocol TCP sorry_server 127.0.0.1 80 real_server 172.31.0.17 80 { weight 1 HTTP_GET { url { path /monitor.html status_code 200 } connect_timeout 1 nb_get_retry 3 delay_before_retry 1 } } real_server 172.31.0.27 80 { weight 1 HTTP_GET { url { path / status_code 200 } connect_timeout 1 nb_get_retry 3 delay_before_retry 1 } } }
传输层检测:TCP_CHECK
TCP_CHECK { connect_ip <IP ADDRESS> #向当前RS的哪个IP地址发起健康状态检测请求 connect_port <PORT> #向当前RS的哪个PORT发起健康状态检测请求 bindto <IP ADDRESS> #发出健康状态检测请求时使用的源地址 bind_port <PORT> #发出健康状态检测请求时使用的源端口 connect_timeout <INTEGER> #客户端请求的超时时长, 等于haproxy的timeout server }
范例:
virtual_server 172.31.0.188 80 { delay_loop 6 lb_algo wrr lb_kind DR #persistence_timeout 120 #会话保持时间 protocol TCP sorry_server 127.0.0.1 80 real_server 172.31.0.17 80 { weight 1 TCP_CHECK { connect_timeout 5 nb_get_retry 3 delay_before_retry 3 connect_port 80 } } real_server 172.31.0.27 80 { weight 1 TCP_CHECK { connect_timeout 5 nb_get_retry 3 delay_before_retry 3 connect_port 80 } } }
准备web服务器并使用脚本绑定VIP至web服务器lo网卡
#准备两台后端RS主机 [root@rs1 ~]# cat lvs_dr_rs.sh #!/bin/bash #Author:xuanlv #Date:2021-06-13 vip=172.31.0.188 mask='255.255.255.255' dev=lo:1 rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null service httpd start &> /dev/null && echo "The httpd Server is Ready!" echo "<h1>`hostname`</h1>" > /var/www/html/index.html case $1 in start) echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce ifconfig $dev $vip netmask $mask #broadcast $vip up #route add -host $vip dev $dev echo "The RS Server is Ready!" ;; stop) ifconfig $dev down echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce echo "The RS Server is Canceled!" ;; *) echo "Usage: $(basename $0) start|stop" exit 1 ;; esac # 执行 [root@rs1 ~]# bash lvs_dr_rs.sh start [root@rs1 ~]# bash lvs_dr_rs.sh start #测试直接访问两台RS
配置keepalived
#ka1节点的配置 [root@ka1-centos8 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { root@localhost } notification_email_from keepalived@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id ka1.longxuan.vip vrrp_mcast_group4 224.0.100.100 } vrrp_instance VI_1 { state MASTER interface eth0 virtual_router_id 66 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } virtual_server 172.31.0.188 80 { delay_loop 3 lb_algo rr lb_kind DR protocol TCP sorry_server 127.0.0.1 80 real_server 172.31.0.27 80 { weight 1 HTTP_GET { #应用层检测 url { path / status_code 200 } connect_timeout 1 nb_get_retry 3 delay_before_retry 1 } } real_server 172.31.0.37 80 { weight 1 TCP_CHECK { #另一台主机使用TCP检测 connect_timeout 5 nb_get_retry 3 delay_before_retry 3 connect_port 80 } } } #ka2节点的配置,配置和ka1基本相同,只需修改三行 [root@ka2-centos8 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { root@localhost } notification_email_from keepalived@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id ka2.longxuan.vip #修改此行 vrrp_mcast_group4 224.0.100.100 } vrrp_instance VI_1 { state BACKUP #修改此行 interface eth0 virtual_router_id 66 priority 80 #修改此行 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 } notify_master "/etc/keepalived/notify.sh master" notify_backup "/etc/keepalived/notify.sh backup" notify_fault "/etc/keepalived/notify.sh fault" } virtual_server 172.31.0.188 80 { delay_loop 3 lb_algo rr lb_kind DR protocol TCP sorry_server 127.0.0.1 80 real_server 172.31.0.27 80 { weight 1 HTTP_GET { url { path / status_code 200 } connect_timeout 1 nb_get_retry 3 delay_before_retry 1 } } real_server 172.31.0.37 80 { weight 1 TCP_CHECK { connect_timeout 5 nb_get_retry 3 delay_before_retry 3 connect_port 80 } } }
访问测试结果
root@long:~# curl 172.31.0.188 <h1>rs2 web</h1> root@long:~# curl 172.31.0.188 <h1>rs1 web</h1>
安装ipvsadm
[root@ka1 ~]# yum install ipvsadm -y # 查看ipvsadm [root@ka1 ~]# ipvsadm -Ln IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP 172.31.0.188:80 rr -> 172.31.0.27:80 Route 1 0 0 -> 172.31.0.37:80 Route 1 0 0
模拟故障
#第一台RS1故障,自动切换至RS2,这里一开始访问还会去RS1检测,经过检测几次发现真的检测不了,就不会往RS1调度,直接调度RS2 root@long:~# curl 172.31.0.188 <h1>rs2 web</h1> root@long:~# curl 172.31.0.188 <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN"> <html><head> <title>403 Forbidden</title> </head><body> <h1>Forbidden</h1> <p>You don't have permission to access /index.html on this server.</p> </body></html> root@long:~# curl 172.31.0.188 <h1>rs2 web</h1> root@long:~# curl 172.31.0.188 #检查ipvsadm还有一台可以调度,说明机器不能调度后,ipvsadm会自动剔除掉 [root@ka2 ~]# ipvsadm -Ln IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP 172.31.0.188:80 rr -> 172.31.0.37:80 Route 1 0 0
当后端两台RS都故障时,就会启用Sorry Server这台服务,前提是这台在ka1必须安装有web服务器,如果后端RS其中一台恢复了也不会访问Sorry Server服务
root@long:~# curl 172.31.0.188 Sorry Server
[root@ka1-centos8 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { notification_email { root@localhost } notification_email_from keepalived@localhost smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id ka1 #另一个节点为ka2 vrrp_mcast_group4 224.0.100.10 } vrrp_instance VI_1 { state MASTER #在另一个结点上为BACKUP interface eth0 virtual_router_id 66 priority 100 #在另一个结点上为80 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.188 dev eth0 label eth0:1 #指定VIP } } vrrp_instance VI_2 { state BACKUP #在另一个结点上为MASTER interface eth0 virtual_router_id 88 priority 80 #在另一个结点上为100 advert_int 1 authentication { auth_type PASS auth_pass 123456 } virtual_ipaddress { 172.31.0.200/24 dev eth0 label eth0:2 #指定VIP2 } } virtual_server 172.31.0.17 80 { delay_loop 6 lb_algo rr lb_kind DR protocol TCP sorry_server 127.0.0.1 80 real_server 172.31.0.27 80 { #指定RS1地址 weight 1 HTTP_GET { url { path / status_code 200 } connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 172.31.0.37 80 { #指定RS2地址 weight 1 HTTP_GET { url { path / status_code 200 } connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } } virtual_server 172.31.0.200 80 { #指定VIP2 delay_loop 6 lb_algo rr lb_kind DR protocol TCP sorry_server 127.0.0.1 80 real_server 172.31.0.27 80 { #指定RS3地址 weight 1 HTTP_GET { url { path / status_code 200 } connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } real_server 172.31.0.37 80 { #指定RS4地址 weight 1 HTTP_GET { url { path / status_code 200 } connect_timeout 3 nb_get_retry 3 delay_before_retry 3 } } }