介绍
vrrp 协议的软件实现，原生设计目的为了高可用 ipvs服务
keepalived 是高可用集群的通用无状态应用解决方案
功能：
基于vrrp协议完成地址流动
为vip地址所在的节点生成ipvs规则(在配置文件中预先定义)
为ipvs集群的各RS做健康状态检测
基于脚本调用接口完成脚本中定义的功能，进而影响集群事务，以此支持nginx、haproxy等服务
Keepalived 架构
官方文档：
https://keepalived.org/doc/
http://keepalived.org/documentation.html用户空间核心组件：
vrrp stack：VIP消息通告
checkers：监测 Real Server
system call：实现 vrrp 协议状态转换时调用脚本的功能
SMTP：邮件组件
IPVS wrapper：生成 IPVS 规则
Netlink Reflector：网络接口
WatchDog：监控进程
控制组件：提供keepalived.conf 的解析器，完成Keepalived配置
IO复用器：针对网络目的而优化的自己的线程抽象
内存管理组件：为某些通用的内存管理功能（例如分配，重新分配，发布等）提供访问权限
Keepalived 环境准备
各节点时间必须同步：ntp, chrony
关闭防火墙及SELinux
各节点之间可通过主机名互相通信：非必须
建议使用/etc/hosts文件实现：非必须
各节点之间的root用户可以基于密钥认证的ssh服务完成互相通信：非必须
Keepalived 相关文件
软件包名：keepalived
主程序文件：/usr/sbin/keepalived
主配置文件：/etc/keepalived/keepalived.conf
配置文件示例：/usr/share/doc/keepalived/
Unit File：/lib/systemd/system/keepalived.service
Unit File的环境配置文件：
/etc/sysconfig/keepalived CentOS
/etc/default/keepalived Ubuntu
注意：CentOS 7 上有 bug，可能有下面情况出现
systemctl restart keepalived #新配置可能无法生效
systemctl stop keepalived;systemctl start keepalived #无法停止进程，需要 kill 停止Keepalived 安装
包安装：
#CentOS
[root@centos ~]#yum -y install keepalived
#ubuntu
[root@ubuntu1804 ~]#apt update;apt -y install keepalived
#默认缺少配置，服务无法启动，提示/etc/keepalived/keepalived.conf 不存在
[root@ubuntu2204 ~]#systemctl status keepalived.service
○ keepalived.service - Keepalive Daemon (LVS and VRRP)
Loaded: loaded (/lib/systemd/system/keepalived.service; enabled; vendor
preset: enabled)
Active: inactive (dead)
Condition: start condition failed at Thu 2023-01-12 15:28:42 CST; 4s ago
└─ ConditionFileNotEmpty=/etc/keepalived/keepalived.conf was not
met
1月 12 15:26:47 ubuntu2204.wang.org systemd[1]: Condition check resulted in
Keepalive Daemon (LVS and VRRP) being skipped.
1月 12 15:28:42 ubuntu2204.wang.org systemd[1]: Condition check resulted in
Keepalive Daemon (LVS and VRRP) being skipped.
#利用模板生成生配置文件
[root@ubuntu2204 ~]#cp /usr/share/doc/keepalived/samples/keepalived.conf.sample /etc/keepalived/keepalived.conf
[root@ubuntu2204 ~]#systemctl start keepalived.service
[root@ubuntu2204 ~]#systemctl status keepalived.service
● keepalived.service - Keepalive Daemon (LVS and VRRP)
Loaded: loaded (/lib/systemd/system/keepalived.service; enabled; vendor
preset: enabled)
Active: active (running) since Thu 2023-01-12 15:29:54 CST; 1s ago
Main PID: 2174 (keepalived)
Tasks: 3 (limit: 2196)
Memory: 4.5M
CPU: 40ms
CGroup: /system.slice/keepalived.service
├─2174 /usr/sbin/keepalived --dont-fork
├─2175 /usr/sbin/keepalived --dont-fork
└─2176 /usr/sbin/keepalived --dont-fork二进制安装：
#安装依赖
#红帽系统安装相关包
[root@centos7 ~]#yum install gcc curl openssl-devel libnl3-devel net-snmp-devel
#ubuntu
root@ubuntu2204[08:35:39]:~#apt update && apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf libipset-dev libnl-3-dev libnl-genl-3-dev libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libmagic-dev libsnmp-dev libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev
root@ubuntu2204[08:35:39]:~#wget https://keepalived.org/software/keepalived-2.2.7.tar.gz
#解压编译
root@ubuntu2204[08:35:39]:~#tar xf keepalived-2.2.7.tar.gz -C /usr/local/src/
root@ubuntu2204[08:35:39]:~#cd /usr/local/src/keepalived-2.2.7
root@ubuntu2204[08:35:39]:~#./configure --prefix=/apps/keepalived
root@ubuntu2204[08:35:39]:~#make && make install
#默认缺少/usr/share/doc/keepalived/samples/keepalived.conf.sample文件，不能启动；但是二进制安装提供了该配置文件模版
root@ubuntu2204[08:35:39]:~#mkdir /etc/keepalived
root@ubuntu2204[08:35:39]:~#cp /apps/keepalived/etc/keepalived/keepalived.conf.sample /etc/keepalived/keepalived.conf
#二进制安装已自动生成了service文件，可直接使用
root@ubuntu2204[08:56:20]:~#ls /lib/systemd/system/keepalived.service
/lib/systemd/system/keepalived.service
root@ubuntu2204[08:56:39]:~#systemctl daemon-reload
root@ubuntu2204[08:56:39]:~#systemctl enable --now keepalivedKeepAlived 配置说明
配置文件组成部分
配置文件
/etc/keepalived/keepalived.conf配置文件组成
GLOBAL CONFIGURATION
Global definitions：定义邮件配置，route_id，vrrp配置，多播地址等
VRRP CONFIGURATION
VRRP instance(s)：定义每个vrrp虚拟路由器
LVS CONFIGURATION
Virtual server group(s)
Virtual server(s)：LVS集群的VS和RS
配置语法说明
帮助
#包安装
man keepalived.conf
#编译安装
man /apps/keepalived/share/man/man5/keepalived.conf.5全局配置
#/etc/keepalived/keepalived.conf
global_defs {
notification_email {
root@localhost #keepalived 发生故障切换时邮件发送的目标邮箱，可以按行区分写多个
root@xxxxxxx.com
xxxxxxxxx@qq.com
}
notification_email_from keepalived@localhost #发邮件的地址
smtp_server 127.0.0.1 #邮件服务器地址
smtp_connect_timeout 30 #邮件服务器连接timeout
router_id ka1.example.com #每个keepalived主机唯一标识，建议使用当前主机名，如果多节点重名可能会影响切换脚本执行
vrrp_skip_check_adv_addr #默认会对所有通告报文都检查，会比较消耗性能，启用此配置后，如果收到的通告报文和上一个报文是同一个路由器，则跳过检查
vrrp_strict #严格遵守VRRP协议,启用此项后以下状况将无法启动服务或工作异常:1.无VIP地址 2.配置了单播邻居 3.在VRRP版本2中有IPv6地址，开启动此项并且没有配置vrrp_iptables时会自动开启iptables防火墙规则，默认导致VIP无法访问,建议不加此项配置
vrrp_garp_interval 0 #gratuitous ARP messages 报文发送延迟，0表示不延迟
vrrp_gna_interval 0 #unsolicited NA messages （不请自来）消息发送延迟
vrrp_mcast_group4 224.0.0.18 #指定组播IP地址范围：224.0.0.0到239.255.255.255,默认值：224.0.0.18，如果配置了单播，此项失效
vrrp_iptables #此项和vrrp_strict同时开启时，则不会添加防火墙规则,如果无配置vrrp_strict项,则无需启用此项配置，注意：新版加此项仍有iptables规则
}配置虚拟路由器
vrrp_instance <STRING> { #<String>为vrrp的实例名,一般为业务名称
配置参数
......
}
#配置参数：
state MASTER|BACKUP #当前节点在此虚拟路由器上的初始状态，状态为MASTER或者BACKUP，当priority相同时，先启动的节点优先获取VIP
interface IFACE_NAME #绑定为当前VRRP虚拟路由器使用的物理接口，如：eth0,bond0,br0,可以和VIP不在一个网卡，实现心跳功能
virtual_router_id VRID #每个虚拟路由器唯一标识，范围：0-255，每个虚拟路由器此值必须唯一，否则服务无法启动，同属一个虚拟路由器的多个keepalived节点必须相同,务必要确认在同一网络中此值必须唯一
priority 100 #当前物理节点在此虚拟路由器的优先级，范围：1-254，每个keepalived主机节点此值不同，如果多节点此值相同，则先来后到原理获取VIP
advert_int 1 #vrrp通告的时间间隔，默认1s
authentication { #认证机制
auth_type AH|PASS #AH为IPSEC认证(不推荐),PASS为简单密码(建议使用)
auth_pass <PASSWORD> #预共享密钥，仅前8位有效，同一个虚拟路由器的多个keepalived节点必须一样
}
virtual_ipaddress { #虚拟IP,生产环境可能指定几十上百个VIP地址
<IPADDR>/<MASK> brd <IPADDR> dev <STRING> scope <SCOPE> label <LABEL>
192.168.200.100 #指定VIP，不指定网卡，默认为eth0,注意：不指定/prefix,默认为/32
192.168.200.101/24 dev eth1 #指定VIP的网卡，建议和interface指令指定的网卡不在一个网卡
192.168.200.102/24 dev eth2 label eth2:1 #指定VIP的网卡label
}
track_interface { #配置监控网络接口，一旦出现故障，则转为FAULT状态实现地址转移
eth0
eth1
…
}范例：
[root@centos7 ~]#cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL
vrrp_skip_check_adv_addr
vrrp_strict #开启限制，会自动生效防火墙设置，导致无访问VIP
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 80 #修改此行
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.200.16
192.168.200.17
192.168.200.18
}
}
[root@centos7 ~]#systemctl start keepalived.service
[root@centos7 ~]#ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group
default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP
group default qlen 1000
link/ether 00:0c:29:33:b4:1a brd ff:ff:ff:ff:ff:ff
inet 10.0.0.17/24 brd 10.0.0.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet 192.168.200.16/32 scope global eth0
valid_lft forever preferred_lft forever
inet 192.168.200.17/32 scope global eth0
valid_lft forever preferred_lft forever
inet 192.168.200.18/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe33:b41a/64 scope link
valid_lft forever preferred_lft forever启用 Keepalived 日志功能
默认 keepalived的日志记录在LOG_DAEMON中，记录在/var/log/syslog或messages, 也支持自定义日志配置
范例：实现日志功能,
注意: 编译安装方式如果实现有问题，可以重启主机可以解决
#包安装修改文件路径
[root@ka1 ~]#vim /etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -S 6"
#编译安装修改文件路径
[root@ka1 ~]#grep ExecStart /lib/systemd/system/keepalived.service
ExecStart=/apps/keepalived/sbin/keepalived --dont-fork $KEEPALIVED_OPTIONS
#编译安装修改文件路径
[root@ka1 ~]#vim /apps/keepalived/etc/sysconfig/keepalived
# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp -P Only run with VRRP subsystem.
# --check -C Only run with Health-checker subsystem.
# --dont-release-vrrp -V Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs -I Dont remove IPVS topology on daemon stop.
# --dump-conf -d Dump the configuration data.
# --log-detail -D Detailed log messages.
# --log-facility -S 0-7 Set local syslog facility (default=LOG_DAEMON)
#
#修改下面行
KEEPALIVED_OPTIONS="-D -S 6"
#修改日志服务配置
[root@ka1 ~]#vim /etc/rsyslog.conf
local6.* /var/log/keepalived.log
#重启服务生效
[root@ka1 ~]#systemctl restart keepalived.service rsyslog.service
#如果不行，就重启主机
[root@ka1 ~]#reboot
[root@ka1 ~]#tail -f /var/log/keepalived.log
Apr 14 09:25:51 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:51 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:51 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:51 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:56 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:56 ka1 Keepalived_vrrp[1263]: (VI_1) Sending/queueing gratuitous
ARPs on eth0 for 10.0.0.10
Apr 14 09:25:56 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:56 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:56 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10
Apr 14 09:25:56 ka1 Keepalived_vrrp[1263]: Sending gratuitous ARP on eth0 for
10.0.0.10实现 Keepalived 独立子配置文件
当生产环境复杂时， /etc/keepalived/keepalived.conf 文件中保存所有集群的配置会导致内容过多，不易管理;可以将不同集群的配置，比如：不同集群的VIP配置放在独立的子配置文件中
利用include 指令可以实现包含子配置文件
格式:
include /path/file范例:
[root@ka1 ~]#mkdir /etc/keepalived/conf.d/
[root@ka1 ~]#vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
29308620@qq.com
}
notification_email_from 29308620@qq.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1.wang.org
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
vrrp_gna_interval 0
}
include /etc/keepalived/conf.d/*.conf #将VRRP相关配置放在子配置文件中
[root@ka1 ~]#vim /etc/keepalived/conf.d/cluster1.confKeepalived 通知脚本配置
当keepalived的状态变化时，可以自动触发脚本的执行，比如：发邮件通知用户
默认以用户keepalived_script身份执行脚本，如果此用户不存在，以root执行脚本
可以用下面指令指定脚本执行用户的身份
global_defs {
......
script_user <USER>
......
}脚本的调用方法
在 vrrp_instance VI_1 语句块的末尾加下面行
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"实现 Keepalived 状态切换的通知脚本
以下脚本支持RHEL和Ubuntu系统
#在所有 keepalived节点配置如下
root@haproxy-slave[17:28:15]:/etc/keepalived#cat notify.sh
#!/bin/bash
contact='l13366769096@163.com'
email_send='1106820263@qq.com'
email_passwd='wogfdpcezxsujdcb'
email_smtp_server='smtp.qq.com'
. /etc/os-release
msg_error() {
echo -e "\033[1;31m$1\033[0m"
}
msg_info() {
echo -e "\033[1;32m$1\033[0m"
}
msg_warn() {
echo -e "\033[1;33m$1\033[0m"
}
color () {
RES_COL=60
MOVE_TO_COL="echo -en \\033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \\033[1;32m"
SETCOLOR_FAILURE="echo -en \\033[1;31m"
SETCOLOR_WARNING="echo -en \\033[1;33m"
SETCOLOR_NORMAL="echo -en \E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
install_sendemail () {
if [[ $ID =~ rhel|centos|rocky ]];then
rpm -q sendemail &> /dev/null || yum install -y sendemail
elif [ $ID = 'ubuntu' ];then
dpkg -l |grep -q sendemail || { apt update; apt install -y libio-socket-ssl-perl libnet-ssleay-perl sendemail ; }
else
color "不支持此操作系统，退出!" 1
exit
fi
}
send_email () {
local email_receive="$1"
local email_subject="$2"
local email_message="$3"
sendemail -f $email_send -t $email_receive -u $email_subject -m $email_message -s $email_smtp_server -o message-charset=utf-8 -o tls=yes -xu $email_send -xp $email_passwd
[ $? -eq 0 ] && color "邮件发送成功!" 0 || color "邮件发送失败!" 1
}
notify() {
if [[ $1 =~ ^(master|backup|fault)$ ]];then
mailsubject="$(hostname) to be $1, vip floating"
mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
send_email "$contact" "$mailsubject" "$mailbody"
else
echo "Usage: $(basename $0) {master|backup|fault}"
exit 1
fi
}
install_sendemail
notify $1Keepalived 实现 VRRP
实现Master/Backup的 Keepalived 单主架构
MASTER配置
[root@ka1 ~]#vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
root@localhost #keepalived 发生故障切换时邮件发送的对象，可以按行区分写多个
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1.example.com
vrrp_skip_check_adv_addr #所有报文都检查比较消耗性能，此配置为如果收到的报文和上一个报文是同一个路由器则跳过检查报文中的源地址
#vrrp_strict #严格遵守VRRP协议,禁止状况:1.无VIP地址,2.配置了单播邻居,3.在VRRP版本2中有IPv6地址
vrrp_garp_interval 0 #ARP报文发送延迟
vrrp_gna_interval 0 #消息发送延迟
vrrp_mcast_group4 224.0.0.18 #默认组播IP地址，可指定组播范围：224.0.0.0到239.255.255.255
}
vrrp_instance VI_1 {
state MASTER #在另一个节点上为BACKUP,如果当priority相同时,先启动的节点优先获取VIP
interface eth0
virtual_router_id 66 #每个虚拟路由器必须唯一，同属一个虚拟路由器的多个keepalived节点必须相同
priority 100 #在另一个结点上为80
advert_int 1
authentication {
auth_type PASS #预共享密钥认证，同一个虚拟路由器的keepalived节点必须一样
auth_pass 12345678
}
virtual_ipaddress {
10.0.0.10 dev eth0 label eth0:0
}
}BACKUP配置
#配置文件和master基本一致，只需修改三行
[root@ka2 ~]#vim /etc/keepalived/keepalived.conf
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka2.example.com #修改此行
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 224.0.0.18
}
vrrp_instance VI_1 {
state BACKUP #修改此行
interface eth0
virtual_router_id 66
priority 80 #修改此行
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
10.0.0.10 dev eth0 label eth0:0
}
}抓包观察VRRP协议
tcpdump -i eth0 -nn host 224.0.0.18脑裂
主备节点同时拥有同一个VIP，此时为脑裂现象
注意：脑裂现象原因
心跳线故障： 注意:在虚拟机环境中测试可以通过修改网卡的工作模式实现模拟，断开网卡方式无法模拟
防火墙错误配置：在从节点服务器执行iptables -A INPUT -s 主服务心跳网卡IP -j DROP 进行模拟
Keepalived 配置错误：多播地址不同，interface错误，virtual_router_id不一致，密码不一致
范例: 发现脑裂
[root@centos7 ~]#arping -I eth1 -c1 10.0.0.100
ARPING 10.0.0.100 from 192.168.10.130 eth1
Unicast reply from 10.0.0.100 [00:0C:29:7E:DA:E6] 0.801ms
Unicast reply from 10.0.0.100 [00:0C:29:97:AF:4F] 0.814ms
Sent 1 probes (1 broadcast(s))
Received 2 response(s)抢占模式和非抢占模式
非抢占模式 nopreempt
默认为抢占模式 preempt，即当高优先级的主机恢复在线后，会抢占低先级的主机的master角色，造成网络抖动，建议设置为非抢占模式 nopreempt ，即高优先级主机恢复后，并不会抢占低优先级主机的master 角色
注意: 非抢占模式下,如果原主机down机, VIP迁移至的新主机, 后续新主机也发生down（（keepalived 服务down））时,VIP还会迁移回修复好的原主机
但如果新主机的服务down掉（keepalived服务正常），原主机也不会接管VIP，仍会由新主机拥有VIP即非抢占式模式，只是适合当主节点宕机，切换到从节点的一次性的高可用性，后续即使当原主节点修复好，仍无法再次起到高用功能
注意：要关闭 VIP抢占，必须将各 Keepalived 服务器 state 配置为 BACKUP
#ha1主机配置
vrrp_instance VI_1 {
state BACKUP #都为BACKUP
interface eth0
virtual_router_id 66
priority 100 #优先级高
advert_int 1
nopreempt #添加此行，设为nopreempt
#ha2主机配置
vrrp_instance VI_1 {
state BACKUP #都为BACKUP
interface eth0
virtual_router_id 66
priority 80 #优先级低
advert_int 1
#nopreempt #注意：如果ka2主机也是非抢占式，会导致ka1即使优先级降低于ka2，VIP也不会切换至ka2抢占延迟模式 preempt_delay
抢占延迟模式，即优先级高的主机恢复后，不会立即抢回VIP，而是延迟一段时间（默认300s）再抢回VIP
但是如果低优先级的主机down机，则立即抢占VIP地址，而不再延迟
preempt_delay # #指定抢占延迟时间为#s，默认延迟300s注意：需要各keepalived服务器state为BACKUP,并且不要启用 vrrp_strict
范例：
#ka1主机配置
vrrp_instance VI_1 {
state BACKUP #都为BACKUP
interface eth0
virtual_router_id 66
priority 100 #优先级高
advert_int 1
preempt_delay 60 #抢占延迟模式，默认延迟300s
#ka2主机配置
vrrp_instance VI_1 {
state BACKUP #都为BACKUP
interface eth0
virtual_router_id 66
priority 80 #优先级低
advert_int 1VIP 单播配置
默认keepalived主机之间利用多播相互通告消息，会造成网络拥塞，可以替换成单播，减少网络流量
另外：有些公有云不支持多播，可以利用单播实现
单播优先与多播，即同时配置，单播生效
注意：启用 vrrp_strict 时，不能启用单播
#在所有节点vrrp_instance语句块中设置对方主机的IP，建议设置为专用于对应心跳线网络的地址，而非使用业务网络
unicast_src_ip <IPADDR> #指定发送单播的源IP
unicast_peer {
<IPADDR> #指定接收单播的对方目标主机IP
......
}范例：
#master 主机配置
[root@ka1 ~]#cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id ka1.wang.org
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 239.0.0.0 #单播优先于多播,即配置了单播后，多播将失效
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 66
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
10.0.0.10/24 dev eth0 label eth0:1
}
unicast_src_ip 10.0.0.8 #本机IP
unicast_peer{
10.0.0.18 #指向对方主机IP
10.0.0.28 #如果有多个keepalived,再加其它节点的IP
}
}
[root@ha1-centos8 ~]#hostname -I
10.0.0.8 10.0.0.10
#slave 主机配置
[root@ka2 ~]#cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id ka2.wang.org
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 239.0.0.0 #单播优先于多播
}
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id ka1.wang.org
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 239.0.0.0 #单播优先于多播,即配置了单播后，多播将失效
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 66
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
10.0.0.10/24 dev eth0 label eth0:1
}
unicast_src_ip 10.0.0.8 #本机IP
unicast_peer{
10.0.0.18 #指向对方主机IP
10.0.0.28 #如果有多个keepalived,再加其它节点的IP
}
}
[root@ha1-centos8 ~]#hostname -I
10.0.0.8 10.0.0.10
#slave 主机配置
[root@ka2 ~]#cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id ka2.wang.org
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 239.0.0.0 #单播优先于多播
}范例: 抓包观察
root@centos7 ~]#tcpdump -i eth0 -nn src host 10.0.0.8 and dst host 10.0.0.18
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes
23:37:48.069158 IP 10.0.0.8 > 10.0.0.18: VRRPv2, Advertisement, vrid 66, prio
100, authtype simple, intvl 1s, length 20
23:37:49.070013 IP 10.0.0.8 > 10.0.0.18: VRRPv2, Advertisement, vrid 66, prio
100, authtype simple, intvl 1s, length 20
23:37:50.071144 IP 10.0.0.8 > 10.0.0.18: VRRPv2, Advertisement, vrid 66, prio
100, authtype simple, intvl 1s, length 20Keepalived 通知脚本配置
当keepalived的状态变化时，可以自动触发脚本的执行，比如：发邮件通知用户
默认以用户keepalived_script身份执行脚本，如果此用户不存在，以root执行脚本
可以用下面指令指定脚本执行用户的身份
global_defs {
......
script_user <USER>
......
}通知脚本类型
当前节点成为主节点时触发的脚本
notify_master <STRING>|<QUOTED-STRING>当前节点转为备节点时触发的脚本
notify_backup <STRING>|<QUOTED-STRING>当前节点转为“失败”状态时触发的脚本
notify_fault <STRING>|<QUOTED-STRING>通用格式的通知触发机制，一个脚本可完成以上三种状态的转换时的通知
notify <STRING>|<QUOTED-STRING>当停止VRRP时触发的脚本
notify_stop <STRING>|<QUOTED-STRING>脚本的调用方法
在 vrrp_instance VI_1 语句块的末尾加下面行
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"实现 Keepalived 状态切换的通知脚本
#!/bin/bash
contact='lxxxxx69096@163.com'
email_send='110xxxx263@qq.com'
email_passwd='wogfdpcezxsqwdcb'
email_smtp_server='smtp.qq.com'
. /etc/os-release
msg_error() {
echo -e "\033[1;31m$1\033[0m"
}
msg_info() {
echo -e "\033[1;32m$1\033[0m"
}
msg_warn() {
echo -e "\033[1;33m$1\033[0m"
}
color () {
RES_COL=60
MOVE_TO_COL="echo -en \\033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \\033[1;32m"
SETCOLOR_FAILURE="echo -en \\033[1;31m"
SETCOLOR_WARNING="echo -en \\033[1;33m"
SETCOLOR_NORMAL="echo -en \E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
install_sendemail () {
if [[ $ID =~ rhel|centos|rocky ]];then
rpm -q sendemail &> /dev/null || yum install -y sendemail
elif [ $ID = 'ubuntu' ];then
dpkg -l |grep -q sendemail || { apt update; apt install -y libio-socket-ssl-perl libnet-ssleay-perl sendemail ; }
else
color "不支持此操作系统，退出!" 1
exit
fi
}
send_email () {
local email_receive="$1"
local email_subject="$2"
local email_message="$3"
sendemail -f $email_send -t $email_receive -u $email_subject -m $email_message -s $email_smtp_server -o message-charset=utf-8 -o tls=yes -xu $email_send -xp $email_passwd
[ $? -eq 0 ] && color "邮件发送成功!" 0 || color "邮件发送失败!" 1
}
notify() {
if [[ $1 =~ ^(master|backup|fault)$ ]];then
mailsubject="$(hostname) to be $1, vip floating"
mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
send_email "$contact" "$mailsubject" "$mailbody"
else
echo "Usage: $(basename $0) {master|backup|fault}"
exit 1
fi
}
install_sendemail
notify $1
#加执行权限验证
[root@ka1 ~]#chmod a+x /etc/keepalived/notify.sh
[root@ka1 ~]#vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
......
virtual_ipaddress {
10.0.0.10 dev eth0 label eth0:1
}
notify_master "/etc/keepalived/notify.sh master"
notify_backup "/etc/keepalived/notify.sh backup"
notify_fault "/etc/keepalived/notify.sh fault"
}
#模拟master故障
[root@ka1 ~]#killall keepalived实现 Master/Master 的 Keepalived 双主架构
master/slave的单主架构，同一时间只有一个Keepalived对外提供服务，此主机繁忙，而另一台主机却很空闲，利用率低下，可以使用master/master的双主架构，解决此问题。
Master/Master 的双主架构：
即将两个或以上VIP分别运行在不同的keepalived服务器，以实现服务器并行提供web访问的目的，提高服务器资源利用率
#ha1主机配置
[root@ka1 ~]#vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@wangxiaochun.com
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka1.wang.org
vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
state MASTER #在另一个主机上为BACKUP
interface eth0
virtual_router_id 66 #每个vrrp_instance唯一
priority 100 #在另一个主机上为80
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
10.0.0.10/24 dev eth0 label eth0:1 #指定vrrp_instance各自的VIP
}
}
vrrp_instance VI_2 { #添加 VI_2 实例
state BACKUP #在另一个主机上为MASTER
interface eth0
virtual_router_id 88 #每个vrrp_instance唯一
priority 80 #在另一个主机上为100
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
10.0.0.20/24 dev eth0 label eth0:1 #指定vrrp_instance各自的VIP
}
}
#ka2主机配置,和ka1配置只需五行不同
[root@ka2 ~]#vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@wangxiaochun.com
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ka2.wang.org #修改此行
vrrp_mcast_group4 224.0.100.100
}
vrrp_instance VI_1 {
state BACKUP #此修改行为BACKUP
interface eth0
virtual_router_id 66
priority 80 #此修改行为80
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
10.0.0.10/24 dev eth0 label eth0:1
}
}
vrrp_instance VI_2 {
state MASTER #修改此行为MASTER
interface eth0
virtual_router_id 88
priority 100 #修改此行为100
advert_int 1
authentication {
auth_type PASS
auth_pass 12345678
}
virtual_ipaddress {
10.0.0.20/24 dev eth0 label eth0:1
}
}实现 IPVS 的高可用性
IPVS 相关配置
虚拟服务器配置结构
每一个虚拟服务器即一个IPVS集群
可以通过下面语法实现
virtual_server IP port {
...
real_server {
...
}
real_server {
...
}
…
}Virtual Server （虚拟服务器）的定义格式
virtual_server IP port #定义虚拟主机IP地址及其端口
virtual_server fwmark int #ipvs的防火墙打标，实现基于防火墙的负载均衡集群
virtual_server group string #使用虚拟服务器组虚拟服务器组
将多个虚拟服务器定义成一个组，统一对外服务，如：http和https定义成一个虚拟服务器组
#参考文档：/usr/share/doc/keepalived/keepalived.conf.virtual_server_group
virtual_server_group <STRING> {
# Virtual IP Address and Port
<IPADDR> <PORT>
<IPADDR> <PORT>
...
# <IPADDR RANGE> has the form
# XXX.YYY.ZZZ.WWW-VVV eg 192.168.200.1-10
# range includes both .1 and .10 address
<IPADDR RANGE> <PORT># VIP range VPORT
<IPADDR RANGE> <PORT>
...
# Firewall Mark (fwmark)
fwmark <INTEGER>
fwmark <INTEGER>
...
}虚拟服务器配置
virtual_server IP port { #VIP和PORT
delay_loop <INT> #检查后端服务器的时间间隔
lb_algo rr|wrr|lc|wlc|lblc|sh|dh #定义调度方法
lb_kind NAT|DR|TUN #集群的类型,注意要大写
persistence_timeout <INT> #持久连接时长
protocol TCP|UDP|SCTP #指定服务协议,一般为TCP
sorry_server <IPADDR> <PORT> #所有RS故障时，备用服务器地址
real_server <IPADDR> <PORT> { #RS的IP和PORT
weight <INT> #RS权重
notify_up <STRING>|<QUOTED-STRING> #RS上线通知脚本
notify_down <STRING>|<QUOTED-STRING> #RS下线通知脚本
HTTP_GET|SSL_GET|TCP_CHECK|SMTP_CHECK|MISC_CHECK { ... } #定义当前主机健康状态检测方法
}
}
#注意:括号必须分行写,两个括号写在同一行,如: }} 会出错应用层监测
应用层检测：HTTP_GET|SSL_GET
HTTP_GET|SSL_GET {
url {
path <URL_PATH> #定义要监控的URL
status_code <INT> #判断上述检测机制为健康状态的响应码，一般为 200
}
connect_timeout <INTEGER> #客户端请求的超时时长, 相当于haproxy的timeout server
nb_get_retry <INT> #重试次数
delay_before_retry <INT> #重试之前的延迟时长
connect_ip <IP ADDRESS> #向当前RS哪个IP地址发起健康状态检测请求
connect_port <PORT> #向当前RS的哪个PORT发起健康状态检测请求
bindto <IP ADDRESS> #向当前RS发出健康状态检测请求时使用的源地址
bind_port <PORT> #向当前RS发出健康状态检测请求时使用的源端口
}范例：
virtual_server 10.0.0.10 80 {
delay_loop 3
lb_algo wrr
lb_kind DR
protocol TCP
sorry_server 127.0.0.1 80
real_server 10.0.0.7 80 {
weight 1
HTTP_GET {
url {
path /monitor.html
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
real_server 10.0.0.17 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 1
nb_get_retry 3
delay_before_retry 1
}
}
}
#在后端服务器可以观察到健康检测日志
[root@web01 ~]#tail /var/log/nginx/access.log
10.0.0.201 - - [13/Jan/2023:11:27:01 +0800] "GET / HTTP/1.0" 200 24 "-"
"KeepAliveClient"
10.0.0.201 - - [13/Jan/2023:11:27:04 +0800] "GET / HTTP/1.0" 200 24 "-"
"KeepAliveClient"
10.0.0.201 - - [13/Jan/2023:11:27:07 +0800] "GET / HTTP/1.0" 200 24 "-"
"KeepAliveClient"TCP监测
传输层检测：TCP_CHECK
TCP_CHECK {
connect_ip <IP ADDRESS> #向当前RS的哪个IP地址发起健康状态检测请求
connect_port <PORT> #向当前RS的哪个PORT发起健康状态检测请求
bindto <IP ADDRESS> #发出健康状态检测请求时使用的源地址
bind_port <PORT> #发出健康状态检测请求时使用的源端口
connect_timeout <INTEGER> #客户端请求的超时时长, 等于haproxy的timeout server
}范例：
virtual_server 10.0.0.10 80 {
delay_loop 6
lb_algo wrr
lb_kind DR
#persistence_timeout 120 #会话保持时间
protocol TCP
sorry_server 127.0.0.1 80
real_server 10.0.0.7 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 10.0.0.17 80 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}