Master LVS DR Mode: Network Setup, Firewall Marking, and High‑Availability

Table of Contents

1. DR Mode Extension

DR mode experiments with VIP and RIP on different subnets.

Main Summary

Virtual router: point all RIP gateways to the virtual router.

Connect the virtual router's other port to the DS switch.

Or directly respond to clients via another router.

Physical Network Setup

Create four virtual machines on the host.

DS and two RIP have a single physical NIC each in bridge mode.

The fourth VM has either two NICs with separate addresses or one NIC with two addresses, enables IP forwarding, points RIP gateways to one address, and connects the other interface to the switch.

1.1 All Configurations in Same Network

Physical NIC addresses

# DS and RSs are in bridge mode
DS physical NIC: 172.16.100.9
RS1 physical NIC: 172.16.100.21
RS2 physical NIC: 172.16.100.22

DS network address configuration

# Configure in the same network
# DS host can use ifconfig, ip, setup commands
ifconfig eno16777724:0 172.16.100.10 netmask 255.255.255.255 broadcast 172.16.100.10 up

# Alias route
route -host 172.16.100.10 dev eno16777724:0

# Verify rules
arp -a

# Using ip to add rules
ip addr add 172.16.100.10/32 dev eno16777724
ip addr list
ip addr del 172.16.100.10/32 dev eno16777724

# Add routing rule (optional)
ip route add 172.16.100.10 dev eno16777724
ip route list
ip route del 172.16.100.10

# Verify again
arp -a

RS network address configuration

# Configure RS hosts, start web service
#!/bin/bash
VIP=172.16.100.10
VIP_MASK=255.255.255.255
case $1 in
    start_dr)
        echo 1 > /proc/sys/net/ipv4/conf/all/arp-ignore
        echo 1 > /proc/sys/net/ipv4/conf/eth0/arp-ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp-announce
        echo 2 > /proc/sys/net/ipv4/conf/eth0/arp-announce
        ;;
    stop_dr)
        echo 0 > /proc/sys/net/ipv4/conf/all/arp-ignore
        echo 0 > /proc/sys/net/ipv4/conf/eth0/arp-ignore
        echo 0 > /proc/sys/net/ipv4/conf/all/arp-announce
        echo 0 > /proc/sys/net/ipv4/conf/eth0/arp-announce
        ;;
    start_ip)
        ifconfig lo:0 $VIP netmask $VIP_MASK broadcast $VIP up
        ;;
    stop_ip)
        ifconfig lo:0 down
        ;;
esac
chmod +x set_lvs_dr.sh
./set_lvs_dr.sh start_dr
./set_lvs_dr.sh start_ip

DS scheduling algorithm configuration

# Set scheduling rules on DS
ipvsadm -A -t 172.16.100.10:80 -s rr
ipvsadm -L -n

ipvsadm -a -t 172.16.100.10:80 -r 172.16.100.21 -g
ipvsadm -a -t 172.16.100.10:80 -r 172.16.100.22 -g

1.2 Configurations Not in Same Network

DS network address configuration

# Remove previous network configuration
ip addr del 172.16.100.10/32 dev eno16777724

# Add non‑local VIP address
ip addr add 192.168.0.10/32 dev eno16777724

# Verify
arp -a

RS network address configuration

# Configure RS hosts, start web service (VIP changed to 192.168.0.10)
#!/bin/bash
VIP=192.168.0.10
VIP_MASK=255.255.255.255
case $1 in
    start_dr)
        echo 1 > /proc/sys/net/ipv4/conf/all/arp-ignore
        echo 1 > /proc/sys/net/ipv4/conf/eth0/arp-ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp-announce
        echo 2 > /proc/sys/net/ipv4/conf/eth0/arp-announce
        ;;
    stop_dr)
        echo 0 > /proc/sys/net/ipv4/conf/all/arp-ignore
        echo 0 > /proc/sys/net/ipv4/conf/eth0/arp-ignore
        echo 0 > /proc/sys/net/ipv4/conf/all/arp-announce
        echo 0 > /proc/sys/net/ipv4/conf/eth0/arp-announce
        ;;
    start_ip)
        ifconfig lo:0 $VIP netmask $VIP_MASK broadcast $VIP up
        ;;
    stop_ip)
        ifconfig lo:0 down
        ;;
esac
chmod +x set_lvs_dr.sh
./set_lvs_dr.sh start_dr
./set_lvs_dr.sh start_ip

DS scheduling algorithm configuration

# Clear previous rules
ipvsadm -C

# Set scheduling for the new VIP
ipvsadm -A -t 192.168.0.10:80 -s rr
ipvsadm -a -t 192.168.0.10:80 -r 172.16.100.21 -g
ipvsadm -a -t 192.168.0.10:80 -r 172.16.100.22 -g

1.3 Example Scripts

DR director script example

#!/bin/bash
vip=172.16.100.33
rip=('172.16.100.8' '172.16.100.9')
weight=('1' '2')
port=80
scheduler=rr
ipvstype='-g'
case $1 in
    start)
        iptables -F -t filter
        ipvsadm -C
        ifconfig eth0:0 $vip broadcast $vip netmask 255.255.255.255 up
        route add -host $vip dev eth0:0
        echo 1 > /proc/sys/net/ipv4/ip_forward
        ipvsadm -A -t $vip:$port -s $scheduler
        for i in $(seq 0 $((${#rip[@]}-1))); do
            ipvsadm -a -t $vip:$port -r ${rip[$i]} $ipvstype -w ${weight[$i]}
        done
        touch /var/lock/subsys/ipvs
        ;;
    stop)
        echo 0 > /proc/sys/net/ipv4/ip_forward
        ipvsadm -C
        ifconfig eth0:0 down
        rm -f /var/lock/subsys/ipvs
        ;;
    status)
        if [ -f /var/lock/subsys/ipvs ]; then
            echo "ipvs is running."
            ipvsadm -L -n
        else
            echo "ipvs is stopped."
        fi
        ;;
    *)
        echo "Usage: $(basename $0) {start|stop|status}"
        exit 3
        ;;
esac

DR RS script example

#!/bin/bash
vip=172.16.100.33
interface="lo:0"
case $1 in
    start)
        echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
        ifconfig $interface $vip broadcast $vip netmask 255.255.255.255 up
        route add -host $vip dev $interface
        ;;
    stop)
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
        ifconfig $interface down
        ;;
    status)
        if ifconfig lo:0 | grep $vip > /dev/null; then
            echo "ipvs is running."
        else
            echo "ipvs is stopped."
        fi
        ;;
    *)
        echo "Usage: $(basename $0) {start|stop|status}"
        exit 1
        ;;
esac

2. LVS Firewall Marking

ipvs is defined in the INPUT chain; firewall marking must be done in the PREROUTING chain, otherwise ipvs cannot process the mark.

Key Points of Firewall Marking

Format: -j MARK --set-mark 10

Mark values range from 0 to 2^32, usually kept below 100.

Can filter specific request types, e.g., tcp/80.

Multiple services can share the same mark, e.g., tcp/80 and tcp/22.

Defining Cluster via Mark

# (1) Define marking rule in DS netfilter mangle PREROUTING
iptables -t mangle -A PREROUTING -d $vip -p $protocol --dports $port -j MARK --set-mark xxx

# (2) Define cluster service based on the firewall mark
ipvsadm -A -f xxx -s scheduler

2.1 Mark Configuration Method

Define cluster using firewall mark

# Use -j MARK to define firewall mark
iptables -t mangle -A PREROUTING -d 192.168.0.10 -p tcp --dport 80 -j MARK --set-mark 10

ipvsadm -C
ipvsadm -A -f 10 -s rr
ipvsadm -a -f 10 -r 172.16.100.21 -g
ipvsadm -a -f 10 -r 172.16.100.22 -g

Mark multiple services

# HTTP
iptables -t mangle -A PREROUTING -d 192.168.0.10 -p tcp --dport 80 -j MARK --set-mark 10
# SSH
iptables -t mangle -A PREROUTING -d 192.168.0.10 -p tcp --dport 22 -j MARK --set-mark 10
# HTTPS
iptables -t mangle -A PREROUTING -d 192.168.0.10 -p tcp --dport 443 -j MARK --set-mark 10

ipvsadm -C
ipvsadm -A -f 10 -s rr
ipvsadm -a -f 10 -r 172.16.100.21 -g
ipvsadm -a -f 10 -r 172.16.100.22 -g

2.2 Configuring HTTPS Service

# DS acts as a CA to issue certificates
cd /etc/pki/CA
umask 077; openssl genrsa -out private/cakey.pem 2048
touch index.txt
echo 01 > serial
openssl -req -new -x509 -key private/cakey.pem -out cacert.pem -days 465
# ... fill certificate information

# RS obtains certificate
mkdir -p /etc/httpd/ssl
cd /etc/httpd/ssl
umask 077; openssl genrsa -out httpd.key 1024
openssl req -new -key httpd.key -out httpd.csr
# Send CSR to CA, receive signed certificate
scp httpd.csr [email protected]:/tmp
# On CA machine
openssl ca -in /tmp/httpd.csr -out /tmp/httpd.crt
scp /tmp/httpd.crt [email protected]:/etc/httpd/ssl
# Copy certificate to the second RS
scp -rp /etc/httpd/ssl [email protected]:/etc/httpd/

# Install and enable SSL on RS
yum install -y mod_ssl
vim /etc/httpd/conf.d/ssl.conf   # set DocumentRoot, SSLCertificateFile, SSLCertificateKeyFile
httpd -t
service httpd restart

# Configure DS ipvs rules for HTTPS
ipvsadm -C
ipvsadm -A -t 192.168.0.10:443 -s rr
ipvsadm -a -t 192.168.0.10:443 -r 172.16.100.21 -g
ipvsadm -a -t 192.168.0.10:443 -r 172.16.100.22 -g

2.3 Session Persistence

Introducing Persistent Connections

HTTP/HTTPS session persistence is difficult; LVS provides its own persistence mechanisms.

Persistence Implementation Methods

Per‑Port Persistence (PPC): persistence for a single service with configurable timeout.

Per‑FWM Persistence (PFWMC): persistence based on a firewall mark, useful when multiple ports share the same mark.

Per‑Client Persistence (PCC): persistence per client across all ports, using a virtual service without a port number.

Persistence Considerations

Consumes memory; the number of entries is limited.

# Example PPC configuration
ipvsadm -C
ipvsadm -A -t 192.168.0.10:80 -s rr -p 3600
ipvsadm -a -t 192.168.0.10:80 -r 172.16.100.21 -g -w 1
ipvsadm -a -t 192.168.0.10:80 -r 172.16.100.22 -g -w 2

# Example PFWMC configuration
iptables -t mangle -A PREROUTING -d 192.168.0.10 -p tcp --dport 80 -j MARK --set-mark 10
iptables -t mangle -A PREROUTING -d 192.168.0.10 -p tcp --dport 443 -j MARK --set-mark 10

ipvsadm -C
ipvsadm -A -f 10 -s rr -p 3600
ipvsadm -a -f 10 -r 172.16.100.21 -g
ipvsadm -a -f 10 -r 172.16.100.22 -g

# Example PCC configuration
ipvsadm -C
ipvsadm -A -t 192.168.0.10:0 -s rr -p 3600
ipvsadm -a -t 192.168.0.10:0 -r 172.16.100.21 -g -w 1
ipvsadm -a -t 192.168.0.10:0 -r 172.16.100.22 -g -w 2

3. LVS High‑Availability Solutions

SPOF: Single Point of Failure

Failure Points

DS server single point → HA cluster

If the DS fails, the whole service becomes unavailable; use Keepalived for HA and health monitoring.

Backend real host failure → DS health checks

DS detects failed RS and redirects traffic to remaining RS; when an RS recovers, DS brings it back online.

Health Check Design

1. Protocol‑layer checks

IP layer: ICMP ping.

Transport layer: port status.

Application layer: request a key resource.

2. Check frequency

Balance between too frequent and too sparse; filter logs as needed.

3. State determination

Down: ok → failure → failure → failure.

Up: failure → ok → ok.

4. Back server

If all RS are down, DS can serve a sorry page.

#!/bin/bash
fwm=6
sorry_server=127.0.0.1
rs=('172.16.100.21' '172.16.100.22')
rw=('1' '2')
type='-g'
chkloop=3
rsstatus=(0 0)
logfile=/var/log/ipvs_health_check.log

addrs() {
    ipvsadm -a -f $fwm -r $1 -w $2 $type
    return $?
}

delrs() {
    ipvsadm -d -f $fwm -r $1
    return $?
}

chkrs() {
    local i=1
    while [ $i -le $chkloop ]; do
        if curl --connect-timeout 1 -s http://$1/.health.html | grep "OK" > /dev/null; then
            return 0
        fi
        let i++
        sleep 1
    done
    return 1
}

initstatus() {
    for host in $(seq 0 $((${#rs[@]}-1))); do
        if chkrs ${rs[$host]}; then
            [ ${rsstatus[$host]} -eq 0 ] && rsstatus[$host]=1
        else
            [ ${rsstatus[$host]} -eq 1 ] && rsstatus[$host]=0
        fi
    done
}

initstatus
while true; do
    for host in $(seq 0 $((${#rs[@]}-1))); do
        if chkrs ${rs[$host]}; then
            [ ${rsstatus[$host]} -eq 0 ] && addrs ${rs[$host]} ${rw[$host]} && rsstatus[$host]=1
        else
            [ ${rsstatus[$host]} -eq 1 ] && delrs ${rs[$host]} ${rw[$host]} && rsstatus[$host]=0
        fi
    done
    sleep 5
done