|
# Tested on CentOS 7
# Note: For the commands here after, [ALL] indicates that command has to be run on the two
# nodes and [ONE] indicates that one needs to run it only on one of the hosts.
# This recipe focuses on the shared storage case where two nodes have access to the shared
# volume but only one node can actively mount and write to it at a time.
# I'll work on a two-node basic pacemaker cluster like the one we built on:
# I'll create a simple active/passive cluster. For that I will add a simple resource like
# an IP, that will allow me to connect to either of the nodes where IP will be active, a
# web service serving pages on the previously configured IP and a filesystem where will
# reside the web pages being served.
# Fencing
# ------------------------------------------------------------------------------------------
# There are many different topologies for fencing as to explain each of them in this recipe
# so, for the moment, I will disable it:
[ONE] pcs property set stonith-enabled=false
# Clustered Volume
# ------------------------------------------------------------------------------------------
# The first step is to configure a volume that will be able to failover between nodes. For
# that, we have to install the extensions to LVM2 to support clusters:
[ALL] yum install lvm2-cluster
# Then, create the PV and VG for the clustered filesystem:
[ONE] pvcreate /dev/sdb
[ONE] vgcreate vg_shared /dev/sdb
[ONE] lvcreate -l 100%VG -n lv_shared vg_shared
# Deactivate and reactivate shared volume group with an exclusive lock
[ONE] vgchange -an vg_shared
[ONE] vgchange -aey vg_shared
# And format the new logical volume
[ONE] mkfs.ext4 /dev/vg_shared/lv_shared
# Now, we have to update the /etc/lvm/lvm.conf configuration file so the CLVM volume is
# never auto-mounted. For that we will limit the volume list to be auto-mounted to 'rootvg'
# (this is the name of my rootvg) and the volume groups tagged with the hostname (cluster
# will take care of tagging volume groups at activation time):
[ALL] LINE=$((`grep -n "# volume_list" /etc/lvm/lvm.conf | cut -f1 -d':'`+1))
[ALL] sed -i.bak "${LINE}i\ volume_list = [ \"rootvg\", \"@`hostname -s`\" ]" \
/etc/lvm/lvm.conf
# Update initramfs and reboot
[ALL] cp /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak
[ALL] dracut -H -f /boot/initramfs-$(uname -r).img $(uname -r)
[ALL] shutdown -r now
# Now we have a logical volume that is accessible from both nodes but inactive for both
# of them
root@nodeA:/root#> lvs
LV VG Attr LSize Pool Origin Data% Meta% Move Log [...]
lv_depot rootvg -wi-ao---- 5.00g
lv_root rootvg -wi-ao---- 2.00g
lv_swap rootvg -wi-ao---- 2.00g
lv_var rootvg -wi-ao---- 2.00g
lv_shared vg_shared -wi------- 1020.00m
root@nodeB:/root#> lvs
LV VG Attr LSize Pool Origin Data% Meta% Move Log [...]
lv_depot rootvg -wi-ao---- 5.00g
lv_root rootvg -wi-ao---- 2.00g
lv_swap rootvg -wi-ao---- 2.00g
lv_var rootvg -wi-ao---- 2.00g
lv_shared vg_shared -wi------- 1020.00m
# We are ready to create the LVM resource ("lar_cl-lv01" is the name of the new resource
# and "lar_cl-rg01" the name of the new Resource Group cluster)
[ONE] pcs resource create lar_cl-lv01 ocf:heartbeat:LVM volgrpname=vg_shared \
exclusive=true --group lar_cl-rg01
# Look at this, resource has been started on one of the nodes of the cluster:
[ONE] pcs status | grep lar_cl-lv01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeA
# and, therefore, logical volume should be active on that node. Let's check it:
root@nodeA:/root#> lvs
LV VG Attr LSize Pool Origin Data% Meta% Move Log [...]
lv_depot rootvg -wi-ao---- 5.00g
lv_root rootvg -wi-ao---- 2.00g
lv_swap rootvg -wi-ao---- 2.00g
lv_var rootvg -wi-ao---- 2.00g
lv_shared vg_shared -wi-a----- 1020.00m
root@nodeB:/root#> lvs
LV VG Attr LSize Pool Origin Data% Meta% Move Log [...]
lv_depot rootvg -wi-ao---- 5.00g
lv_root rootvg -wi-ao---- 2.00g
lv_swap rootvg -wi-ao---- 2.00g
lv_var rootvg -wi-ao---- 2.00g
lv_shared vg_shared -wi------- 1020.00m
# Well, so far so good.
# Let's do the same thing for the filesystem resource (pay attention to the Resource Group
# name, it must be the the one used before for the LVM resource):
[ONE] pcs resource create lar_cl-fs01 ocf:heartbeat:Filesystem \
device="/dev/vg_shared/lv_shared" directory="/cluster_fs_01" fstype="ext4" \
--group lar_cl-rg01
[ONE] pcs status | grep "lar_cl-"
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeA
lar_cl-fs01 (ocf:heartbeat:Filesystem): Started nodeA <---
# Filesystem has been mounted on the right node (obvious):
root@nodeA:/root#> df -h | grep shared
/dev/mapper/vg_shared-lv_shared 988M 2.6M 919M 1% /cluster_fs_01
root@nodeB:/root#> df -h | grep shared
root@nodeB:/root#>
# Let's check if manual failover works fine:
[ONE] pcs status | grep "lar_cl-"
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeA
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeA
[ONE] pcs resource move lar_cl-fs01 nodeB
[ONE] pcs status | grep "lar_cl-"
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeB
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeB
root@nodeA:/root#> df -h | grep shared
root@nodeA:/root#>
root@nodeB:/root#> df -h | grep shared
/dev/mapper/vg_shared-lv_shared 988M 2.6M 919M 1% /cluster_fs_01
# And now, to check automatic failover, I will forcefully poweroff nodeB:
root@nodeB:/root#> echo 'o' > /proc/sysrq-trigger
# we can see that Resource Group failed over back to nodeA:
root@nodeA:/root#> pcs status | grep "lar_cl-"
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeA
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeA
root@nodeA:/root#> df -h | grep lv_shared
/dev/mapper/vg_shared-lv_shared 988M 2.6M 919M 1% /cluster_fs_01
# Everything worked as expected ...the only thing is that as soon as powered-off node
# becomes available again, Resource Group will switch back to it. In order to minimize the
# unavailability of services we may not want Resource Group switching back automatically. I
# will take care of this behaviour on a different recipe.
# Let's continue.
# Now I'm creating an IP resource, for our future clustered web server:
[ONE] pcs resource create lar_cl-ip01 ocf:heartbeat:IPaddr2 ip=192.168.56.111 \
cidr_netmask=24 op monitor interval=30s
# After a little while IP becomes available at cluster level:
[ONE] pcs status
Cluster name: lar_cluster
Stack: corosync
Current DC: nodeA (version 1.1.16-12.el7-94ff4df) - partition with quorum
Last updated: Tue Feb 13 16:57:16 2018
Last change: Tue Feb 13 16:57:02 2018 by root via cibadmin on nodeA
2 nodes configured
3 resources configured
Online: [ nodeA nodeB ]
Full list of resources:
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeB
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeB
lar_cl-ip01 (ocf::heartbeat:IPaddr2): Started nodeA <---
Daemon Status:
corosync: active/enabled
pacemaker: active/enabled
pcsd: active/enabled
# And is mounted
root@pacem01:/root#> grep lar_cl-ip01 /etc/hosts
192.168.56.111 lar_cl-ip01
root@nodeA:/root#> ip a s eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 08:00:27:5f:e2:2c brd ff:ff:ff:ff:ff:ff
inet 192.168.56.101/24 brd 192.168.56.255 scope global eth0
valid_lft forever preferred_lft forever
inet 192.168.56.111/24 brd 192.168.56.255 scope global secondary eth0
valid_lft forever preferred_lft forever
inet6 fe80::a00:27ff:fe5f:e22c/64 scope link
valid_lft forever preferred_lft forever
# What we've done here is to define a resource of type ocf:heartbeat:IPaddr2 this is:
# ocf: is the standard to which the resource script conforms and where to find it.
# heartbeat: is standard-specific; for OCF resources, it tells the cluster which OCF
# namespace the resource script is in.
# IPaddr2: is the name of the resource script.
# To obtain a list of the available resource standards, OCF providers and resource agents,
# run:
[ONE] pcs resource standards
ocf
lsb
service
systemd
stonith
[ONE] pcs resource providers
heartbeat
openstack
pacemaker
[ONE] pcs resource agents ocf:heartbeat
CTDB
Delay
Dummy
Filesystem
IPaddr
IPaddr2
[...]
rsyncd
slapd
symlink
tomcat
# The only thing is that IP resource doesn't belong to the LVM/filesystem Resource Group
# (I forgot to indicate "--group lar_cl-rg01" when creating the resource and we need
# everything to work together, so I will move IP resource to the right Resource Group by
# running following command:
[ONE] pcs resource group add lar_cl-rg01 lar_cl-ip01
# In addition, this will stop/start IP resource in the right node if necessary:
[ONE] pcs status
[...]
Full list of resources:
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeB
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeB
lar_cl-ip01 (ocf::heartbeat:IPaddr2): Started nodeB <---
[...]
# Ok, we have a Resource Group with a filesystem and an IP. Let's create an associated
# service on the top of that. I will add an Apache HTTP Server as a service.
[ALL] yum install httpd wget
# Create a page to serve. As long as the site must be available on a service that may
# failover between nodes, we have to create the file on the clustered filesystem we
# created previously. Then, on the right node (cluster filesystem must be mounted):
[ONE] mkdir -p /cluster_fs_01/www/html
[ONE] cat << EOF >/cluster_fs_01/www/html/index.html
<html>
<body>My clustered Apache service</body>
</html>
EOF
# And as the default Apache document root is /var/www/html:
[ALL] ln -s /cluster_fs_01/www/html/index.html /var/www/html/index.html
# Then, in order to monitor the health of my Apache instance, and recover it if it fails,
# the resource agent used by Pacemaker assumes the server-status URL is available:
[ALL] cat << EOF >/etc/httpd/conf.d/status.conf
<Location /server-status>
SetHandler server-status
Require local
</Location>
EOF
# Finally, I create the resrouce for my web site in the right Resource Group
[ONE] pcs resource create lar_cl-wb01 ocf:heartbeat:apache \
configfile=/etc/httpd/conf/httpd.conf \
statusurl="http://localhost/server-status" \
op monitor interval=1min --group lar_cl-rg01
[ONE] pcs status | grep "lar_cl-"
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeB
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeB
lar_cl-ip01 (ocf::heartbeat:IPaddr2): Started nodeB
lar_cl-wb01 (ocf::heartbeat:apache): Started nodeB <---
# Who is automatically started on the right node:
root@nodeA:/root#> ps -ef | grep -v grep | grep httpd
root@nodeA:/root#>
root@nodeB:/root#> ps -ef | grep -v grep | grep httpd
root 20504 1 0 17:36 ? 00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache 20505 20504 0 17:36 ? 00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache 20506 20504 0 17:36 ? 00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache 20507 20504 0 17:36 ? 00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache 20508 20504 0 17:36 ? 00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache 20509 20504 0 17:36 ? 00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
[ONE] pcs status
Cluster name: lar_cluster
Stack: corosync
Current DC: nodeB (version 1.1.16-12.el7-94ff4df) - partition with quorum
Last updated: Tue Feb 13 17:40:34 2018
Last change: Tue Feb 13 17:36:24 2018 by root via cibadmin on nodeA
2 nodes configured
4 resources configured
Online: [ nodeA nodeB ]
Full list of resources:
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeB
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeB
lar_cl-ip01 (ocf::heartbeat:IPaddr2): Started nodeB
lar_cl-wb01 (ocf::heartbeat:apache): Started nodeB
Daemon Status:
corosync: active/enabled
pacemaker: active/enabled
pcsd: active/enabled
# A test on the URL shows that our active/passive cluster configuration is ok and that the
# service is accesible:
[ONE] wget --spider http://192.168.56.111/index.html
Spider mode enabled. Check if remote file exists.
--2018-02-14 15:31:02-- http://192.168.56.111/index.html
Connecting to 192.168.56.111:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 62 [text/html]
Remote file exists and could contain further links,
but recursion is disabled -- not retrieving.
# so it is after switching the service to the other node (first I had to clear some
# remaining constraints - we will see that in a different recipe)
[ONE] pcs resource move lar_cl-rg01 nodeA
[ONE] pcs status
Cluster name: lar_cluster
Stack: corosync
Current DC: nodeB (version 1.1.16-12.el7-94ff4df) - partition with quorum
Last updated: Wed Feb 14 16:36:42 2018
Last change: Wed Feb 14 16:36:33 2018 by root via crm_resource on nodeA
2 nodes configured
4 resources configured
Online: [ nodeA nodeB ]
Full list of resources:
Resource Group: lar_cl-rg01
lar_cl-lv01 (ocf::heartbeat:LVM): Started nodeA
lar_cl-fs01 (ocf::heartbeat:Filesystem): Started nodeA
lar_cl-ip01 (ocf::heartbeat:IPaddr2): Started nodeA
lar_cl-wb01 (ocf::heartbeat:apache): Started nodeA
Daemon Status:
corosync: active/enabled
pacemaker: active/enabled
pcsd: active/enabled
[ONE] wget --spider http://192.168.56.111/index.html
Spider mode enabled. Check if remote file exists.
--2018-02-14 16:39:04-- http://192.168.56.111/index.html
Connecting to 192.168.56.111:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 62 [text/html]
Remote file exists and could contain further links,
but recursion is disabled -- not retrieving.
# My active/passive cluster is ready to work!
|