RHCS: Configure an active/backup pacemaker cluster

RHCS: Configure an active/backup pacemaker cluster

# Tested on CentOS 7


# Note: For the commands here after, [ALL] indicates that command has to be run on the two
#       nodes and [ONE] indicates that one needs to run it only on one of the hosts.

# This recipe focuses on the shared storage case where two nodes have access to the shared
# volume but only one node can actively mount and write to it at a time.


# I'll work on a two-node basic pacemaker cluster like the one we built on:


# I'll create a simple active/passive cluster. For that I will add a simple resource like
# an IP, that will allow me to connect to either of the nodes where IP will be active, a
# web service serving pages on the previously configured IP and a filesystem where will
# reside the web pages being served.
 


# Fencing
# ------------------------------------------------------------------------------------------

# There are many different topologies for fencing as to explain each of them in this recipe
# so, for the moment, I will disable it:

[ONE] pcs property set stonith-enabled=false



# Clustered Volume
# ------------------------------------------------------------------------------------------

# The first step is to configure a volume that will be able to failover between nodes. For
# that, we have to install the extensions to LVM2 to support clusters:

[ALL] yum install lvm2-cluster

# Then, create the PV and VG for the clustered filesystem:

[ONE] pvcreate /dev/sdb

[ONE] vgcreate vg_shared /dev/sdb
[ONE] lvcreate -l 100%VG -n lv_shared vg_shared

# Deactivate and reactivate shared volume group with an exclusive lock

[ONE] vgchange -an vg_shared

[ONE] vgchange -aey vg_shared

# And format the new logical volume

[ONE] mkfs.ext4 /dev/vg_shared/lv_shared

 
# Now, we have to update the /etc/lvm/lvm.conf configuration file so the CLVM volume is
# never auto-mounted. For that we will limit the volume list to be auto-mounted to 'rootvg'
(this is the name of my rootvg) and the volume groups tagged with the hostname (cluster
will take care of tagging volume groups at activation time):

[ALL] LINE=$((`grep -n "# volume_list" /etc/lvm/lvm.conf | cut -f1 -d':'`+1))

[ALL] sed -i.bak "${LINE}i\        volume_list = [ \"rootvg\", \"@`hostname -s`\" ]" \
         /etc/lvm/lvm.conf

# Update initramfs and reboot

[ALL] cp /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak
 
[ALL] dracut -H -f /boot/initramfs-$(uname -r).img $(uname -r)
 
[ALL] shutdown -r now

# Now we have a logical volume that is accessible from both nodes but inactive for both
# of them

root@nodeA:/root#> lvs
  LV        VG        Attr       LSize    Pool Origin Data%  Meta%  Move Log [...]
  lv_depot  rootvg    -wi-ao----    5.00g
  lv_root   rootvg    -wi-ao----    2.00g
  lv_swap   rootvg    -wi-ao----    2.00g
  lv_var    rootvg    -wi-ao----    2.00g
  lv_shared vg_shared -wi------- 1020.00m

root@nodeB:/root#> lvs
  LV        VG        Attr       LSize    Pool Origin Data%  Meta%  Move Log [...]
  lv_depot  rootvg    -wi-ao----    5.00g
  lv_root   rootvg    -wi-ao----    2.00g
  lv_swap   rootvg    -wi-ao----    2.00g
  lv_var    rootvg    -wi-ao----    2.00g
  lv_shared vg_shared -wi------- 1020.00m



# We are ready to create the LVM resource ("lar_cl-lv01" is the name of the new resource
# and "lar_cl-rg01" the name of the new Resource Group cluster)

[ONE] pcs resource create lar_cl-lv01 ocf:heartbeat:LVM volgrpname=vg_shared \
         exclusive=true --group lar_cl-rg01


# Look at this, resource has been started on one of the nodes of the cluster:

[ONE] pcs status | grep lar_cl-lv01

     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeA

# and, therefore, logical volume should be active on that node. Let's check it:

root@nodeA:/root#> lvs
  LV        VG        Attr       LSize    Pool Origin Data%  Meta%  Move Log [...]
  lv_depot  rootvg    -wi-ao----    5.00g
  lv_root   rootvg    -wi-ao----    2.00g
  lv_swap   rootvg    -wi-ao----    2.00g
  lv_var    rootvg    -wi-ao----    2.00g
  lv_shared vg_shared -wi-a----- 1020.00m

root@nodeB:/root#> lvs
  LV        VG        Attr       LSize    Pool Origin Data%  Meta%  Move Log [...]
  lv_depot  rootvg    -wi-ao----    5.00g
  lv_root   rootvg    -wi-ao----    2.00g
  lv_swap   rootvg    -wi-ao----    2.00g
  lv_var    rootvg    -wi-ao----    2.00g
  lv_shared vg_shared -wi------- 1020.00m

# Well, so far so good.


# Let's do the same thing for the filesystem resource (pay attention to the Resource Group
# name, it must be the the one used before for the LVM resource):

[ONE] pcs resource create lar_cl-fs01 ocf:heartbeat:Filesystem \
         device="/dev/vg_shared/lv_shared" directory="/cluster_fs_01" fstype="ext4" \
         --group lar_cl-rg01

[ONE] pcs status | grep "lar_cl-"
 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeA
     lar_cl-fs01        (ocf:heartbeat:Filesystem):    Started nodeA    <---

# Filesystem has been mounted on the right node (obvious):

root@nodeA:/root#> df -h | grep shared
/dev/mapper/vg_shared-lv_shared  988M  2.6M  919M   1% /cluster_fs_01

root@nodeB:/root#> df -h | grep shared
root@nodeB:/root#>


# Let's check if manual failover works fine:

[ONE] pcs status | grep "lar_cl-"
 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeA
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeA

[ONE] pcs resource move lar_cl-fs01 nodeB

[ONE] pcs status | grep "lar_cl-"
 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeB
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeB

root@nodeA:/root#> df -h | grep shared
root@nodeA:/root#>

root@nodeB:/root#> df -h | grep shared
/dev/mapper/vg_shared-lv_shared  988M  2.6M  919M   1% /cluster_fs_01


# And now, to check automatic failover, I will forcefully poweroff nodeB:

root@nodeB:/root#> echo 'o' > /proc/sysrq-trigger

# we can see that Resource Group failed over back to nodeA:

root@nodeA:/root#> pcs status | grep "lar_cl-"
 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeA
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeA

root@nodeA:/root#> df -h | grep lv_shared
/dev/mapper/vg_shared-lv_shared  988M  2.6M  919M   1% /cluster_fs_01



# Everything worked as expected ...the only thing is that as soon as powered-off node
# becomes available again, Resource Group will switch back to it. In order to minimize the
# unavailability of services we may not want Resource Group switching back automatically. I
# will take care of this behaviour on a different recipe.


# Let's continue.


# Now I'm creating an IP resource, for our future clustered web server:

[ONE] pcs resource create lar_cl-ip01 ocf:heartbeat:IPaddr2 ip=192.168.56.111 \
         cidr_netmask=24 op monitor interval=30s

# After a little while IP becomes available at cluster level:

[ONE] pcs status
Cluster name: lar_cluster
Stack: corosync
Current DC: nodeA (version 1.1.16-12.el7-94ff4df) - partition with quorum
Last updated: Tue Feb 13 16:57:16 2018
Last change: Tue Feb 13 16:57:02 2018 by root via cibadmin on nodeA

2 nodes configured
3 resources configured

Online: [ nodeA nodeB ]

Full list of resources:

 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeB
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeB
 lar_cl-ip01    (ocf::heartbeat:IPaddr2):       Started nodeA   <---

Daemon Status:
  corosync: active/enabled
  pacemaker: active/enabled
  pcsd: active/enabled

# And is mounted
 
root@pacem01:/root#> grep lar_cl-ip01 /etc/hosts
192.168.56.111 lar_cl-ip01

root@nodeA:/root#> ip a s eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 08:00:27:5f:e2:2c brd ff:ff:ff:ff:ff:ff
    inet 192.168.56.101/24 brd 192.168.56.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet 192.168.56.111/24 brd 192.168.56.255 scope global secondary eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe5f:e22c/64 scope link
       valid_lft forever preferred_lft forever

 
# What we've done here is to define a resource of type ocf:heartbeat:IPaddr2 this is:

# ocf: is the standard to which the resource script conforms and where to find it.
# heartbeat: is standard-specific; for OCF resources, it tells the cluster which OCF
#       namespace the resource script is in.
# IPaddr2: is the name of the resource script.

# To obtain a list of the available resource standards, OCF providers and resource agents,
# run:

[ONE] pcs resource standards
ocf
lsb
service
systemd
stonith

[ONE] pcs resource providers
heartbeat
openstack
pacemaker

[ONE] pcs resource agents ocf:heartbeat
CTDB
Delay
Dummy
Filesystem
IPaddr
IPaddr2
[...]
rsyncd
slapd
symlink
tomcat



# The only thing is that IP resource doesn't belong to the LVM/filesystem Resource Group
# (I forgot to indicate "--group lar_cl-rg01" when creating the resource and we need
# everything to work together, so I will move IP resource to the right Resource Group by
# running following command:

[ONE] pcs resource group add lar_cl-rg01 lar_cl-ip01

# In addition, this will stop/start IP resource in the right node if necessary:

[ONE] pcs status
[...]

Full list of resources:

 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeB
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeB
     lar_cl-ip01        (ocf::heartbeat:IPaddr2):       Started nodeB   <---
[...]
 


# Ok, we have a Resource Group with a filesystem and an IP. Let's create an associated
# service on the top of that. I will add an Apache HTTP Server as a service.

[ALL] yum install httpd wget

# Create a page to serve. As long as the site must be available on a service that may
# failover between nodes, we have to create the file on the clustered filesystem we
# created previously. Then, on the right node (cluster filesystem must be mounted):

[ONE] mkdir -p /cluster_fs_01/www/html
[ONE] cat << EOF >/cluster_fs_01/www/html/index.html
 <html>
    <body>My clustered Apache service</body>
 </html>
EOF

# And as the default Apache document root is /var/www/html:

[ALL] ln -s /cluster_fs_01/www/html/index.html /var/www/html/index.html


# Then, in order to monitor the health of my Apache instance, and recover it if it fails,
# the resource agent used by Pacemaker assumes the server-status URL is available:

[ALL] cat << EOF >/etc/httpd/conf.d/status.conf
 <Location /server-status>
    SetHandler server-status
    Require local
 </Location>
EOF


# Finally, I create the resrouce for my web site in the right Resource Group

[ONE] pcs resource create lar_cl-wb01 ocf:heartbeat:apache \
      configfile=/etc/httpd/conf/httpd.conf \
      statusurl="http://localhost/server-status" \
      op monitor interval=1min --group lar_cl-rg01


[ONE] pcs status | grep "lar_cl-"
 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeB
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeB
     lar_cl-ip01        (ocf::heartbeat:IPaddr2):       Started nodeB
     lar_cl-wb01        (ocf::heartbeat:apache):        Started nodeB   <---
 
# Who is automatically started on the right node:
 
root@nodeA:/root#> ps -ef | grep -v grep | grep httpd
root@nodeA:/root#>

root@nodeB:/root#> ps -ef | grep -v grep | grep httpd
root     20504     1  0 17:36 ?  00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache   20505 20504  0 17:36 ?  00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache   20506 20504  0 17:36 ?  00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache   20507 20504  0 17:36 ?  00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache   20508 20504  0 17:36 ?  00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]
apache   20509 20504  0 17:36 ?  00:00:00 /sbin/httpd -DSTATUS -f /etc/httpd/conf/httpd[...]


 
[ONE] pcs status
Cluster name: lar_cluster
Stack: corosync
Current DC: nodeB (version 1.1.16-12.el7-94ff4df) - partition with quorum
Last updated: Tue Feb 13 17:40:34 2018
Last change: Tue Feb 13 17:36:24 2018 by root via cibadmin on nodeA

2 nodes configured
4 resources configured

Online: [ nodeA nodeB ]

Full list of resources:

 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeB
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeB
     lar_cl-ip01        (ocf::heartbeat:IPaddr2):       Started nodeB
     lar_cl-wb01        (ocf::heartbeat:apache):        Started nodeB

Daemon Status:
  corosync: active/enabled
  pacemaker: active/enabled
  pcsd: active/enabled


# A test on the URL shows that our active/passive cluster configuration is ok and that the
# service is accesible:

[ONE] wget --spider http://192.168.56.111/index.html
Spider mode enabled. Check if remote file exists.
--2018-02-14 15:31:02--  http://192.168.56.111/index.html
Connecting to 192.168.56.111:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 62 [text/html]
Remote file exists and could contain further links,
but recursion is disabled -- not retrieving.


# so it is after switching the service to the other node (first I had to clear some
# remaining constraints - we will see that in a different recipe)

[ONE] pcs resource move lar_cl-rg01 nodeA

[ONE] pcs status
Cluster name: lar_cluster
Stack: corosync
Current DC: nodeB (version 1.1.16-12.el7-94ff4df) - partition with quorum
Last updated: Wed Feb 14 16:36:42 2018
Last change: Wed Feb 14 16:36:33 2018 by root via crm_resource on nodeA

2 nodes configured
4 resources configured

Online: [ nodeA nodeB ]

Full list of resources:

 Resource Group: lar_cl-rg01
     lar_cl-lv01        (ocf::heartbeat:LVM):   Started nodeA
     lar_cl-fs01        (ocf::heartbeat:Filesystem):    Started nodeA
     lar_cl-ip01        (ocf::heartbeat:IPaddr2):       Started nodeA
     lar_cl-wb01        (ocf::heartbeat:apache):        Started nodeA

Daemon Status:
  corosync: active/enabled
  pacemaker: active/enabled
  pcsd: active/enabled


[ONE] wget --spider http://192.168.56.111/index.html
Spider mode enabled. Check if remote file exists.
--2018-02-14 16:39:04--  http://192.168.56.111/index.html
Connecting to 192.168.56.111:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 62 [text/html]
Remote file exists and could contain further links,
but recursion is disabled -- not retrieving.


# My active/passive cluster is ready to work!
0 (0)
Article Rating (No Votes)
Rate this article
Attachments
There are no attachments for this article.
Comments
There are no comments for this article. Be the first to post a comment.
Full Name
Email Address
Security Code Security Code
Related Articles RSS Feed
RHCS6: Show/Add GFS2/GFS journals
Viewed 12671 times since Sun, Jun 3, 2018
Transform XML to CSV Format | Unix String Pattern Manipulation The Ugly Way
Viewed 5285 times since Sun, Jan 9, 2022
Automatic YUM Updates with Yum-cron
Viewed 10547 times since Fri, Oct 26, 2018
Tip: SSD and Linux. Enable TRIM and check if it works
Viewed 16940 times since Fri, May 15, 2020
stunnel basics and pki standards
Viewed 9398 times since Fri, Sep 28, 2018
Method 2 – Use shell scripts How to install yum cron on a CentOS/RHEL 6.x/7.x
Viewed 4074 times since Tue, Dec 4, 2018
RHEL7: How to get started with Firewalld.
Viewed 11995 times since Wed, May 22, 2019
LVM: Extend an existing Logical Volume / Filesystem
Viewed 2400 times since Sat, Jun 2, 2018
How to stop and disable auditd on RHEL 7
Viewed 39795 times since Tue, Aug 6, 2019
BIND for the Small LAN
Viewed 3336 times since Sun, May 20, 2018