0

I've a cluster setup with 3 nodes [having nodes A,B (VIP),C] using pacemaker and corosync. When I pull network cable one of the nodes B intentionally (as a disaster recovery test), the node A or C takes VIP but when I put cable back B on after sometime then VIP switches to B which should not be the case.

I would expect A or C to retain VIP, below is my pacemaker config

configure
primitive baseos-ping-check ocf:pacemaker:ping params host_list="1.2.3.4" multiplier="1000" dampen="0" attempts="2" \
        op start interval="0s" timeout="60s" \
        op monitor interval="2s" timeout="60s" \
        op stop interval="0s" timeout="60s" on-fail="ignore"
primitive baseos-vip-master ocf:heartbeat:IPaddr2 \
        params ip="192.67.23.145" iflabel="MR" cidr_netmask="255.255.255.0" \
        op start interval="0s" \
        op monitor interval="10s" \
        op stop interval="0s"
clone cl_baseos-ping-check baseos-ping-check meta interleave="true"
location loc-vip-master vip-master \
        rule $id="loc-vip-master-rule" $role="master" 100: #uname eq ECS01 \
        rule $id="loc--vip-master-rule-0" $role="master" -inf: not_defined pingd or pingd lte 0
property expected-quorum-votes="1"
property stonith-enabled="false"
property maintenance-mode="false"
property cluster-recheck-interval="5min"
property default-action-timeout="60s"
property pe-error-series-max="500"
property pe-input-series-max="500"
property pe-warn-series-max="500"
property no-quorum-policy="ignore"
property dc-version="1.1.16-94ff4df"
property cluster-infrastructure="corosync"
rsc_defaults resource-stickiness="150"
rsc_defaults migration-threshold="3"
commit
quit

My corosync config looks like:

quorum {
    provider: corosync_votequorum
    expected_votes : 3
}


totem {
    version: 2

    # How long before declaring a token lost (ms)
    token: 3000

    # How many token retransmits before forming a new configuration
    token_retransmits_before_loss_const: 10

    # How long to wait for join messages in the membership protocol (ms)
    join: 60

    # How long to wait for consensus to be achieved before starting a new round of membership configuration (ms)
    consensus: 3600

    # Turn off the virtual synchrony filter
    vsftype: none

    # Number of messages that may be sent by one processor on receipt of the token
    max_messages: 20

    # Limit generated nodeids to 31-bits (positive signed integers)
    clear_node_high_bit: yes

    # Disable encryption
    secauth: on

    # How many threads to use for encryption/decryption
    threads: 0

    # Optionally assign a fixed node id (integer)
    # nodeid: 1234

    # This specifies the mode of redundant ring, which may be none, active, or passive.
    rrp_mode: none

    interface {
        # The following values need to be set based on your environment
        ringnumber: 0
        bindnetaddr: 10.98.4.0
        #mcastaddr: 0.0.0.0
        mcastport: 5876
        member {
            memberaddr: 10.98.4.103
        }
        member {
            memberaddr: 10.98.4.173
        }
    }
    transport: udpu
}

amf {
    mode: disabled
}

service {
    # Load the Pacemaker Cluster Resource Manager
    ver:       0
    name:      pacemaker
}

aisexec {
        user:   root
        group:  root
}

logging {
        fileline: off
        to_stderr: yes
        to_logfile: no
        to_syslog: yes
        syslog_facility: daemon
        debug: off
        timestamp: on
        logger_subsys {
                subsys: AMF
                debug: off
                tags: enter|leave|trace1|trace2|trace3|trace4|trace6
        }
}


My cib.xml looks like below:

<cib crm_feature_set="3.0.11" validate-with="pacemaker-2.6" epoch="4" num_updates="0" admin_epoch="0" cib-last-written="Wed Sep 11 14:33:08 2019" update-origin="testje" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="183387233">
  <configuration>
    <crm_config>
      <cluster_property_set id="cib-bootstrap-options">
        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-94ff4df"/>
        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
        <nvpair name="expected-quorum-votes" value="1" id="cib-bootstrap-options-expected-quorum-votes"/>
        <nvpair name="stonith-enabled" value="false" id="cib-bootstrap-options-stonith-enabled"/>
        <nvpair name="maintenance-mode" value="false" id="cib-bootstrap-options-maintenance-mode"/>
        <nvpair name="cluster-recheck-interval" value="5min" id="cib-bootstrap-options-cluster-recheck-interval"/>
        <nvpair name="default-action-timeout" value="60s" id="cib-bootstrap-options-default-action-timeout"/>
        <nvpair name="pe-error-series-max" value="500" id="cib-bootstrap-options-pe-error-series-max"/>
        <nvpair name="pe-input-series-max" value="500" id="cib-bootstrap-options-pe-input-series-max"/>
        <nvpair name="pe-warn-series-max" value="500" id="cib-bootstrap-options-pe-warn-series-max"/>
        <nvpair name="no-quorum-policy" value="ignore" id="cib-bootstrap-options-no-quorum-policy"/>
      </cluster_property_set>
    </crm_config>
    <nodes>
      <node id="183387233" uname="testje"/>
      <node id="183387244" uname="0060E057BC25"/>
      <node id="183387230" uname="d66c4b1997a0"/>
    </nodes>
    <resources>
      <primitive id="baseos-vip-master" class="ocf" provider="heartbeat" type="IPaddr2">
        <instance_attributes id="baseos-vip-master-instance_attributes">
          <nvpair name="ip" value="10.238.68.134" id="baseos-vip-master-instance_attributes-ip"/>
          <nvpair name="iflabel" value="MR" id="baseos-vip-master-instance_attributes-iflabel"/>
          <nvpair name="cidr_netmask" value="24" id="baseos-vip-master-instance_attributes-cidr_netmask"/>
        </instance_attributes>
        <operations>
          <op name="start" interval="0s" id="baseos-vip-master-start-0s"/>
          <op name="monitor" interval="10s" id="baseos-vip-master-monitor-10s"/>
          <op name="stop" interval="0s" id="baseos-vip-master-stop-0s"/>
        </operations>
      </primitive>
      <clone id="cl_baseos-ping-check">
        <meta_attributes id="cl_baseos-ping-check-meta_attributes">
          <nvpair name="interleave" value="true" id="cl_baseos-ping-check-meta_attributes-interleave"/>
        </meta_attributes>
        <primitive id="baseos-ping-check" class="ocf" provider="pacemaker" type="ping">
          <instance_attributes id="baseos-ping-check-instance_attributes">
            <nvpair name="host_list" value="10.238.68.1" id="baseos-ping-check-instance_attributes-host_list"/>
            <nvpair name="multiplier" value="1000" id="baseos-ping-check-instance_attributes-multiplier"/>
            <nvpair name="dampen" value="0" id="baseos-ping-check-instance_attributes-dampen"/>
            <nvpair name="attempts" value="2" id="baseos-ping-check-instance_attributes-attempts"/>
          </instance_attributes>
          <operations>
            <op name="start" interval="0s" timeout="60s" id="baseos-ping-check-start-0s"/>
            <op name="monitor" interval="2s" timeout="60s" id="baseos-ping-check-monitor-2s"/>
            <op name="stop" interval="0s" timeout="60s" on-fail="ignore" id="baseos-ping-check-stop-0s"/>
          </operations>
        </primitive>
      </clone>
    </resources>
    <constraints>
      <rsc_location id="loc-baseos-vip-master" rsc="baseos-vip-master">
        <rule id="loc-baseos-vip-master-rule" role="master" score="100">
          <expression attribute="#uname" operation="eq" value="testje" id="loc-baseos-vip-master-rule-expression"/>
        </rule>
        <rule id="loc-baseos-vip-master-rule-0" role="master" score="-INFINITY" boolean-op="or">
          <expression attribute="pingd" operation="not_defined" id="loc-baseos-vip-master-rule-0-expression"/>
          <expression attribute="pingd" operation="lte" value="0" id="loc-baseos-vip-master-rule-0-expression-0"/>
        </rule>
      </rsc_location>
    </constraints>
    <rsc_defaults>
      <meta_attributes id="rsc-options">
        <nvpair name="resource-stickiness" value="150" id="rsc-options-resource-stickiness"/>
        <nvpair name="migration-threshold" value="3" id="rsc-options-migration-threshold"/>
      </meta_attributes>
    </rsc_defaults>
  </configuration>
</cib>

The scenario which I described above happens only when I pull network cable of a node to make it offline, however if I reboot the node (i.e. B) then the VIP is sticking to current node i.e either A or C.

One thing which I noticed that when I put back network cable of Node B then IPaddr2 resource is invoking findif which is failing, since I'm not using nic name parameter but I do provide cidr_netmask, so ideally findif should resolve the ip address of node B.

Is there a way I can avoid failing of findif?

5
  • this bad behaviour is not related to the cable, but to the resource stickness, you can find more information in the clusterlabs site
    – c4f4t0r
    Sep 12, 2019 at 7:30
  • @c4f4t0r I've resource sickness in place resource-stickiness="150" even then it is not sticking to node
    – Akhil
    Sep 12, 2019 at 8:39
  • you can try to use crm_simulate to get more information about this
    – c4f4t0r
    Sep 12, 2019 at 8:54
  • What is happening on B while it's unplugged? Is the VIP stopping or failing to stop when the node B is unplugged? Disabling STONITH and ignoring quorum is a recipe for strange behavior. Sep 12, 2019 at 17:27
  • 1
    @MattKereczman the IPaddr2 continues to run on node B and tried to find node id using findif and continues to fail, what I suspect is that when node B comes online then the cluster is in split brain situation because the VIP resource is still assigned to it and then pacemaker decide to make B as master i.e. switch VIP to node B
    – Akhil
    Sep 13, 2019 at 4:52

1 Answer 1

0

As we've worked out in the comments under your question: When the node is plugged back into the network it rejoins the cluster, the VIP is found to be running on more than one node, so the cluster must recover the service (stop the VIP everywhere and then start it back up) and it happens to be picking node B.

In a production cluster, you would use Fencing/STONITH and not ignore quorum. When you unplug node B from the network in that configuration the out of band STONITH agent would forcibly power off node B, therefore causing node B to rejoin the cluster in a "fresh state" with no services running and the VIP would not failback to node B.

3
  • so, to configure a fence do I need to have an extra piece of hardware which will kill other node in case it becomes unreachable? Because I'm not sure if I configure resource level fencing and if I pull out cable then how the fence device would reach out to the offline node to kill (restart) it because the network is down
    – Akhil
    Sep 16, 2019 at 5:15
  • @Apollo Usually yes, you'd require a piece of hardware like a generic IPMI device, Dell Drac card, HP ILO card, etc. Smart PDU's can achieve this as well. If you're running these nodes as VMs in a hypervisor, there are fence agents for libvirt, xen, vmware, openstack, etc. Here is a link to the projects github: github.com/ClusterLabs/fence-agents Sep 16, 2019 at 17:59
  • Actually I'm creating a base cluster setup which can run on VMs and on separate physical machines, so I would require a hardware and fence agents for both cases
    – Akhil
    Sep 17, 2019 at 5:09

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .