Enhance galera to interact over multiple clusters

This change adds a new resource agent "stretch_galera"
which builds off of the existing "galera" agent.
To accommodate this, the "galera" agent's shell script
structure is modified slightly so that it can be sourced
for its functions.

The new resource agent adds a new parameter "remote_node_map"
to the Galera resource agent which allows it to consider
galera node names that are in other clusters as part of its
Galera quorum.  To achieve this, it launches read-only pcs
commands to the remote clusters in order to view and modify
remote state variables.

WIP.

Author: zzzeek

heartbeat/galera

@@ -25,7 +25,7 @@
 
 ##
 # README.
-# 
+#
 # This agent only supports being configured as a multistate Master
 # resource.
 #
@@ -49,15 +49,15 @@
 # pcs resource create db galera enable_creation=true \
 # wsrep_cluster_address="gcomm://rhel7-auto1,rhel7-auto2,rhel7-auto3" meta master-max=3 --master
 #
-# By setting the 'enable_creation' option, the database will be automatically 
+# By setting the 'enable_creation' option, the database will be automatically
 # generated at startup. The meta attribute 'master-max=3' means that all 3
 # nodes listed in the wsrep_cluster_address list will be allowed to connect
 # to the galera cluster and perform replication.
 #
 # NOTE: If you have more nodes in the pacemaker cluster then you wish
 # to have in the galera cluster, make sure to use location contraints to prevent
 # pacemaker from attempting to place a galera instance on a node that is
-# not in the 'wsrep_cluster_address" list. 
+# not in the 'wsrep_cluster_address" list.
 #
 ##
 
@@ -66,9 +66,11 @@
 
 : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
 . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
-. ${OCF_FUNCTIONS_DIR}/mysql-common.sh
 
-NODENAME=$(ocf_attribute_target)
+if [ "$__OCF_ACTION" != "meta-data" ]; then
+    . ${OCF_FUNCTIONS_DIR}/mysql-common.sh
+    NODENAME=$(ocf_attribute_target)
+fi
 
 # It is common for some galera instances to store
 # check user that can be used to query status
@@ -99,7 +101,9 @@ UEND
 }
 
 meta_data() {
-   cat <<END
+    extra_parameters="$1"
+
+    cat <<END
 <?xml version="1.0"?>
 <!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 <resource-agent name="galera">
@@ -247,6 +251,8 @@ Cluster check user password
 <content type="string" default="" />
 </parameter>
 
+${extra_parameters}
+
 </parameters>
 
 <actions>
@@ -329,7 +335,7 @@ get_last_commit()
 
     if [ -z "$node" ]; then
        ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
-    else 
+    else
        ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
     fi
 }
@@ -418,7 +424,7 @@ clear_master_score()
     local node=$(ocf_attribute_target $1)
     if [ -z "$node" ]; then
         $CRM_MASTER -D
-    else 
+    else
         $CRM_MASTER -D -N $node
     fi
 }
@@ -429,7 +435,7 @@ set_master_score()
 
     if [ -z "$node" ]; then
         $CRM_MASTER -v 100
-    else 
+    else
         $CRM_MASTER -N $node -v 100
     fi
 }
@@ -912,64 +918,70 @@ galera_validate()
     mysql_common_validate
 }
 
-case "$1" in
-  meta-data)    meta_data
-        exit $OCF_SUCCESS;;
-  usage|help)   usage
-        exit $OCF_SUCCESS;;
-esac
-
-galera_validate
-rc=$?
-LSB_STATUS_STOPPED=3
-if [ $rc -ne 0 ]; then
+cmd_main() {
     case "$1" in
-        stop) exit $OCF_SUCCESS;;
-        monitor) exit $OCF_NOT_RUNNING;;
-        status) exit $LSB_STATUS_STOPPED;;
-        *) exit $rc;;
+      meta-data)    meta_data
+            exit $OCF_SUCCESS;;
+      usage|help)   usage
+            exit $OCF_SUCCESS;;
     esac
-fi
 
-if [ -z "${OCF_RESKEY_check_passwd}" ]; then
-    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
-    OCF_RESKEY_check_passwd=${MYSQL_PASSWORD}
-fi
-if [ -z "${OCF_RESKEY_check_user}" ]; then
-    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
-    OCF_RESKEY_check_user=${MYSQL_USERNAME}
-fi
-: ${OCF_RESKEY_check_user="root"}
+    galera_validate
+    rc=$?
+    LSB_STATUS_STOPPED=3
+    if [ $rc -ne 0 ]; then
+        case "$1" in
+            stop) exit $OCF_SUCCESS;;
+            monitor) exit $OCF_NOT_RUNNING;;
+            status) exit $LSB_STATUS_STOPPED;;
+            *) exit $rc;;
+        esac
+    fi
 
-MYSQL_OPTIONS_CHECK="-nNE --user=${OCF_RESKEY_check_user}"
-if [ -n "${OCF_RESKEY_check_passwd}" ]; then
-    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK --password=${OCF_RESKEY_check_passwd}"
-fi
+    if [ -z "${OCF_RESKEY_check_passwd}" ]; then
+        # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+        OCF_RESKEY_check_passwd=${MYSQL_PASSWORD}
+    fi
+    if [ -z "${OCF_RESKEY_check_user}" ]; then
+        # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+        OCF_RESKEY_check_user=${MYSQL_USERNAME}
+    fi
+    : ${OCF_RESKEY_check_user="root"}
 
-# This value is automatically sourced from /etc/sysconfig/checkcluster if available
-if [ -n "${MYSQL_HOST}" ]; then
-    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -h ${MYSQL_HOST}"
-fi
+    MYSQL_OPTIONS_CHECK="-nNE --user=${OCF_RESKEY_check_user}"
+    if [ -n "${OCF_RESKEY_check_passwd}" ]; then
+        MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK --password=${OCF_RESKEY_check_passwd}"
+    fi
 
-# This value is automatically sourced from /etc/sysconfig/checkcluster if available
-if [ -n "${MYSQL_PORT}" ]; then
-    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -P ${MYSQL_PORT}"
-fi
+    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+    if [ -n "${MYSQL_HOST}" ]; then
+        MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -h ${MYSQL_HOST}"
+    fi
 
+    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+    if [ -n "${MYSQL_PORT}" ]; then
+        MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -P ${MYSQL_PORT}"
+    fi
 
+    # What kind of method was invoked?
+    case "$1" in
+      start)    galera_start;;
+      stop)     galera_stop;;
+      status)   mysql_common_status err;;
+      monitor)  galera_monitor;;
+      promote)  galera_promote;;
+      demote)   galera_demote;;
+      validate-all) exit $OCF_SUCCESS;;
+
+     *)     usage
+            exit $OCF_ERR_UNIMPLEMENTED;;
+    esac
+}
 
-# What kind of method was invoked?
-case "$1" in
-  start)    galera_start;;
-  stop)     galera_stop;;
-  status)   mysql_common_status err;;
-  monitor)  galera_monitor;;
-  promote)  galera_promote;;
-  demote)   galera_demote;;
-  validate-all) exit $OCF_SUCCESS;;
+# run 'main' if we aren't "sourceonly"
+if [ $1 != "sourceonly" ]; then
+    cmd_main "$@"
+fi
 
- *)     usage
-        exit $OCF_ERR_UNIMPLEMENTED;;
-esac
 
 # vi:sw=4:ts=4:et: