#!/bin/sh
#
# Description:  Manages a PostgreSQL Server as an OCF High-Availability
#               resource
#
# Authors:      Serge Dubrouski (sergeyfd@gmail.com) -- original RA
#               Florian Haas (florian@linbit.com) -- makeover
#               Takatoshi MATSUO (matsuo.tak@gmail.com) -- support replication
#               David Corlette (dcorlette@netiq.com) -- add support for non-standard library locations and non-standard port
#
# Copyright:    2006-2012 Serge Dubrouski <sergeyfd@gmail.com>
#                         and other Linux-HA contributors
# License:      GNU General Public License (GPL)
#
###############################################################################
# Initialization:

: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs

# Use runuser if available for SELinux.
if [ -x /sbin/runuser ]; then
    SU=runuser
else
    SU=su
fi

#
# Get PostgreSQL Configuration parameter
#
get_pgsql_param() {
    local param_name

    param_name=$1
    perl_code="if (/^\s*$param_name[\s=]+\s*(.*)$/) {
       \$dir=\$1;
       \$dir =~ s/\s*\#.*//;
       \$dir =~ s/^'(\S*)'/\$1/;
       print \$dir;}"

    perl -ne "$perl_code" < $OCF_RESKEY_config
}

# Defaults
OCF_RESKEY_pgctl_default=/usr/bin/pg_ctl
OCF_RESKEY_psql_default=/usr/bin/psql
OCF_RESKEY_pgdata_default=/var/lib/pgsql/data
OCF_RESKEY_pgdba_default=postgres
OCF_RESKEY_pghost_default=""
OCF_RESKEY_pgport_default=5432
OCF_RESKEY_pglibs_default=/usr/lib
OCF_RESKEY_start_opt_default=""
OCF_RESKEY_ctl_opt_default=""
OCF_RESKEY_pgdb_default=template1
OCF_RESKEY_logfile_default=/dev/null
OCF_RESKEY_socketdir_default=""
OCF_RESKEY_stop_escalate_default=90
OCF_RESKEY_monitor_user_default=""
OCF_RESKEY_monitor_password_default=""
OCF_RESKEY_monitor_sql_default="select now();"
OCF_RESKEY_check_wal_receiver_default="false"
# Defaults for replication
OCF_RESKEY_rep_mode_default=none
OCF_RESKEY_node_list_default=""
OCF_RESKEY_restore_command_default=""
OCF_RESKEY_archive_cleanup_command_default=""
OCF_RESKEY_recovery_end_command_default=""
OCF_RESKEY_master_ip_default=""
OCF_RESKEY_repuser_default="postgres"
OCF_RESKEY_primary_conninfo_opt_default=""
OCF_RESKEY_restart_on_promote_default="false"
OCF_RESKEY_tmpdir_default="/var/lib/pgsql/tmp"
OCF_RESKEY_xlog_check_count_default="3"
OCF_RESKEY_crm_attr_timeout_default="5"
OCF_RESKEY_stop_escalate_in_slave_default=90
OCF_RESKEY_replication_slot_name_default=""

: ${OCF_RESKEY_pgctl=${OCF_RESKEY_pgctl_default}}
: ${OCF_RESKEY_psql=${OCF_RESKEY_psql_default}}
: ${OCF_RESKEY_pgdata=${OCF_RESKEY_pgdata_default}}
: ${OCF_RESKEY_pgdba=${OCF_RESKEY_pgdba_default}}
: ${OCF_RESKEY_pghost=${OCF_RESKEY_pghost_default}}
: ${OCF_RESKEY_pgport=${OCF_RESKEY_pgport_default}}
: ${OCF_RESKEY_pglibs=${OCF_RESKEY_pglibs_default}}
: ${OCF_RESKEY_config=${OCF_RESKEY_pgdata}/postgresql.conf}
: ${OCF_RESKEY_start_opt=${OCF_RESKEY_start_opt_default}}
: ${OCF_RESKEY_ctl_opt=${OCF_RESKEY_ctl_opt_default}}
: ${OCF_RESKEY_pgdb=${OCF_RESKEY_pgdb_default}}
: ${OCF_RESKEY_logfile=${OCF_RESKEY_logfile_default}}
: ${OCF_RESKEY_socketdir=${OCF_RESKEY_socketdir_default}}
: ${OCF_RESKEY_stop_escalate=${OCF_RESKEY_stop_escalate_default}}
: ${OCF_RESKEY_monitor_user=${OCF_RESKEY_monitor_user_default}}
: ${OCF_RESKEY_monitor_password=${OCF_RESKEY_monitor_password_default}}
: ${OCF_RESKEY_monitor_sql=${OCF_RESKEY_monitor_sql_default}}
: ${OCF_RESKEY_check_wal_receiver=${OCF_RESKEY_check_wal_receiver_default}}

# for replication
: ${OCF_RESKEY_rep_mode=${OCF_RESKEY_rep_mode_default}}
: ${OCF_RESKEY_node_list=${OCF_RESKEY_node_list_default}}
: ${OCF_RESKEY_restore_command=${OCF_RESKEY_restore_command_default}}
: ${OCF_RESKEY_archive_cleanup_command=${OCF_RESKEY_archive_cleanup_command_default}}
: ${OCF_RESKEY_recovery_end_command=${OCF_RESKEY_recovery_end_command_default}}
: ${OCF_RESKEY_master_ip=${OCF_RESKEY_master_ip_default}}
: ${OCF_RESKEY_repuser=${OCF_RESKEY_repuser_default}}
: ${OCF_RESKEY_primary_conninfo_opt=${OCF_RESKEY_primary_conninfo_opt_default}}
: ${OCF_RESKEY_restart_on_promote=${OCF_RESKEY_restart_on_promote_default}}
: ${OCF_RESKEY_tmpdir=${OCF_RESKEY_tmpdir_default}}
: ${OCF_RESKEY_xlog_check_count=${OCF_RESKEY_xlog_check_count_default}}
: ${OCF_RESKEY_crm_attr_timeout=${OCF_RESKEY_crm_attr_timeout_default}}
: ${OCF_RESKEY_stop_escalate_in_slave=${OCF_RESKEY_stop_escalate_in_slave_default}}
: ${OCF_RESKEY_replication_slot_name=${OCF_RESKEY_replication_slot_name_default}}

usage() {
    cat <<EOF
        usage: $0 start|stop|status|monitor|promote|demote|notify|meta-data|validate-all|methods

        $0 manages a PostgreSQL Server as an HA resource.

        The 'start' operation starts the PostgreSQL server.
        The 'stop' operation stops the PostgreSQL server.
        The 'status' operation reports whether the PostgreSQL is up.
        The 'monitor' operation reports whether the PostgreSQL is running.
        The 'promote' operation promotes the PostgreSQL server.
        The 'demote' operation demotes the PostgreSQL server.
        The 'validate-all' operation reports whether the parameters are valid.
        The 'methods' operation reports on the methods $0 supports.
EOF
  return $OCF_ERR_ARGS
}

meta_data() {
    cat <<EOF
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="pgsql" version="1.0">
<version>1.0</version>

<longdesc lang="en">
Resource script for PostgreSQL. It manages a PostgreSQL as an HA resource.
</longdesc>
<shortdesc lang="en">Manages a PostgreSQL database instance</shortdesc>

<parameters>
<parameter name="pgctl" unique="0" required="0">
<longdesc lang="en">
Path to pg_ctl command.
</longdesc>
<shortdesc lang="en">pgctl</shortdesc>
<content type="string" default="${OCF_RESKEY_pgctl_default}" />
</parameter>

<parameter name="start_opt" unique="0" required="0">
<longdesc lang="en">
Start options (-o start_opt in pg_ctl). "-i -p 5432" for example.
</longdesc>
<shortdesc lang="en">start_opt</shortdesc>
<content type="string" default="${OCF_RESKEY_start_opt_default}" />

</parameter>
<parameter name="ctl_opt" unique="0" required="0">
<longdesc lang="en">
Additional pg_ctl options (-w, -W etc..).
</longdesc>
<shortdesc lang="en">ctl_opt</shortdesc>
<content type="string" default="${OCF_RESKEY_ctl_opt_default}" />
</parameter>

<parameter name="psql" unique="0" required="0">
<longdesc lang="en">
Path to psql command.
</longdesc>
<shortdesc lang="en">psql</shortdesc>
<content type="string" default="${OCF_RESKEY_psql_default}" />
</parameter>

<parameter name="pgdata" unique="0" required="0">
<longdesc lang="en">
Path to PostgreSQL data directory.
</longdesc>
<shortdesc lang="en">pgdata</shortdesc>
<content type="string" default="${OCF_RESKEY_pgdata_default}" />
</parameter>

<parameter name="pgdba" unique="0" required="0">
<longdesc lang="en">
User that owns PostgreSQL.
</longdesc>
<shortdesc lang="en">pgdba</shortdesc>
<content type="string" default="${OCF_RESKEY_pgdba_default}" />
</parameter>

<parameter name="pghost" unique="0" required="0">
<longdesc lang="en">
Hostname/IP address where PostgreSQL is listening
</longdesc>
<shortdesc lang="en">pghost</shortdesc>
<content type="string" default="${OCF_RESKEY_pghost_default}" />
</parameter>

<parameter name="pgport" unique="0" required="0">
<longdesc lang="en">
Port where PostgreSQL is listening
</longdesc>
<shortdesc lang="en">pgport</shortdesc>
<content type="integer" default="${OCF_RESKEY_pgport_default}" />
</parameter>

<parameter name="pglibs" unique="0" required="0">
<longdesc lang="en">
Custom location of the Postgres libraries. If not set, the standard location
will be used.
</longdesc>
<shortdesc lang="en">pglibs</shortdesc>
<content type="string" default="${OCF_RESKEY_pglibs_default}" />
</parameter>

<parameter name="monitor_user" unique="0" required="0">
<longdesc lang="en">
PostgreSQL user that pgsql RA will user for monitor operations. If it's not set
pgdba user will be used.
</longdesc>
<shortdesc lang="en">monitor_user</shortdesc>
<content type="string" default="${OCF_RESKEY_monitor_user_default}" />
</parameter>

<parameter name="monitor_password" unique="0" required="0">
<longdesc lang="en">
Password for monitor user.
</longdesc>
<shortdesc lang="en">monitor_password</shortdesc>
<content type="string" default="${OCF_RESKEY_monitor_password_default}" />
</parameter>

<parameter name="monitor_sql" unique="0" required="0">
<longdesc lang="en">
SQL script that will be used for monitor operations.
</longdesc>
<shortdesc lang="en">monitor_sql</shortdesc>
<content type="string" default="${OCF_RESKEY_monitor_sql_default}" />
</parameter>

<parameter name="config" unique="0" required="0">
<longdesc lang="en">
Path to the PostgreSQL configuration file for the instance.
</longdesc>
<shortdesc lang="en">Configuration file</shortdesc>
<content type="string" default="${OCF_RESKEY_pgdata}/postgresql.conf" />
</parameter>

<parameter name="pgdb" unique="0" required="0">
<longdesc lang="en">
Database that will be used for monitoring.
</longdesc>
<shortdesc lang="en">pgdb</shortdesc>
<content type="string" default="${OCF_RESKEY_pgdb_default}" />
</parameter>

<parameter name="logfile" unique="0" required="0">
<longdesc lang="en">
Path to PostgreSQL server log output file.
</longdesc>
<shortdesc lang="en">logfile</shortdesc>
<content type="string" default="${OCF_RESKEY_logfile_default}" />
</parameter>

<parameter name="socketdir" unique="0" required="0">
<longdesc lang="en">
Unix socket directory for PostgreSQL.

If you use PostgreSQL 9.3 or higher and define unix_socket_directories in the postgresql.conf, then you must set socketdir to determine which directory is used for psql command.
</longdesc>
<shortdesc lang="en">socketdir</shortdesc>
<content type="string" default="${OCF_RESKEY_socketdir_default}" />
</parameter>

<parameter name="stop_escalate" unique="0" required="0">
<longdesc lang="en">
Number of seconds to wait for stop (using -m fast) before resorting to -m immediate
</longdesc>
<shortdesc lang="en">stop escalation</shortdesc>
<content type="integer" default="${OCF_RESKEY_stop_escalate_default}" />
</parameter>

<parameter name="rep_mode" unique="0" required="0">
<longdesc lang="en">
Replication mode may be set to "async" or "sync" or "slave".
They require PostgreSQL 9.1 or later.
Once set, "async" and "sync" require node_list, master_ip, and
restore_command parameters,as well as configuring PostgreSQL
for replication (in postgresql.conf and pg_hba.conf).

"slave" means that RA only makes recovery.conf before starting
to connect to primary which is running somewhere.
It doesn't need master/slave setting.
It requires master_ip restore_command parameters.
</longdesc>
<shortdesc lang="en">rep_mode</shortdesc>
<content type="string" default="${OCF_RESKEY_rep_mode_default}" />
</parameter>

<parameter name="node_list" unique="0" required="0">
<longdesc lang="en">
All node names. Please separate each node name with a space.
This is optional for replication. Defaults to all nodes in the cluster
</longdesc>
<shortdesc lang="en">node list</shortdesc>
<content type="string" default="${OCF_RESKEY_node_list_default}" />
</parameter>

<parameter name="restore_command" unique="0" required="0">
<longdesc lang="en">
restore_command for recovery.conf.
This is required for replication.
</longdesc>
<shortdesc lang="en">restore_command</shortdesc>
<content type="string" default="${OCF_RESKEY_restore_command_default}" />
</parameter>

<parameter name="archive_cleanup_command" unique="0" required="0">
<longdesc lang="en">
archive_cleanup_command for recovery.conf.
This is used for replication and is optional.
</longdesc>
<shortdesc lang="en">archive_cleanup_command</shortdesc>
<content type="string" default="${OCF_RESKEY_archive_cleanup_command_default}" />
</parameter>

<parameter name="recovery_end_command" unique="0" required="0">
<longdesc lang="en">
recovery_end_command for recovery.conf.
This is used for replication and is optional.
</longdesc>
<shortdesc lang="en">recovery_end_command</shortdesc>
<content type="string" default="${OCF_RESKEY_recovery_end_command_default}" />
</parameter>

<parameter name="master_ip" unique="0" required="0">
<longdesc lang="en">
Master's floating IP address to be connected from hot standby.
This parameter is used for "primary_conninfo" in recovery.conf.
This is required for replication.
</longdesc>
<shortdesc lang="en">master ip</shortdesc>
<content type="string" default="${OCF_RESKEY_master_ip_default}" />
</parameter>

<parameter name="repuser" unique="0" required="0">
<longdesc lang="en">
User used to connect to the master server.
This parameter is used for "primary_conninfo" in recovery.conf.
This is required for replication.
</longdesc>
<shortdesc lang="en">repuser</shortdesc>
<content type="string" default="${OCF_RESKEY_repuser_default}" />
</parameter>

<parameter name="primary_conninfo_opt" unique="0" required="0">
<longdesc lang="en">
primary_conninfo options of recovery.conf except host, port, user and application_name.
This is optional for replication.
</longdesc>
<shortdesc lang="en">primary_conninfo_opt</shortdesc>
<content type="string" default="${OCF_RESKEY_primary_conninfo_opt_default}" />
</parameter>

<parameter name="restart_on_promote" unique="0" required="0">
<longdesc lang="en">
If this is true, RA deletes recovery.conf and restarts PostgreSQL
on promote to keep Timeline ID. It probably makes fail-over slower.
It's recommended to set on-fail of promote up as fence.
This is optional for replication.
</longdesc>
<shortdesc lang="en">restart_on_promote</shortdesc>
<content type="boolean" default="${OCF_RESKEY_restart_on_promote_default}" />
</parameter>

<parameter name="replication_slot_name" unique="0" required="0">
<longdesc lang="en">
Set this option when using replication slots.
Can only use lower case letters, numbers and underscore for replication_slot_name.

The replication slots would be created for each node, with the name adding the node name as postfix.
For example, replication_slot_name is "sample" and 2 slaves which are "node1" and "node2" connect to
their slots, the slots names are "sample_node1" and "sample_node2".
If the node name contains a upper case letter, hyphen and dot, those characters will be converted to a lower case letter or an underscore.
For example, Node-1.example.com to node_1_example_com.

pgsql RA doesn't monitor and delete the replication slot.
When the slave node has been disconnected in failure or the like, execute one of the following manually.
Otherwise it may eventually cause a disk full because the master node will continue to accumulate the unsent WAL.
1. recover and reconnect the slave node to the master node as soon as possible.
2. delete the slot on the master node by following psql command.
$ select pg_drop_replication_slot('replication_slot_name');
</longdesc>
<shortdesc lang="en">replication_slot_name</shortdesc>
<content type="string" default="${OCF_RESKEY_replication_slot_name_default}" />
</parameter>

<parameter name="tmpdir" unique="0" required="0">
<longdesc lang="en">
Path to temporary directory.
This is optional for replication.
</longdesc>
<shortdesc lang="en">tmpdir</shortdesc>
<content type="string" default="${OCF_RESKEY_tmpdir_default}" />
</parameter>

<parameter name="xlog_check_count" unique="0" required="0">
<longdesc lang="en">
Number of checks of xlog on monitor before promote.
This is optional for replication.

Note: For backward compatibility, the terms are unified with PostgreSQL 9.
      If you are using PostgreSQL 10 or later, replace "xlog" with "wal".
      Likewise, replacing "location" with "lsn".
</longdesc>
<shortdesc lang="en">xlog check count</shortdesc>
<content type="integer" default="${OCF_RESKEY_xlog_check_count_default}" />
</parameter>

<parameter name="crm_attr_timeout" unique="0" required="0">
<longdesc lang="en">
The timeout of crm_attribute forever update command.
Default value is 5 seconds.
This is optional for replication.
</longdesc>
<shortdesc lang="en">The timeout of crm_attribute forever update command.</shortdesc>
<content type="integer" default="${OCF_RESKEY_crm_attr_timeout_default}" />
</parameter>

<parameter name="stop_escalate_in_slave" unique="0" required="0">
<longdesc lang="en">
Number of seconds to wait for stop (using -m fast) before resorting to -m immediate
in slave state.
This is optional for replication.
</longdesc>
<shortdesc lang="en">stop escalation_in_slave</shortdesc>
<content type="integer" default="${OCF_RESKEY_stop_escalate_in_slave_default}" />
</parameter>

<parameter name="check_wal_receiver" unique="0" required="0">
<longdesc lang="en">
If this is true, RA checks wal_receiver process on monitor
and notifies its status using "(resource name)-receiver-status" attribute.
It's useful for checking whether PostgreSQL (hot standby) connects to primary.
The attribute shows status as "normal" or "normal (master)" or "ERROR".
Note that if you configure PostgreSQL as master/slave resource, then
wal receiver is not running in the master and the attribute shows status as
"normal (master)" consistently because it is normal status.
</longdesc>
<shortdesc lang="en">check_wal_receiver</shortdesc>
<content type="boolean" default="${OCF_RESKEY_check_wal_receiver_default}" />
</parameter>
</parameters>

<actions>
<action name="start" timeout="120s" />
<action name="stop" timeout="120s" />
<action name="status" timeout="60s" />
<action name="monitor" depth="0" timeout="30s" interval="30s"/>
<action name="monitor" depth="0" timeout="30s" interval="29s" role="Promoted" />
<action name="promote" timeout="120s" />
<action name="demote" timeout="120s" />
<action name="notify"   timeout="90s" />
<action name="meta-data" timeout="5s" />
<action name="validate-all" timeout="5s" />
<action name="methods" timeout="5s" />
</actions>
</resource-agent>
EOF
}


#
#   Run the given command in the Resource owner environment...
#
runasowner() {
    local quietrun=""
    local loglevel="-err"
    local var

    for var in 1 2
    do
        case "$1" in
            "-q")
                quietrun="-q"
                shift 1;;
            "info"|"warn"|"err"|"debug")
                loglevel="-$1"
                shift 1;;
            *)
                ;;
        esac
    done

    ocf_run $quietrun $loglevel $SU $OCF_RESKEY_pgdba -c "cd $OCF_RESKEY_pgdata; $*"
}

#
#       Shell escape
#
escape_string() {
    echo "$*" | sed -e "s/'/'\\\\''/g"
}


#
# methods: What methods/operations do we support?
#

pgsql_methods() {
    cat <<EOF
    start
    stop
    status
    monitor
    promote
    demote
    notify
    methods
    meta-data
    validate-all
EOF
}


# Execulte SQL and return the result.
exec_sql() {
    local sql="$1"
    local output
    local rc

    output=`$SU $OCF_RESKEY_pgdba -c "cd $OCF_RESKEY_pgdata; \
                $OCF_RESKEY_psql $psql_options -U $OCF_RESKEY_pgdba \
                -Atc \"$sql\""`
    rc=$?

    echo $output
    return $rc
}


#pgsql_real_start: Starts PostgreSQL
pgsql_real_start() {
    local pgctl_options
    local postgres_options
    local rc

    pgsql_real_monitor info
    rc=$?
    if [ $rc -eq $OCF_SUCCESS -o $rc -eq $OCF_RUNNING_MASTER ]; then
        ocf_log info "PostgreSQL is already running. PID=`cat $PIDFILE`"
        if is_replication; then
            return $OCF_ERR_GENERIC
        else
            return $OCF_SUCCESS
        fi
    fi

    # Remove postmaster.pid if it exists
    rm -f $PIDFILE

    # Remove backup_label if it exists
    if [ -f $BACKUPLABEL ] && ! is_replication; then
        ocf_log info "Removing $BACKUPLABEL. The previous backup might have failed."
        rm -f $BACKUPLABEL
    fi

    # Check if we need to create a log file
    if ! check_log_file $OCF_RESKEY_logfile
    then
        ocf_exit_reason "PostgreSQL can't write to the log file: $OCF_RESKEY_logfile"
        return $OCF_ERR_PERM
    fi

    # Check socket directory
    if [ -n "$OCF_RESKEY_socketdir" ]
    then
        check_socket_dir
    fi

    check_stat_temp_directory

    if [ "$OCF_RESKEY_rep_mode" = "slave" ]; then
        rm -f $RECOVERY_CONF
        make_recovery_conf || return $OCF_ERR_GENERIC
    fi

    # Set options passed to pg_ctl
    pgctl_options="$OCF_RESKEY_ctl_opt -D $OCF_RESKEY_pgdata -l $OCF_RESKEY_logfile"

    # Set options passed to the PostgreSQL server process
    postgres_options="-c config_file=${OCF_RESKEY_config}"

    if [ -n "$OCF_RESKEY_pghost" ]; then
        postgres_options="$postgres_options -h $OCF_RESKEY_pghost"
    fi
    if [ -n "$OCF_RESKEY_start_opt" ]; then
        postgres_options="$postgres_options $OCF_RESKEY_start_opt"
    fi

    # Tack pass-through options onto pg_ctl options
    pgctl_options="$pgctl_options -o '$postgres_options'"

    # Invoke pg_ctl
    runasowner "unset PGUSER; unset PGPASSWORD; $OCF_RESKEY_pgctl $pgctl_options -W start"

    if [ $? -eq 0 ]; then
        # Probably started.....
        ocf_log info "PostgreSQL start command sent."
    else
        ocf_exit_reason "Can't start PostgreSQL."
        return $OCF_ERR_GENERIC
    fi

    while :
    do
        pgsql_real_monitor warn
        rc=$?
        if [ $rc -eq $OCF_SUCCESS -o $rc -eq $OCF_RUNNING_MASTER ]; then
            break;
        fi
        sleep 1
        ocf_log debug "PostgreSQL still hasn't started yet. Waiting..."
    done

    # delete replication slots on all nodes. On master node will be created during promotion.
    if use_replication_slot; then
        delete_replication_slots
        if [ $? -eq $OCF_ERR_GENERIC ]; then
            ocf_exit_reason "PostgreSQL can't clean up replication_slot."
            return $OCF_ERR_GENERIC
        fi
    fi

    ocf_log info "PostgreSQL is started."
    return $rc
}

pgsql_replication_start() {
    local rc
    local synchronous_standby_names

    # initializing for replication
    change_pgsql_status "$NODENAME" "STOP"
    delete_master_baseline
    exec_with_retry 0 ocf_promotion_score -v $CAN_NOT_PROMOTE
    rm -f ${XLOG_NOTE_FILE}.* $REP_MODE_CONF $RECOVERY_CONF
    if ! make_recovery_conf || ! delete_xlog_location || ! set_async_mode_all; then
        return $OCF_ERR_GENERIC
    fi

    if [ -f $PGSQL_LOCK ]; then
        ocf_exit_reason "My data may be inconsistent. You have to remove $PGSQL_LOCK file to force start."
        return $OCF_ERR_GENERIC
    fi

    # start
    pgsql_real_start
    if [ $? -ne $OCF_SUCCESS ]; then
        return $OCF_ERR_GENERIC
    fi

    synchronous_standby_names=$(exec_sql "${CHECK_SYNCHRONOUS_STANDBY_NAMES_SQL}")
    if [ -n "${synchronous_standby_names}" ]; then
        ocf_exit_reason "Invalid synchronous_standby_names is set in postgresql.conf."
        return $OCF_ERR_CONFIGURED
    fi

    change_pgsql_status "$NODENAME" "HS:alone"
    return $OCF_SUCCESS
}

#pgsql_start: pgsql_real_start() wrapper for replication
pgsql_start() {
    if ! is_replication; then
        pgsql_real_start
        return $?
    else
        pgsql_replication_start
        return $?
    fi
}

#pgsql_promote: Promote PostgreSQL
pgsql_promote() {
    local output
    local target
    local rc

    if ! is_replication; then
        ocf_exit_reason "Not in a replication mode."
        return $OCF_ERR_CONFIGURED
    fi

    output=`exec_sql "${CHECK_MS_SQL}"`
    if [ $? -ne 0 ]; then
        report_psql_error $rc $loglevel "Can't get PostgreSQL recovery status on promote."
        return $OCF_ERR_GENERIC
    fi

    if [ "$output" = "f" ]; then
        ocf_log info "PostgreSQL is already Master. Don't execute promote."
        return $OCF_SUCCESS
    fi

    rm -f ${XLOG_NOTE_FILE}.*

    for target in $NODE_LIST; do
        [ "$target" = "$NODENAME" ] && continue
        change_data_status "$target" "DISCONNECT"
        change_master_score "$target" "$CAN_NOT_PROMOTE"
    done

    ocf_log info "Creating $PGSQL_LOCK."
    touch $PGSQL_LOCK
    show_master_baseline

    if ocf_is_true ${OCF_RESKEY_restart_on_promote}; then
        ocf_log info "Restarting PostgreSQL instead of promote."
        #stop : this function returns $OCF_SUCCESS only.
        pgsql_real_stop slave
        if "${USE_STANDBY_SIGNAL}"; then
            rm -f ${OCF_RESKEY_pgdata}/standby.signal
        else
            rm -f $RECOVERY_CONF
        fi
        pgsql_real_start
        rc=$?
        if [ $rc -ne $OCF_RUNNING_MASTER ]; then
            ocf_exit_reason "Can't start PostgreSQL as primary on promote."
            if [ $rc -ne $OCF_SUCCESS ]; then
                change_pgsql_status "$NODENAME" "STOP"
            fi
            return $OCF_ERR_GENERIC
        fi
    else
        runasowner "$OCF_RESKEY_pgctl -D $OCF_RESKEY_pgdata -W promote"
        if [ $? -eq 0 ]; then
            ocf_log info "PostgreSQL promote command sent."
        else
            ocf_exit_reason "Can't promote PostgreSQL."
            return $OCF_ERR_GENERIC
        fi

        while :
        do
            pgsql_real_monitor warn
            rc=$?
            if [ $rc -eq $OCF_RUNNING_MASTER ]; then
                break;
            elif [ $rc -eq $OCF_ERR_GENERIC ]; then
                ocf_exit_reason "Can't promote PostgreSQL."
                return $rc
            fi
            sleep 1
            ocf_log debug "PostgreSQL still hasn't promoted yet. Waiting..."
        done
        ocf_log info "PostgreSQL is promoted."
    fi

    # create replication slots on master after promotion
    if use_replication_slot; then
	create_replication_slots
	if [ $? -eq $OCF_ERR_GENERIC ]; then
	    ocf_exit_reason "PostgreSQL can't create replication_slot."
	    return $OCF_ERR_GENERIC
	fi
    fi

    change_data_status "$NODENAME" "LATEST"
    exec_with_retry 0 ocf_promotion_score -v $PROMOTE_ME
    change_pgsql_status "$NODENAME" "PRI"
    return $OCF_SUCCESS
}

#pgsql_demote: Demote PostgreSQL
pgsql_demote() {
    local rc

    if ! is_replication; then
        ocf_exit_reason "Not in a replication mode."
        return $OCF_ERR_CONFIGURED
    fi

    exec_with_retry 0 ocf_promotion_score -v $CAN_NOT_PROMOTE
    delete_master_baseline

    if ! pgsql_status; then
        ocf_log info "PostgreSQL is already stopped on demote."
    else
        ocf_log info "Stopping PostgreSQL on demote."
        pgsql_real_stop master
        rc=$?
        if [ "$rc" -ne "$OCF_SUCCESS" ]; then
            change_pgsql_status "$NODENAME" "UNKNOWN"
            return $rc
        fi
    fi
    change_pgsql_status "$NODENAME" "STOP"
    return $OCF_SUCCESS
}

#pgsql_real_stop: Stop PostgreSQL
pgsql_real_stop() {
    local rc
    local count
    local stop_escalate

    if ocf_is_true ${OCF_RESKEY_check_wal_receiver}; then
        attrd_updater -n "$PGSQL_WAL_RECEIVER_STATUS_ATTR" -D
    fi

    if ! pgsql_status
    then
        #Already stopped
        return $OCF_SUCCESS
    fi

    stop_escalate=$OCF_RESKEY_stop_escalate
    if [ "$1" = "slave" ]; then
        stop_escalate="$OCF_RESKEY_stop_escalate_in_slave"
    fi
    # adjust stop_escalate time when it is longer than the timeout
    if [ -n "$OCF_RESKEY_CRM_meta_timeout" ] && \
        [ "$stop_escalate" -ge $((OCF_RESKEY_CRM_meta_timeout/1000)) ]; then
        stop_escalate=$(((OCF_RESKEY_CRM_meta_timeout/1000) - 10))
        ocf_log info "stop_escalate(or stop_escalate_in_slave) time is adjusted to ${stop_escalate} based on the configured timeout."
    fi

    # Stop PostgreSQL, do not wait for clients to disconnect
    if [ $stop_escalate -gt 0 ]; then
            runasowner "$OCF_RESKEY_pgctl -W -D $OCF_RESKEY_pgdata stop -m fast"
    fi

    # stop waiting
    count=0
    while [ $count -lt $stop_escalate ]
    do
        if ! pgsql_status
        then
            #PostgreSQL stopped
            break;
        fi
        count=`expr $count + 1`
        sleep 1
    done

    if pgsql_status
    then
        #PostgreSQL is still up. Use another shutdown mode.
        ocf_log info "PostgreSQL failed to stop after ${stop_escalate}s using -m fast. Trying -m immediate..."
        runasowner "$OCF_RESKEY_pgctl -W -D $OCF_RESKEY_pgdata stop -m immediate"
    fi

    while :
    do
        pgsql_real_monitor
        rc=$?
        if [ $rc -eq $OCF_NOT_RUNNING ]; then
            # An unnecessary debug log is prevented.
            break;
        fi
        sleep 1
        ocf_log debug "PostgreSQL still hasn't stopped yet. Waiting..."
    done

    # Remove postmaster.pid if it exists
    rm -f $PIDFILE

    if  [ "$1" = "master" -a "$OCF_RESKEY_CRM_meta_notify_slave_uname" = " " ]; then
        ocf_log info "Removing $PGSQL_LOCK."
        rm -f $PGSQL_LOCK
    fi
    return $OCF_SUCCESS
}

pgsql_replication_stop() {
    local rc

    exec_with_retry 5 ocf_promotion_score -v $CAN_NOT_PROMOTE
    delete_xlog_location

    if ! pgsql_status
    then
        ocf_log info "PostgreSQL is already stopped."
        change_pgsql_status "$NODENAME" "STOP"
        return $OCF_SUCCESS
    fi

    pgsql_real_stop slave
    rc=$?
    if [ $rc -ne $OCF_SUCCESS ]; then
        change_pgsql_status "$NODENAME" "UNKNOWN"
        return $rc
    fi

    change_pgsql_status "$NODENAME" "STOP"
    set_async_mode_all
    delete_master_baseline
    return $OCF_SUCCESS
}

#pgsql_stop: pgsql_real_stop() wrapper for replication
pgsql_stop() {
    if ! is_replication; then
        pgsql_real_stop
        return $?
    else
        pgsql_replication_stop
        return $?
    fi
}

#
# pgsql_status: is PostgreSQL up?
#

pgsql_status() {
     if [ -f $PIDFILE ]
     then
         PID=`head -n 1 $PIDFILE`
         runasowner -q debug "kill -s 0 $PID"
         return $?
     fi

     # No PID file
     false
}

pgsql_wal_receiver_status() {
    local PID
    local receiver_parent_pids
    local pgsql_real_monitor_status=$1

    PID=`head -n 1 $PIDFILE`
    receiver_parent_pids=`ps -ef | tr -s " " | grep "[w]al\s*receiver" | cut -d " " -f 3`

    if echo "$receiver_parent_pids" | grep -q -w "$PID" ; then
        attrd_updater -n "$PGSQL_WAL_RECEIVER_STATUS_ATTR" -v "normal"
        return 0
    fi

    if [ $pgsql_real_monitor_status -eq "$OCF_RUNNING_MASTER" ]; then
        attrd_updater -n "$PGSQL_WAL_RECEIVER_STATUS_ATTR" -v "normal (master)"
        return 0
    fi

    attrd_updater -n "$PGSQL_WAL_RECEIVER_STATUS_ATTR" -v "ERROR"
    ocf_log warn "wal receiver process is not running"
    return 1
}

#
# pgsql_real_monitor
#

pgsql_real_monitor() {
    local loglevel
    local rc
    local output

    # Set the log level of the error message
    loglevel=${1:-err}

    if ! pgsql_status
    then
        ocf_log info "PostgreSQL is down"
        return $OCF_NOT_RUNNING
    fi

    if is_replication; then
        #Check replication state
        output=`exec_sql "${CHECK_MS_SQL}"`
        rc=$?

        if [ $rc -ne  0 ]; then
            report_psql_error $rc $loglevel "Can't get PostgreSQL recovery status."
            return $OCF_ERR_GENERIC
        fi

        case "$output" in
            f)  ocf_log debug "PostgreSQL is running as a primary."
                if [ "$OCF_RESKEY_monitor_sql" = "$OCF_RESKEY_monitor_sql_default" ]; then
                    if ocf_is_probe; then
                        # Set initial score for primary.
                        exec_with_retry 0 ocf_promotion_score -v $PROMOTE_ME
                    fi
                    return $OCF_RUNNING_MASTER
                fi
                ;;

            t)  ocf_log debug "PostgreSQL is running as a hot standby."
                if ocf_is_probe; then
                    # Set initial score for hot standby.
                    exec_with_retry 0 ocf_promotion_score -v $CAN_NOT_PROMOTE
                fi
                return $OCF_SUCCESS;;

            *)  ocf_exit_reason "$CHECK_MS_SQL output is $output"
                return $OCF_ERR_GENERIC;;
        esac
    fi

    OCF_RESKEY_monitor_sql=`escape_string "$OCF_RESKEY_monitor_sql"`
    runasowner -q $loglevel "$OCF_RESKEY_psql $psql_options \
                  -c '$OCF_RESKEY_monitor_sql'"
    rc=$?
    if [ $rc -ne  0 ]; then
        report_psql_error $rc $loglevel "PostgreSQL $OCF_RESKEY_pgdb isn't running."
        return $OCF_ERR_GENERIC
    fi

    if is_replication; then
        return $OCF_RUNNING_MASTER
    fi
    return $OCF_SUCCESS
}

pgsql_replication_monitor() {
    local rc

    rc=$1
    if [ $rc -ne $OCF_SUCCESS -a $rc -ne "$OCF_RUNNING_MASTER" ]; then
        return $rc
    fi
    # If I am Master
    if [ $rc -eq $OCF_RUNNING_MASTER ]; then
        change_data_status "$NODENAME" "LATEST"
        change_pgsql_status "$NODENAME" "PRI"
        control_slave_status || return $OCF_ERR_GENERIC
        if [ "$RE_CONTROL_SLAVE" = "true" ]; then
            sleep 2
            ocf_log info "re-controlling slave status."
            RE_CONTROL_SLAVE="none"
            control_slave_status || return $OCF_ERR_GENERIC
        fi
        return $rc
    fi

    # I can't get master node name from $OCF_RESKEY_CRM_meta_notify_master_uname on monitor,
    # so I will get master node name using crm_mon -n
    print_crm_mon | grep -q -i -E "<resource id=\"${RESOURCE_NAME}\" .* role=\"(Promoted|Master)\""
    if [ $? -ne 0 ] ; then
        # If I am Slave and Master is not exist
        ocf_log info "Master does not exist."
        change_pgsql_status "$NODENAME" "HS:alone"
        have_master_right
        if [ $? -eq 0 ]; then
            rm -f ${XLOG_NOTE_FILE}.*
        fi
    else
        output=`exec_with_retry 0 $CRM_ATTR_FOREVER -N "$NODENAME" \
                -n "$PGSQL_DATA_STATUS_ATTR" -G -q`
        if [ "$output" = "DISCONNECT" ]; then
            change_pgsql_status "$NODENAME" "HS:alone"
        fi
    fi
    return $rc
}

#pgsql_monitor: pgsql_real_monitor() wrapper for replication
pgsql_monitor() {
    local rc

    pgsql_real_monitor
    rc=$?

    if ocf_is_true ${OCF_RESKEY_check_wal_receiver}; then
        pgsql_wal_receiver_status $rc
    fi

    if ! is_replication; then
        return $rc
    else
        pgsql_replication_monitor $rc
        return $?
    fi
}

# pgsql_post_demote
pgsql_post_demote() {
    DEMOTE_NODE=`echo $OCF_RESKEY_CRM_meta_notify_demote_uname | sed "s/ /\n/g" | head -1 | tr '[A-Z]' '[a-z]'`
    ocf_log debug "post-demote called. Demote uname is $DEMOTE_NODE"
    if [ "$DEMOTE_NODE" != "$NODENAME" ]; then
        if ! echo $OCF_RESKEY_CRM_meta_notify_master_uname | tr '[A-Z]' '[a-z]' | grep $NODENAME; then
            show_master_baseline
            change_pgsql_status "$NODENAME" "HS:alone"
        fi
    fi
    return $OCF_SUCCESS
}

pgsql_pre_promote() {
    local master_baseline
    local my_master_baseline
    local cmp_location
    local number_of_nodes

    # If my data is newer than new master's one, I fail my resource.
    PROMOTE_NODE=`echo $OCF_RESKEY_CRM_meta_notify_promote_uname | \
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       