#!/bin/bash

set -eu

# shellcheck source=/dev/null
source "${BASE}/env_${ENVIRONMENT}"

all_ok=true

for host in "${AZURE_HOSTS[@]}"
do
  echo "Checking replication for host $host"
  echo
  if [ "$host" != "$AZURE_MASTER" ]
  then
    if ! echo "select pg_is_in_recovery()" \
      | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
      | grep -q 't'
    then
      >&2 echo "Host $host is not standby"
      all_ok=false
    fi
  else
    if ! echo "select pg_is_in_recovery()" \
      | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
      | grep -q 'f'
    then
      >&2 echo "Host $host is not master"
      all_ok=false
    fi
    if ! echo "select count(1) from pg_stat_replication" \
      | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
      | grep -q '4'
    then
      >&2 echo "Host $host is not replicated by 4 nodes:"
      echo "select * from pg_stat_replication" \
        | >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres
      all_ok=false
    fi
    for slave_host in "${AZURE_SLAVES[@]}" "$GCP_MASTER_CANDIDATE"
    do
      if ! echo "select client_addr||'-'||state from pg_stat_replication" \
        | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
        | grep -q "$(host -t A "$slave_host"|cut -d ' ' -f 4)/32-streaming"
      then
        >&2 echo "Host $host is not correclty replicated by host $slave_host:"
        echo "select * from pg_stat_replication" \
          | >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres
        all_ok=false
      fi
    done
  fi
  echo
done

for host in "${GCP_HOSTS[@]}"
do
  echo "Checking replication for host $host"
  echo
  if [ "$host" != "$GCP_MASTER_CANDIDATE" ]
  then
    if ! echo "select pg_is_in_recovery()" \
      | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
      | grep -q 't'
    then
      >&2 echo "Host $host is not standby"
      all_ok=false
    fi
  else
    if ! echo "select pg_is_in_recovery()" \
      | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
      | grep -q 't'
    then
      >&2 echo "Host $host is not standby"
      all_ok=false
    fi
    if ! echo "select count(1) from pg_stat_replication" \
      | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
      | grep -q '3'
    then
      >&2 echo "Host $host is not replicated by 3 nodes:"
      echo "select * from pg_stat_replication" \
        | >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres
      all_ok=false
    fi
    for slave_host in "${GCP_SLAVES[@]}"
    do
      if ! echo "select client_addr||'-'||state from pg_stat_replication" \
        | ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres -A -t \
        | grep -q "$(host -t A "$slave_host"|cut -d ' ' -f 4)/32-streaming"
      then
        >&2 echo "Host $host is not correclty replicated by host $slave_host:"
        echo "select * from pg_stat_replication" \
          | >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d postgres
        all_ok=false
      fi
    done
  fi
  echo
done

$all_ok