051-check-gcp-replication-delay.sh 1.12 KB
Newer Older
1 2 3 4
#!/bin/bash

set -eu

5
# shellcheck source=/dev/null
6 7
source "${BASE}/env_${ENVIRONMENT}"

8 9
tombstone_msg=$(date +'%Y%m%d_%H%M%S')"_${ENVIRONMENT}"
echo "Insert '$tombstone_msg' into tombstone" 
Matteo Melli's avatar
Matteo Melli committed
10
echo "insert into tombstone(note) values('${tombstone_msg}') returning *" \
11
  | ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -v ON_ERROR_STOP=1 -d tombstone
12 13 14
# wait until the change is propagated
while true
do
Matteo Melli's avatar
Matteo Melli committed
15
  find_new_msg="$(echo "select created_at from tombstone where note = '$tombstone_msg'" \
16
    | ssh_remote "$GCP_MASTER_CANDIDATE" sudo gitlab-psql -v ON_ERROR_STOP=1 -A -t -d tombstone)"
17 18
  if [[ -z "${find_new_msg+x}" ]] || [[ "$find_new_msg" == "" ]]
  then
Matteo Melli's avatar
Matteo Melli committed
19
    gcp_cur_rep_delay="$(echo "select round(extract(epoch from (now() - pg_last_xact_replay_timestamp())))" \
20
      | ssh_remote "$GCP_MASTER_CANDIDATE" sudo gitlab-psql -v ON_ERROR_STOP=1 -A -t -d postgres)"
21 22 23 24 25 26 27
    echo "New tombstone message is not seen on $GCP_MASTER_CANDIDATE (GCP MASTER CANDIDATE). The replication delay: ${gcp_cur_rep_delay}s. Wait 3 seconds..."
    sleep 3
  else
    echo "New tombstone message arrived to $GCP_MASTER_CANDIDATE."
    break
  fi
done