Commit c42e037b authored by Matteo Melli's avatar Matteo Melli
Browse files

Fixed some bugs

parent 48da8a37
......@@ -6,7 +6,7 @@ set -eu
source "${BASE}/env_${ENVIRONMENT}"
echo "Create tombstone database and table if not already existing"
ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-c "drop database if exists tombstone; create database tombstone"
ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql tombstone \
-c "create table if not exists tombstone (created_at timestamptz default now() primary key, note text)"
echo "drop database if exists tombstone; create database tombstone" \
| ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d postgres
echo "create table if not exists tombstone (created_at timestamptz default now() primary key, note text)" \
| ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d tombstone
......@@ -7,15 +7,17 @@ source "${BASE}/env_${ENVIRONMENT}"
tombstone_msg=$(date +'%Y%m%d_%H%M%S')"_${ENVIRONMENT}"
echo "Insert '$tombstone_msg' into tombstone"
ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql tombstone -c "insert into tombstone(note) values('${tombstone_msg}') returning *"
echo "insert into tombstone(note) values('${tombstone_msg}') returning *" \
| ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d tombstone
# wait until the change is propagated
while true
do
find_new_msg="$(ssh_remote "$GCP_MASTER_CANDIDATE" sudo gitlab-psql -Atd tombstone -c "select created_at from tombstone where note = '$tombstone_msg'")"
find_new_msg="$(echo "select created_at from tombstone where note = '$tombstone_msg'" \
| ssh_remote "$GCP_MASTER_CANDIDATE" sudo gitlab-psql -A -t -d tombstone)"
if [[ -z "${find_new_msg+x}" ]] || [[ "$find_new_msg" == "" ]]
then
gcp_cur_rep_delay="$(ssh_remote "$GCP_MASTER_CANDIDATE"
sudo gitlab-psql -Atd postgres -c "select round(extract(epoch from (now() - pg_last_xact_replay_timestamp())))")"
gcp_cur_rep_delay="$(echo "select round(extract(epoch from (now() - pg_last_xact_replay_timestamp())))" \
| ssh_remote "$GCP_MASTER_CANDIDATE" sudo gitlab-psql -A -t -d postgres)"
echo "New tombstone message is not seen on $GCP_MASTER_CANDIDATE (GCP MASTER CANDIDATE). The replication delay: ${gcp_cur_rep_delay}s. Wait 3 seconds..."
sleep 3
else
......
......@@ -5,5 +5,5 @@ set -eu
# shellcheck source=/dev/null
source "${BASE}/env_${ENVIRONMENT}"
ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d gitlab_repmgr -c \
"TRUNCATE repmgr_gitlab_cluster.repl_nodes"
echo "TRUNCATE repmgr_gitlab_cluster.repl_nodes" \
| ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d gitlab_repmgr
......@@ -5,12 +5,13 @@ set -eu
# shellcheck source=/dev/null
source "${BASE}/env_${ENVIRONMENT}"
if ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select pg_is_in_recovery()" | grep -q 't'
if echo "select pg_is_in_recovery()" \
| ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d postgres -t -A \
| grep -q 't'
then
echo "$AZURE_MASTER is standby"
return 0
exit 0
else
>&2 echo "$AZURE_MASTER is standby"
return 1
>&2 echo "$AZURE_MASTER is NOT standby"
exit 1
fi
......@@ -5,18 +5,18 @@ set -eu
# shellcheck source=/dev/null
source "${BASE}/env_${ENVIRONMENT}"
azure_master_lsn="$(ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select case when pg_is_in_recovery()
azure_master_lsn="$(echo "select case when pg_is_in_recovery()
then pg_last_xlog_replay_location()
else pg_current_xlog_location() end;")"
gcp_master_candidate_lsn="$(ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select case when pg_is_in_recovery()
else pg_current_xlog_location() end;")" \
| ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d postgres -t -A
gcp_master_candidate_lsn="$(echo "select case when pg_is_in_recovery()
then pg_last_xlog_replay_location()
else pg_current_xlog_location() end;")"
else pg_current_xlog_location() end;" \
| ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql gitlab-psql -d postgres -t -A)"
if [ "$azure_master_lsn" == "$gcp_master_candidate_lsn" ]
then
echo "GCP and Azure have same LSN: $azure_master_lsn"
return 0
exit 0
fi
echo "GCP and Azure have NOT same LSN. Current LSNs are: Azure/$azure_master_lsn GCP/$gcp_master_candidate_lsn"
reutrn 1
exit 1
......@@ -5,12 +5,13 @@ set -eu
# shellcheck source=/dev/null
source "${BASE}/env_${ENVIRONMENT}"
if ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select pg_is_in_recovery()" | grep -q 'f'
if echo "select pg_is_in_recovery()" \
| ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql gitlab-psql -d postgres -t -A \
| grep -q 'f'
then
echo "$GCP_MASTER_CANDIDATE is master"
return 0
exit 0
else
>&2 echo "$GCP_MASTER_CANDIDATE is master"
return 1
>&2 echo "$GCP_MASTER_CANDIDATE is NOT master"
exit 1
fi
......@@ -9,8 +9,8 @@ echo "Checking state of $GCP_MASTER_CANDIDATE"
if ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-consul gitlab-ctl repmgr-check-master 2> /dev/null
then
echo "$GCP_MASTER_CANDIDATE is repmgr master"
return 0
exit 0
else
>&2 echo "$GCP_MASTER_CANDIDATE is not repmgr master"
return 1
exit 1
fi
......@@ -8,11 +8,11 @@ source "${BASE}/env_${ENVIRONMENT}"
for host in "${GCP_PGBOUNCERS[@]}"
do
echo "Check pgbouncer on $host"
ssh_remote "$host" gitlab-ctl pgb-console -c "SHOW DATABASES"
ssh_remote "$host" gitlab-ctl pgb-console -c "SHOW SERVERS"
echo "SHOW DATABASES" | ssh_remote "$host" gitlab-ctl pgb-console
echo "SHOW SERVERS" | ssh_remote "$host" gitlab-ctl pgb-console
read -r -s -N 1 -p "Press [y] to continue, any other key to abort." key
if [ "$key" != "y" ]
then
return 1
exit 1
fi
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment