Commit aa8e7d92 authored by Matteo Melli's avatar Matteo Melli
Browse files

Sourcing env_<environment> in each script (so can be run independently)

parent 5f6cf999
Pipeline #88675 failed with stage
in 17 seconds
......@@ -126,7 +126,7 @@
1. [ ] 🐘 {+ Database-Wrangler +}: Ensure repmgr has desired states on all database nodes
* Ensure repmgr has desired states on all database nodes. `/opt/gitlab-migration/bin/scripts/01_preflight/050_configuration_checks/110-check-repmgr-state.sh`
* Check replication topology is the expected. `/opt/gitlab-migration/bin/scripts/01_preflight/050_configuration_checks/111-check-topology.sh`
* Check replication topology is the expected. `/opt/gitlab-migration/bin/scripts/01_preflight/050_configuration_checks/111-check-replication-topology.sh`
## Ensure Geo replication is up to date
......
......@@ -48,3 +48,10 @@ pgbouncer-02-db-gprd.c.gitlab-production.internal
)
# Generic
function ssh_remote(){
ssh "$@"
}
export -f ssh_remote
......@@ -2,7 +2,6 @@
AZURE_MASTER="postgres02.db.stg.gitlab.com"
GCP_MASTER_CANDIDATE="postgres-02-db-gstg.c.gitlab-staging-1.internal"
AZURE_HOSTS=(
postgres01.db.stg.gitlab.com
postgres02.db.stg.gitlab.com
......@@ -42,3 +41,10 @@ pgbouncer-01-db-gstg.c.gitlab-staging-1.internal
)
# Generic
function ssh_remote(){
ssh "$@"
}
export -f ssh_remote
......@@ -2,7 +2,6 @@
AZURE_MASTER="postgres-dbteam-02.db.stg.gitlab.com"
GCP_MASTER_CANDIDATE="postgres-dbteam-02-db-gstg.c.gitlab-staging-1.internal"
AZURE_HOSTS=(
postgres-dbteam-04.db.stg.gitlab.com
postgres-dbteam-03.db.stg.gitlab.com
......@@ -18,7 +17,7 @@ postgres-dbteam-03-db-gstg.c.gitlab-staging-1.internal
)
AZURE_SLAVES=(
$(for host in "${GCP_HOSTS[@]}"
$(for host in "${AZURE_HOSTS[@]}"
do
if [ "$host" != "$AZURE_MASTER" ]
then
......@@ -47,3 +46,10 @@ postgres-dbteam-05-db-gstg.c.gitlab-staging-1.internal
)
# Generic
function ssh_remote(){
ssh $SSH_OPTS "$@"
}
export -f ssh_remote
......@@ -4,15 +4,16 @@ set -eu
[[ $# -lt 2 ]] && { echo "Specify the environment and role"; exit 1 ; }
# Because some of the script use one or another, we clone both,
# although we need to homogenize
# although we need to homogenize
export BASE="$(dirname "$0")"
export ENVIRONMENT=$1
export ROLE=$2
export GITLAB_ENV=$ENVIRONMENT
source source_vars
source env_${ENVIRONMENT} # That is, .env_staging or .env_production (test also supported)
source utilities
source steps_${ROLE}
source "${BASE}/source_vars"
source "${BASE}/env_${ENVIRONMENT}" # That is, .env_staging or .env_production (test also supported)
source "${BASE}/utilities"
source "${BASE}/steps_${ROLE}"
#Check all steps have a script
for step in "${steps[@]}"
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
for host in "${AZURE_HOSTS[@]}" "${GCP_HOSTS[@]}"
do
echo "Checking repmgr state for host $host"
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
all_ok=true
for host in "${AZURE_HOSTS[@]}"
......@@ -10,32 +12,43 @@ do
echo
if [ "$host" != "$AZURE_MASTER" ]
then
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t -c 'select pg_is_in_recovery()' | grep -q 't'
if ! echo "select pg_is_in_recovery()" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres -A -t \
| grep -q 't'
then
>&2 echo "Host $host is not standby"
all_ok=false
fi
else
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t -c 'select pg_is_in_recovery()' | grep -q 'f'
if ! echo "select pg_is_in_recovery()" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres -A -t \
| grep -q 'f'
then
>&2 echo "Host $host is not master"
all_ok=false
fi
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t -c 'select count(1) from pg_stat_replication' | grep -q '4'
if ! echo "select count(1) from pg_stat_replication" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres -A -t \
| grep -q '4'
then
>&2 echo "Host $host is not replicated by 4 nodes:"
>&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -c 'select * from pg_stat_replication'
echo "select * from pg_stat_replication" \
| >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres
all_ok=false
fi
for slave_host in "${AZURE_SLAVES[@]}" "$GCP_MASTER_CANDIDATE"
do
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t \
-c "select client_addr||'-'||state from pg_stat_replication"\
if ! echo "select client_addr||'-'||state from pg_stat_replication" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres -A -t \
| grep -q "$(host "$slave_host"|cut -d ' ' -f 4)-streaming"
>&2 echo "Host $host is not replicated by host $slave_host:"
>&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -c 'select * from pg_stat_replication'
all_ok=false
then
>&2 echo "Host $host is not correclty replicated by host $slave_host:"
echo "select * from pg_stat_replication" \
| >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres
all_ok=false
fi
done
fi
echo
done
......@@ -45,32 +58,43 @@ do
echo
if [ "$host" != "$GCP_MASTER_CANDIDATE" ]
then
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t -c 'select pg_is_in_recovery()' | grep -q 't'
if ! echo "select pg_is_in_recovery()" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres -A -t \
| grep -q 't'
then
>&2 echo "Host $host is not standby"
all_ok=false
fi
else
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t -c 'select pg_is_in_recovery()' | grep -q 'f'
if ! echo "select pg_is_in_recovery()" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres -A -t \
| grep -q 't'
then
>&2 echo "Host $host is not standby"
all_ok=false
fi
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t -c 'select count(1) from pg_stat_replication' | grep -q '3'
if ! echo "select count(1) from pg_stat_replication" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres -A -t \
| grep -q '3'
then
>&2 echo "Host $host is not replicated by 3 nodes:"
>&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -c 'select * from pg_stat_replication'
echo "select * from pg_stat_replication" \
| >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres
all_ok=false
fi
for slave_host in "${GCP_SLAVES[@]}"
do
if ! ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t \
-c "select client_addr||'-'||state from pg_stat_replication"\
if ! echo "select client_addr||'-'||state from pg_stat_replication" \
| ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -A -t \
| grep -q "$(host "$slave_host"|cut -d ' ' -f 4)-streaming"
>&2 echo "Host $host is not replicated by host $slave_host:"
>&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -c 'select * from pg_stat_replication'
all_ok=false
then
>&2 echo "Host $host is not correclty replicated by host $slave_host:"
echo "select * from pg_stat_replication" \
| >&2 ssh_remote "$host" sudo -u gitlab-psql gitlab-psql -d postgres
all_ok=false
fi
done
fi
echo
done
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
echo "Create tombstone database and table if not already existing"
ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-c "drop database if exists tombstone; create database tombstone"
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
tombstone_msg=$(date +'%Y%m%d_%H%M%S')"_${ENVIRONMENT}"
echo "Insert '$tombstone_msg' into tombstone"
ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql tombstone -c "insert into tombstone(note) values('${tombstone_msg}') returning *"
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
for host in "${AZURE_HOSTS[@]}" "${GCP_HOSTS[@]}"; do
echo "Stopping chef on $host"
ssh_remote "$host" sudo service chef-client stop
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
for host in "${AZURE_PGBOUNCERS[@]}"
do
echo "Stopping consul on $host"
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
for host in "${GCP_HOSTS[@]}"
do
echo "Stopping repmgrd on $host"
......
......@@ -2,5 +2,7 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql -d gitlab_repmgr -c \
"TRUNCATE repmgr_gitlab_cluster.repl_nodes"
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
echo "standby_mode = 'on'
primary_conninfo = 'user=gitlab_repmgr host=''$GCP_MASTER_CANDIDATE'' password=$GITLAB_REPMGR_PASSWORD port=5432 fallback_application_name=repmgr sslmode=prefer sslcompression=1 application_name=''$AZURE_MASTER'''
primary_slot_name = secondary_azureprd
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
if ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select pg_is_in_recovery()" | grep -q 't'
then
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
azure_master_lsn="$(ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select case when pg_is_in_recovery()
then pg_last_xlog_replay_location()
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
# WARNING WARNING WARNING here switchover happens!
ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql /opt/gitlab/embedded/bin/pg_ctl \
promote -D /var/lib/opt/gitlab/postgresql/data
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
if ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select pg_is_in_recovery()" | grep -q 'f'
then
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
echo "Register $GCP_MASTER_CANDIDATE as master with repmgr"
ssh_remote "$GCP_MASTER_CANDIDATE" sudo gitlab-ctl repmgr register master
for host in "${GCP_HOSTS[@]}"
......
......@@ -2,6 +2,8 @@
set -eu
source "${BASE}/env_${ENVIRONMENT}"
echo "Checking state of $GCP_MASTER_CANDIDATE"
if ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-consul gitlab-ctl repmgr-check-master 2> /dev/null
then
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment