Commit eb56736d authored by Matteo Melli's avatar Matteo Melli
Browse files

Adapted migration script to new scripts structure

Enable migration to use script files and script functions.
Created scripts to execute steps inside scripts folder tree.
Merged production and staging steps and added support to specify a role
(associated to steps_$role).
Sourced source_vars into migration script.
parent 4d374cdf
Pipeline #88514 failed with stage
in 47 seconds
#!/bin/bash
set -eu
if ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select pg_is_in_recovery()" | grep -q 't'
then
echo "$AZURE_MASTER is standby"
return 0
else
>&2 echo "$AZURE_MASTER is standby"
return 1
fi
#!/bin/bash
set -eu
azure_master_lsn="$(ssh_remote "$AZURE_MASTER" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select case when pg_is_in_recovery()
then pg_last_xlog_replay_location()
else pg_current_xlog_location() end;")"
gcp_master_candidate_lsn="$(ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select case when pg_is_in_recovery()
then pg_last_xlog_replay_location()
else pg_current_xlog_location() end;")"
if [ "$azure_master_lsn" == "$gcp_master_candidate_lsn" ]
then
echo "GCP and Azure have same LSN: $azure_master_lsn"
return 0
fi
echo "GCP and Azure have NOT same LSN. Current LSNs are: Azure/$azure_master_lsn GCP/$gcp_master_candidate_lsn"
reutrn 1
#!/bin/bash
set -eu
# WARNING WARNING WARNING here switchover happens!
ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql /opt/gitlab/embedded/bin/pg_ctl \
promote -D /var/lib/opt/gitlab/postgresql/data
#!/bin/bash
set -eu
if ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-psql gitlab-psql postgres \
-t -A -c "select pg_is_in_recovery()" | grep -q 'f'
then
echo "$GCP_MASTER_CANDIDATE is master"
return 0
else
>&2 echo "$GCP_MASTER_CANDIDATE is master"
return 1
fi
#!/bin/bash
set -eu
echo "Register $GCP_MASTER_CANDIDATE as master with repmgr"
ssh_remote "$GCP_MASTER_CANDIDATE" sudo gitlab-ctl repmgr register master
for host in "${GCP_HOSTS[@]}"
do
if [ "$GCP_MASTER_CANDIDATE" == "$host" ]
then
continue;
fi
echo "Register $host as standby with repmgr"
ssh_remote "$host" sudo gitlab-ctl repmgr register standby
done
echo "Starting repmgrd on $GCP_MASTER_CANDIDATE"
ssh_remote "$GCP_MASTER_CANDIDATE" sudo sv start /opt/gitlab/sv/repmgrd
for host in "${GCP_HOSTS[@]}"
do
if [ "$GCP_MASTER_CANDIDATE" == "$host" ]
then
continue;
fi
echo "Starting repmgrd on $host"
ssh_remote "$host" sudo sv start /opt/gitlab/sv/repmgrd
done
#!/bin/bash
set -eu
echo "Checking state of $GCP_MASTER_CANDIDATE"
if ssh_remote "$GCP_MASTER_CANDIDATE" sudo -u gitlab-consul gitlab-ctl repmgr-check-master 2> /dev/null
then
echo "$GCP_MASTER_CANDIDATE is repmgr master"
return 0
else
>&2 echo "$GCP_MASTER_CANDIDATE is not repmgr master"
return 1
fi
#!/bin/bash
set -eu
# chef
for host in "${AZURE_HOSTS[@]}" "${GCP_HOSTS[@]}"; do
echo "Starting chef-client on $host"
ssh_remote "$host" sudo mv /etc/chef.migration /etc/chef
ssh_remote "$host" sudo service chef-client start
done
#!/bin/bash
set -eu
for host in "${GCP_PGBOUNCERS[@]}"
do
echo "Check pgbouncer on $host"
ssh_remote "$host" gitlab-ctl pgb-console -c "SHOW DATABASES"
ssh_remote "$host" gitlab-ctl pgb-console -c "SHOW SERVERS"
read -s -N 1 -p "Press [y] to continue, any other key to abort." key
if [ "$key" != "y" ]
then
return 1
fi
done
......@@ -15,6 +15,7 @@ export ZOOM_LINK="https://gitlab.zoom.us/j/859814316" # Find this in the calenda
export GITLAB_INSTANCE="https://dev.gitlab.org/" # Issue tracker in which to create the failover issues
export GITLAB_MIGRATION_PROJECT_PATH="gitlab-com/migration" # Migration Project
export GITLAB_TOKEN="__REQUIRED__" # Available at https://gitlab.1password.com/vaults/ljcrnm55wwhnh5ynr7edbxgowm/allitems/dx2qetzqbbcsrlrj3vtunexagy
export GITLAB_REPMGR_PASSWORD="__REQUIRED__" # Available at ???
export TEAM_COORDINATOR="@nick" # Coordinator (backup: @digitalmoksha)
export TEAM_CHEF_RUNNER="@ahmadsherif" # Chef Runner (backup: @alejandro)
......
export steps=(
000_get-rid-of-could-not-change-directory-to-message
001_create-tombstone-table
002_check-gcp-replication-delay
003_disable-chef
004_disable-consul
005_disable-automatic-failover
006_reset-automatic-failover-state
007_convert-azure-master-to-standby
008_check-azure-master-is-standby
009_check-gcp-nodes-has-same-azure-lsn
010_perform-gcp-candidate-master-promote
011_check-gcp-candidate-master-is-master
012_enable-automatic-failover-on-gcp-only
013_check-repmgr-master
014_check-pgbouncer-node-in-gcp
015_enable-chef-on-gcp-only
016_enable-consul-on-gcp-only
017_restore-could-not-change-directory-to-message
function:000_get-rid-of-could-not-change-directory-to-message
file:001_scripts/01_preflight/050_configuration_checks/110-check-repmgr-state.sh
file:002_scripts/02_failover/060_go/p03/050-create-tombstone-table.sh
file:003_scripts/02_failover/060_go/p03/051-check-gcp-replication-delay.sh
file:004_scripts/02_failover/060_go/p04/040-disable-chef.sh
file:005_scripts/02_failover/060_go/p04/041-disable-consul.sh
file:006_scripts/02_failover/060_go/p04/042-disable-automatic-failover.sh
file:007_scripts/02_failover/060_go/p04/043-reset-automatic-failover-state.sh
file:008_scripts/02_failover/060_go/p04/050-convert-azure-master-to-standby.sh
file:009_scripts/02_failover/060_go/p04/051-check-azure-master-is-standby.sh
file:010_scripts/02_failover/060_go/p04/052-check-gcp-nodes-has-same-azure-lsn.sh
file:011_scripts/02_failover/060_go/p04/060-perform-gcp-candidate-master-promote.sh
file:012_scripts/02_failover/060_go/p04/061-check-gcp-candidate-master-is-master.sh
file:013_scripts/02_failover/060_go/p04/070-enable-automatic-failover-on-gcp-only.sh
file:014_scripts/02_failover/060_go/p04/071-check-repmgr-master.sh
file:015_scripts/02_failover/060_go/p04/072-enable-consul-on-gcp-only.sh
file:016_scripts/02_failover/060_go/p04/073-check-pgbouncer-node-in-gcp.sh
function:017_restore-could-not-change-directory-to-message
)
function get-rid-of-could-not-change-directory-to-message(){
......@@ -118,7 +118,7 @@ primary_conninfo = 'user=gitlab_repmgr host=''$GCP_MASTER_CANDIDATE'' password=$
primary_slot_name = secondary_azureprd
restore_command = '/usr/bin/envdir /etc/wal-e.d/env /opt/wal-e/bin/wal-e wal-fetch -p 32 "%f" "%p"'
recovery_target_timeline = 'latest'" | \
ssh_remote "$AZURE_MASTER" sudo tee /var/lib/opt/gitlab/postgresql/data/recovery.conf
ssh_remote "$AZURE_MASTER" sudo tee /var/lib/opt/gitlab/postgresql/data/recovery.conf > /dev/null
ssh_remote "$AZURE_MASTER" sudo chown postgres:postgres /var/lib/opt/gitlab/postgresql/data/recovery.conf
ssh_remote "$AZURE_MASTER" sudo chmod 600 /var/lib/opt/gitlab/postgresql/data/recovery.conf
ssh_remote "$AZURE_MASTER" sudo sv -W 1 stop /opt/gitlab/sv/postgres \
......
......@@ -11,8 +11,12 @@ function do_step(){
}
function step_type(){
echo "$1"|cut -d _ -f 1|cut -d : -f 1
}
function step_3digit_number(){
echo "$1"|cut -d _ -f 1
echo "$1"|cut -d _ -f 1|cut -d : -f 2
}
function step_number(){
......@@ -36,6 +40,17 @@ function has_step(){
$found
}
function step_check(){
script_type="$(step_type "$1")"
script="$(step_script "$1")"
if [ "$script_type" == "function" ]
then
type "$script" > /dev/null 2>&1
else
[ -x "$script" ]
fi
}
function do_menu(){
next_step="$(step_3digit_number "${steps[0]}")"
while true
......@@ -98,10 +113,16 @@ function do_menu(){
esac
if [ ! -z "$step" ]
then
script_type="$(step_type "$step")"
script="$(step_script "$step")"
echo "Step $step will be executed:"
echo "Step $step will be executed ($script_type $script):"
echo
type "$script" | head -n -1 | tail -n +4
if [ "$script_type" == "function" ]
then
type "$script" | head -n -1 | tail -n +4
else
cat "$script"
fi
echo
read -s -N 1 -p "Press [y] to continue, any other key to abort." key
if [ "$key" == "y" ]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment