Commit 53aaf065 authored by Nikolay's avatar Nikolay
Browse files

add tombstone table on azure master; no camel style in bash

parent a50d1e90
......@@ -321,7 +321,7 @@ state of the secondary to converge.
* Production: `postgres-01-db-gprd.c.gitlab-production.internal`
```shell
shopt -s expand_aliases
alias ssh_remote="ssh -F ~/.ssh/gitlab.conf" # TODO(NikolayS) change to "knife ..."
alias ssh_remote="ssh " # TODO(NikolayS) change to "knife ..."
# TODO(NikolayS) move globals to the top / init stage
export GITLAB_ENV="staging"
......@@ -347,22 +347,32 @@ state of the secondary to converge.
# Returns:
# None
#######################################
checkGcpReplicationDelay() { # put definition to the top
maxRepDelay=10 # TODO(NikolayS) is 10s ok? Double-check after turning SR on
check_gcp_replication_delay() { # put definition to the top
max_rep_delay=10 # TODO(NikolayS) is 10s ok? Double-check after turning SR on
# TODO(NikolayS) write to the tombstone table right before checking
gcpCurRepDelay=$(ssh_remote "$GCP_MASTER_CANDIDATE" \
ssh_remote "$AZURE_MASTER" $(cat << EOF
cd /tmp;
sudo -u gitlab-psql gitlab-psql postgres \
-c "drop database if exists tombstone" \
-c "create database tombstone";
sudo -u gitlab-psql gitlab-psql tombstone \
-c "create table tombstone (created_at timestamptz default now() primary key, note text)" \
-c "insert into tombstone(note) values('pre-switchover check') returning *"
EOF
)
gcp_cur_rep_delay=$(ssh_remote "$GCP_MASTER_CANDIDATE" \
"sudo gitlab-psql -Atd postgres -c 'select round(extract(epoch from (now() - pg_last_xact_replay_timestamp())))'")
if [[ ! -z ${gcpCurRepDelay+x} && "$maxRepDelay" -gt "$gcpCurRepDelay" ]]; then # TODO(NikolayS) do it in morchestra?
echo "Check if GCP delay <${maxRepDelay} s): OK (delay: ~${gcpCurRepDelay}s)"
if [[ ! -z ${gcp_cur_rep_delay+x} && "$max_rep_delay" -gt "$gcp_cur_rep_delay" ]]; then # TODO(NikolayS) do it in morchestra?
echo "Check if GCP delay < ${max_rep_delay}s): OK (delay: ~${gcp_cur_rep_delay}s)"
else
echo "Check if GCP delay <${maxRepDelay} s): FAIL (delay: ~${gcpCurRepDelay}s)"
echo "Check if GCP delay < ${max_rep_delay}s): FAIL (delay: ~${gcp_cur_rep_delay}s)"
fi
}
checkGcpReplicationDelay
check_gcp_replication_delay
```
1. [ ] 🐺 {+ Coordinator +}: Now disable all sidekiq-cron jobs on the secondary
* In a dedicated rails console on the **secondary**:
......@@ -413,10 +423,10 @@ of errors while it is being promoted.
- [ ] `altssh.gitlab.com A 35.190.168.187`
- [ ] `*.gitlab.io A 35.185.44.232`
- **DO NOT** change `gitlab.io`.
1. [ ] 🐘 {+ Database-Wrangler +}: Update the priority of GCP nodes in the repmgr database. Run the following on the current primary:
1. [ ] 🐘 {+ Database-Wrangler +}: Shut down repmgr
```shell
# gitlab-psql -d gitlab_repmgr -c "update repmgr_gitlab_cluster.repl_nodes set priority=100 where name like '%gstg%'"
```
1. [ ] 🐘 {+ Database-Wrangler +}: **Gracefully** turn off the **Azure** postgresql standby instances.
* Keep everything, just ensure it’s turned off
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment