Added scripts, still WIP

bin/database-wrangler/azure/4_1_5_forbid_writes_azure

+#!/bin/bash
+
+for i in $(seq 1 $N_OF_HOSTS); do
+  host="${AZURE_HOST_PREFIX}$i${AZURE_HOST_SUFFIX}"
+  ssh_remote "$host" "sudo -u gitlab-psql sh -c \"echo 'local all all peer map=gitlab' > /var/opt/gitlab/postgresql/data/pg_hba.conf.tmp\""
+  ssh_remote "$host" "sudo -u gitlab-psql sh -c \"echo 'host replication gitlab_repmgr 0.0.0.0/0 md5' >> /var/opt/gitlab/postgresql/data/pg_hba.conf.tmp\""
+  ssh_remote "$host" "sudo -u gitlab-psql cp /var/opt/gitlab/postgresql/data/pg_hba.conf /var/opt/gitlab/postgresql/data/pg_hba.conf.backup"
+  ssh_remote "$host" "sudo -u gitlab-psql mv /var/opt/gitlab/postgresql/data/pg_hba.conf.tmp /var/opt/gitlab/postgresql/data/pg_hba.conf"
+  ssh_remote "$host" "cd /tmp; sudo -u gitlab-psql gitlab-psql -c 'select pg_reload_conf()'"
+  ssh_remote "$host" "cd /tmp; sudo -u gitlab-psql gitlab-psql -c 'select pg_terminate_backend(pid) from pg_stat_activity where datname = 'gitlabhq_production''"
+done
+
+

bin/database-wrangler/encv_staging

+# Staging
+N_OF_HOSTS=2
+AZURE_HOST_PREFIX="postgres0"
+AZURE_HOST_SUFFIX=".db.stg.gitlab.com"
+GCP_HOST_PREFIX="postgres-0"
+GCP_HOST_SUFFIX="-db-gstg.c.gitlab-staging-1.internal"
+AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
+GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}2${GCP_HOST_SUFFIX}"
+

bin/database-wrangler/env_production

+# Production
+
+N_OF_HOSTS=4
+AZURE_HOST_PREFIX="postgres-0"
+AZURE_HOST_SUFFIX=".db.prd.gitlab.com"
+GCP_HOST_PREFIX="postgres-0"
+GCP_HOST_SUFFIX="-db-gprd.c.gitlab-production.internal"
+AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
+GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}1${GCP_HOST_SUFFIX}"
+

bin/database-wrangler/gcp/03_05_check_repl_in_GCP

+#!/bin/bash
+
+

bin/database-wrangler/global/3_5_tombstone

+shopt -s expand_aliases
+alias ssh_remote="ssh " # TODO(NikolayS) change to "knife ..."
+
+# TODO(NikolayS) move globals to the top / init stage
+export GITLAB_ENV="staging"
+
+if [[ "$GITLAB_ENV" == "production" ]]; then
+  export N_OF_HOSTS=4
+  export AZURE_HOST_PREFIX="postgres-0"
+  export AZURE_HOST_SUFFIX=".db.prd.gitlab.com"
+  export GCP_HOST_PREFIX="postgres-0"
+  export GCP_HOST_SUFFIX="-db-gprd.c.gitlab-production.internal"
+  export AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
+  export GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}1${GCP_HOST_SUFFIX}"
+elif [[ "$GITLAB_ENV" == "staging" ]]; then
+  export N_OF_HOSTS=2
+  export AZURE_HOST_PREFIX="postgres0"
+  export AZURE_HOST_SUFFIX=".db.stg.gitlab.com"
+  export GCP_HOST_PREFIX="postgres-0"
+  export GCP_HOST_SUFFIX="-db-gstg.c.gitlab-staging-1.internal"
+  export AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
+  export GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}2${GCP_HOST_SUFFIX}"
+else
+  >&2 echo "\$GITLAB_ENV is not correctly defined ($GITLAB_ENV). Stop all work and exit (enter anything to proceed)."
+  read smth
+  exit 1
+fi
+
+#######################################
+# Check that GCP "main" replica is not lagging too much
+# Globals:
+#   AZURE_MASTER
+#   GCP_MASTER_CANDIDATE
+# Arguments:
+#   None
+# Returns:
+#   None
+#######################################
+handle_gcp_replication_delay() { # put definition to the top
+  max_rep_delay=10 # TODO(NikolayS) is 10s ok? Double-check after turning SR on
+
+  # "create database if not exists" is not supported in Postgres,
+  # so to make the following action idempotent and not depending on the pre-actions,
+  # we better re-create tombstone DB and table from scratch
+  ssh_remote "$AZURE_MASTER" $(cat << EOF
+    cd /tmp;
+    sudo -u gitlab-psql gitlab-psql postgres \
+      -c "drop database if exists tombstone";
+    sudo -u gitlab-psql gitlab-psql postgres \
+      -c "create database tombstone";
+    sudo -u gitlab-psql gitlab-psql tombstone \
+      -c "create table if not exists tombstone (created_at timestamptz default now() primary key, note text)"
+EOF
+)
+  tombstone_msg=$(date +'%Y%m%d_%H%M%S')"_${GITLAB_ENV}"
+  ssh_remote "$AZURE_MASTER" \
+    "cd /tmp; sudo -u gitlab-psql gitlab-psql tombstone -c \"insert into tombstone(note) values('${tombstone_msg}') returning *\""
+
+  # wait until the change is propagated
+  while [[ true ]]; do
+    find_new_msg=$(
+      ssh_remote "$GCP_MASTER_CANDIDATE" \
+        "cd /tmp; sudo gitlab-psql -Atd tombstone -c \"select created_at from tombstone where note = '$tombstone_msg'\""
+    )
+    if [[ -z ${find_new_msg+x} ]] || [[ "$find_new_msg" == "" ]]; then
+      gcp_cur_rep_delay=$(
+        ssh_remote "$GCP_MASTER_CANDIDATE" \
+          "cd /tmp; sudo gitlab-psql -Atd postgres -c 'select round(extract(epoch from (now() - pg_last_xact_replay_timestamp())))'"
+      )
+      echo "New tombstone message is not seen on $GCP_MASTER_CANDIDATE. The replication delay: ${gcp_cur_rep_delay}s. Wait 3 seconds..."
+      sleep 3
+    else
+      echo "New tombstone message arrived to $GCP_MASTER_CANDIDATE, continue."
+      break
+    fi
+  done
+}
+
+handle_gcp_replication_delay
+

bin/database-wrangler/global/4_1_4_stop_repmgr_consul

+#######################################
+# Stop or start chef, consul, and repmgr
+# Globals:
+#   GITLAB_ENV, AZURE_MASTER, AZURE_HOST_PREFIX,
+#   GCP_HOST_PREFIX, GCP_HOST_SUFFIX, N_OF_HOSTS
+# Arguments:
+#   None
+# Returns:
+#   None
+#######################################
+db_switch_chef_consul_repmgr_state() {
+  if [[ -z "${GITLAB_ENV+x}" ]]; then
+    >&2 echo "\$GITLAB_ENV is empty, cannot proceed."
+    return
+  fi
+  echo "We're about to stop disable chef and stop repmgr on $GITLAB_ENV environment. To proceed, type '$GITLAB_ENV':"
+  read proceed_cmd
+  if [[ "$proceed_cmd" != "$GITLAB_ENV" ]]; then
+    >&2 echo "Stop."
+    return
+  fi
+
+  # chef
+  for i in $(seq 1 $N_OF_HOSTS); do
+    for host in "${AZURE_HOST_PREFIX}$i${AZURE_HOST_SUFFIX}" "${GCP_HOST_PREFIX}$i${GCP_HOST_SUFFIX}"; do
+      echo "${1}ing chef on $host"
+      # WARNING: the following line modifies the state
+      if [[ "$1" == "stop" ]]; then
+        ssh_remote "$host" "sudo service chef-client stop"
+        ssh_remote "$host" "sudo mv /etc/chef /etc/chef.migration"
+      elif [[ "$1" == "start" ]]; then
+        ssh_remote "$host" "sudo mv /etc/chef.migration /etc/chef"
+        ssh_remote "$host" "sudo service chef-client start"
+      else
+        >&2 echo "1st argument must be either \"stop\" or \"start\" (provided: $1)."
+        return
+      fi
+    done
+  done
+
+  # consul
+  # TODO(NikolayS) commands to stop consul see https://gitlab.com/gitlab-com/migration/issues/718#note_90342057
+
+  # repmgr
+  # WARNING: the following line modifies the state
+  ssh_remote "$AZURE_MASTER" "sudo sv $1 /opt/gitlab/sv/repmgrd"
+}
+
+db_switch_chef_consul_repmgr_state stop
+