Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
What's new
4
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
gitlab-com
migration
Commits
72b45095
Commit
72b45095
authored
Aug 02, 2018
by
Emanuel Calvo
Committed by
Matteo Melli
Aug 03, 2018
Browse files
Added scripts, still WIP
parent
bb3ac928
Changes
8
Hide whitespace changes
Inline
Side-by-side
bin/database-wrangler/.env_test
0 → 100644
View file @
72b45095
bin/database-wrangler/azure/4_1_5_forbid_writes_azure
0 → 100644
View file @
72b45095
#!/bin/bash
for
i
in
$(
seq
1
$N_OF_HOSTS
)
;
do
host
=
"
${
AZURE_HOST_PREFIX
}
$i
${
AZURE_HOST_SUFFIX
}
"
ssh_remote
"
$host
"
"sudo -u gitlab-psql sh -c
\"
echo 'local all all peer map=gitlab' > /var/opt/gitlab/postgresql/data/pg_hba.conf.tmp
\"
"
ssh_remote
"
$host
"
"sudo -u gitlab-psql sh -c
\"
echo 'host replication gitlab_repmgr 0.0.0.0/0 md5' >> /var/opt/gitlab/postgresql/data/pg_hba.conf.tmp
\"
"
ssh_remote
"
$host
"
"sudo -u gitlab-psql cp /var/opt/gitlab/postgresql/data/pg_hba.conf /var/opt/gitlab/postgresql/data/pg_hba.conf.backup"
ssh_remote
"
$host
"
"sudo -u gitlab-psql mv /var/opt/gitlab/postgresql/data/pg_hba.conf.tmp /var/opt/gitlab/postgresql/data/pg_hba.conf"
ssh_remote
"
$host
"
"cd /tmp; sudo -u gitlab-psql gitlab-psql -c 'select pg_reload_conf()'"
ssh_remote
"
$host
"
"cd /tmp; sudo -u gitlab-psql gitlab-psql -c 'select pg_terminate_backend(pid) from pg_stat_activity where datname = 'gitlabhq_production''"
done
bin/database-wrangler/encv_staging
0 → 100644
View file @
72b45095
# Staging
N_OF_HOSTS=2
AZURE_HOST_PREFIX="postgres0"
AZURE_HOST_SUFFIX=".db.stg.gitlab.com"
GCP_HOST_PREFIX="postgres-0"
GCP_HOST_SUFFIX="-db-gstg.c.gitlab-staging-1.internal"
AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}2${GCP_HOST_SUFFIX}"
bin/database-wrangler/env_production
0 → 100644
View file @
72b45095
# Production
N_OF_HOSTS=4
AZURE_HOST_PREFIX="postgres-0"
AZURE_HOST_SUFFIX=".db.prd.gitlab.com"
GCP_HOST_PREFIX="postgres-0"
GCP_HOST_SUFFIX="-db-gprd.c.gitlab-production.internal"
AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}1${GCP_HOST_SUFFIX}"
bin/database-wrangler/gcp/03_05_check_repl_in_GCP
0 → 100644
View file @
72b45095
#!/bin/bash
bin/database-wrangler/global/.3_5_tombstone.swp
0 → 100644
View file @
72b45095
File added
bin/database-wrangler/global/3_5_tombstone
0 → 100644
View file @
72b45095
shopt -s expand_aliases
alias ssh_remote="ssh " # TODO(NikolayS) change to "knife ..."
# TODO(NikolayS) move globals to the top / init stage
export GITLAB_ENV="staging"
if [[ "$GITLAB_ENV" == "production" ]]; then
export N_OF_HOSTS=4
export AZURE_HOST_PREFIX="postgres-0"
export AZURE_HOST_SUFFIX=".db.prd.gitlab.com"
export GCP_HOST_PREFIX="postgres-0"
export GCP_HOST_SUFFIX="-db-gprd.c.gitlab-production.internal"
export AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
export GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}1${GCP_HOST_SUFFIX}"
elif [[ "$GITLAB_ENV" == "staging" ]]; then
export N_OF_HOSTS=2
export AZURE_HOST_PREFIX="postgres0"
export AZURE_HOST_SUFFIX=".db.stg.gitlab.com"
export GCP_HOST_PREFIX="postgres-0"
export GCP_HOST_SUFFIX="-db-gstg.c.gitlab-staging-1.internal"
export AZURE_MASTER="${AZURE_HOST_PREFIX}2${AZURE_HOST_SUFFIX}"
export GCP_MASTER_CANDIDATE="${GCP_HOST_PREFIX}2${GCP_HOST_SUFFIX}"
else
>&2 echo "\$GITLAB_ENV is not correctly defined ($GITLAB_ENV). Stop all work and exit (enter anything to proceed)."
read smth
exit 1
fi
#######################################
# Check that GCP "main" replica is not lagging too much
# Globals:
# AZURE_MASTER
# GCP_MASTER_CANDIDATE
# Arguments:
# None
# Returns:
# None
#######################################
handle_gcp_replication_delay() { # put definition to the top
max_rep_delay=10 # TODO(NikolayS) is 10s ok? Double-check after turning SR on
# "create database if not exists" is not supported in Postgres,
# so to make the following action idempotent and not depending on the pre-actions,
# we better re-create tombstone DB and table from scratch
ssh_remote "$AZURE_MASTER" $(cat << EOF
cd /tmp;
sudo -u gitlab-psql gitlab-psql postgres \
-c "drop database if exists tombstone";
sudo -u gitlab-psql gitlab-psql postgres \
-c "create database tombstone";
sudo -u gitlab-psql gitlab-psql tombstone \
-c "create table if not exists tombstone (created_at timestamptz default now() primary key, note text)"
EOF
)
tombstone_msg=$(date +'%Y%m%d_%H%M%S')"_${GITLAB_ENV}"
ssh_remote "$AZURE_MASTER" \
"cd /tmp; sudo -u gitlab-psql gitlab-psql tombstone -c \"insert into tombstone(note) values('${tombstone_msg}') returning *\""
# wait until the change is propagated
while [[ true ]]; do
find_new_msg=$(
ssh_remote "$GCP_MASTER_CANDIDATE" \
"cd /tmp; sudo gitlab-psql -Atd tombstone -c \"select created_at from tombstone where note = '$tombstone_msg'\""
)
if [[ -z ${find_new_msg+x} ]] || [[ "$find_new_msg" == "" ]]; then
gcp_cur_rep_delay=$(
ssh_remote "$GCP_MASTER_CANDIDATE" \
"cd /tmp; sudo gitlab-psql -Atd postgres -c 'select round(extract(epoch from (now() - pg_last_xact_replay_timestamp())))'"
)
echo "New tombstone message is not seen on $GCP_MASTER_CANDIDATE. The replication delay: ${gcp_cur_rep_delay}s. Wait 3 seconds..."
sleep 3
else
echo "New tombstone message arrived to $GCP_MASTER_CANDIDATE, continue."
break
fi
done
}
handle_gcp_replication_delay
bin/database-wrangler/global/4_1_4_stop_repmgr_consul
0 → 100644
View file @
72b45095
#######################################
# Stop or start chef, consul, and repmgr
# Globals:
# GITLAB_ENV, AZURE_MASTER, AZURE_HOST_PREFIX,
# GCP_HOST_PREFIX, GCP_HOST_SUFFIX, N_OF_HOSTS
# Arguments:
# None
# Returns:
# None
#######################################
db_switch_chef_consul_repmgr_state() {
if [[ -z "${GITLAB_ENV+x}" ]]; then
>&2 echo "\$GITLAB_ENV is empty, cannot proceed."
return
fi
echo "We're about to stop disable chef and stop repmgr on $GITLAB_ENV environment. To proceed, type '$GITLAB_ENV':"
read proceed_cmd
if [[ "$proceed_cmd" != "$GITLAB_ENV" ]]; then
>&2 echo "Stop."
return
fi
# chef
for i in $(seq 1 $N_OF_HOSTS); do
for host in "${AZURE_HOST_PREFIX}$i${AZURE_HOST_SUFFIX}" "${GCP_HOST_PREFIX}$i${GCP_HOST_SUFFIX}"; do
echo "${1}ing chef on $host"
# WARNING: the following line modifies the state
if [[ "$1" == "stop" ]]; then
ssh_remote "$host" "sudo service chef-client stop"
ssh_remote "$host" "sudo mv /etc/chef /etc/chef.migration"
elif [[ "$1" == "start" ]]; then
ssh_remote "$host" "sudo mv /etc/chef.migration /etc/chef"
ssh_remote "$host" "sudo service chef-client start"
else
>&2 echo "1st argument must be either \"stop\" or \"start\" (provided: $1)."
return
fi
done
done
# consul
# TODO(NikolayS) commands to stop consul see https://gitlab.com/gitlab-com/migration/issues/718#note_90342057
# repmgr
# WARNING: the following line modifies the state
ssh_remote "$AZURE_MASTER" "sudo sv $1 /opt/gitlab/sv/repmgrd"
}
db_switch_chef_consul_repmgr_state stop
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment