Commit 687940de authored by Nikolay's avatar Nikolay
Browse files

Merge branch 'master' of gitlab.com:gitlab-com/migration into db_steps_automation

parents b415be33 d3fe20fc
source_vars
image: alpine:latest
stages:
- test
# Ensure scripts are well-written
shellcheck:
stage: test
before_script:
- wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz -O - | xzcat | tar -xv
script:
- find ./bin/azure ./bin/gcp -name '*.sh' | xargs ./shellcheck-stable/shellcheck -x
- ./shellcheck-stable/shellcheck -x ./bin/check-script-references ./bin/workflow-script-commons.sh ./bin/source_vars_template.sh ./bin/start-failover-procedure.sh
references:
stage: test
before_script:
- apk add --no-cache bash
script:
- bash -x ./bin/check-script-references
# Failover Team
| Role | Assigned To |
| -----------------------------------------------------------------------|-------------|
| 🐺 Coordinator | |
| 🔪 Chef-Runner | |
| ☎ Comms-Handler | |
| 🐘 Database-Wrangler | |
| ☁ Cloud-conductor | |
| 🏆 Quality | |
| ↩ Fail-back Handler (_Staging Only_) | |
| 🎩 Head Honcho (_Production Only_) | |
| -----------------------------------------------------------------------|----------------------------|
| 🐺 Coordinator | __TEAM_COORDINATOR__ |
| 🔪 Chef-Runner | __TEAM_CHEF_RUNNER__ |
| ☎ Comms-Handler | __TEAM_COMMS_HANDLER__ |
| 🐘 Database-Wrangler | __TEAM_DATABASE_WRANGLER__ |
| ☁ Cloud-conductor | __TEAM_CLOUD_CONDUCTOR__ |
| 🏆 Quality | __TEAM_QUALITY__ |
| ↩ Fail-back Handler (_Staging Only_) | __TEAM_FAILBACK_HANDLER__ |
| 🎩 Head Honcho (_Production Only_) | __TEAM_HEAD_HONCHO__ |
(try to ensure that 🔪, ☁ and ↩ are always the same person for any given run)
......@@ -20,10 +20,10 @@ Perform these steps when the issue is created.
- [ ] 🐺 {+ Coordinator +}: Fill out the names of the failover team in the table above.
- [ ] 🐺 {+ Coordinator +}: Fill out dates/times and links in this issue:
- `START_TIME` & `END_TIME`
- `GOOGLE_DOC_LINK` (for PRODUCTION, create a new doc and make it writable for GitLabbers, and readable for the world)
- **PRODUCTION ONLY** `LINK_TO_BLOG_POST`
- **PRODUCTION ONLY** `END_TIME`
- Start Time: `__MAINTENANCE_START_TIME__` & End Time: `__MAINTENANCE_END_TIME__`
- Google Working Doc: __GOOGLE_DOC_URL__ (for PRODUCTION, create a new doc and make it writable for GitLabbers, and readable for the world)
- **PRODUCTION ONLY** Blog Post: __BLOG_POST_URL__
- **PRODUCTION ONLY** End Time: __MAINTENANCE_END_TIME__
# Support Options
......@@ -107,18 +107,17 @@ These dashboards might be useful during the failover:
- Details of specific situations with very-long running CI jobs which may loose their artifacts and logs if they don't complete before the maintenance window
1. [ ] ☎ {+ Comms-Handler +}: Ensure that YouTube stream will be available for Zoom call
1. [ ] ☎ {+ Comms-Handler +}: Tweet blog post from `@gitlab` and `@gitlabstatus`
- `Reminder: GitLab.com will be undergoing 2 hours maintenance on Saturday XX June 2018, from START_TIME - END_TIME UTC. Follow @gitlabstatus for more details. LINK_TO_BLOG_POST`
- `Reminder: GitLab.com will be undergoing 2 hours maintenance on Saturday XX June 2018, from __MAINTENANCE_START_TIME__ - __MAINTENANCE_END_TIME__ UTC. Follow @gitlabstatus for more details. __BLOG_POST_URL__`
1. [ ] 🔪 {+ Chef-Runner +}: Ensure the GCP environment is inaccessible to the outside world
# T minus 1 day (Date TBD)
1. [ ] 🐺 {+ Coordinator +}: Perform (or coordinate) Preflight Checklist
1. [ ] **PRODUCTION ONLY** ☎ {+ Comms-Handler +}: Tweet from `@gitlab`
- `Reminder: GitLab.com will be undergoing 2 hours maintenance tomorrow, from START_TIME - END_TIME UTC. Follow @gitlabstatus for more details. LINK_TO_BLOG_POST`
1. [ ] **PRODUCTION ONLY** ☎ {+ Comms-Handler +}: Tweet from `@gitlab`.
- Tweet content from `./bin/azure/02_failover/t-1d/010_gitlab_twitter_announcement.sh`
1. [ ] **PRODUCTION ONLY** ☎ {+ Comms-Handler +}: Retweet `@gitlab` tweet from `@gitlabstatus` with further details
- `Reminder: GitLab.com will be undergoing 2 hours maintenance tomorrow. We'll be live on YouTube. Working doc: GOOGLE_DOC_LINK, Blog: LINK_TO_BLOG_POST`
- Tweet content from `./bin/azure/02_failover/t-1d/020_gitlabstatus_twitter_announcement.sh`
# T minus 3 hours (Date TBD)
......@@ -139,10 +138,9 @@ much as possible, we'll stop any new runner jobs from being picked up, starting
an hour before the scheduled maintenance window.
1. [ ] **PRODUCTION ONLY** ☎ {+ Comms-Handler +}: Tweet from `@gitlabstatus`
- `As part of upcoming GitLab.com maintenance work, CI runners will not be accepting new jobs until END_TIME UTC. GitLab.com will undergo maintenance in 1 hour. Working doc: GOOGLE_DOC_LINK`
- `As part of upcoming GitLab.com maintenance work, CI runners will not be accepting new jobs until __MAINTENANCE_END_TIME__ UTC. GitLab.com will undergo maintenance in 1 hour. Working doc: __GOOGLE_DOC_URL__`
1. [ ] ☎ {+ Comms-Handler +}: Post to #announcements on Slack:
* Staging: `We're rehearsing the failover of GitLab.com in *1 hour* by migrating staging.gitlab.com to GCP. Come watch us at ZOOM_LINK! Notes in GOOGLE_DOC_LINK!`
* Production: `GitLab.com is being migrated to GCP in *1 hour*. There is a 2-hour downtime window. We'll be live on YouTube. Notes in GOOGLE_DOC_LINK!`
- `./bin/azure/02_failover/t-1h/020_slack_announcement.sh`
1. [ ] **PRODUCTION ONLY** ☁ {+ Cloud-conductor +}: Create a maintenance window in PagerDuty for [GitLab Production service](https://gitlab.pagerduty.com/services/PATDFCE) for 2 hours starting in an hour from now.
1. [ ] **PRODUCTION ONLY** ☁ {+ Cloud-conductor +}: [Create an alert silence](https://alerts.gitlab.com/#/silences/new) for 2 hours starting in an hour from now with the following matcher(s):
- `environment`: `prd`
......
......@@ -34,7 +34,7 @@
- Staging
- GCP `gstg`: https://dev.gitlab.org/cookbooks/chef-repo/blob/master/roles/gstg-base-lb-fe.json#L48
- Production
- GCP `gprd`: https://dev.gitlab.org/cookbooks/chef-repo/blob/master/roles/gprd-base-lb-fe.json#L48
- GCP `gprd`: https://dev.gitlab.org/cookbooks/chef-repo/blob/master/roles/gprd-base-lb-fe.json#L56
## Object storage
......
......@@ -44,23 +44,20 @@ The plan will also involve testing during two distinct phases of the failover pl
```
# Tab 1: This should take approximately 4.5 minutes
# Make sure to replace `11.1.0-rc4-ee` with the version exposed at http://staging.gitlab.com/help
› gitlab-qa Test::Instance::Any dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ee:11.1.0-rc4-ee https://staging.gitlab.com -- qa/specs/features/api/ qa/specs/features/login/ qa/specs/features/merge_request/
› gitlab-qa Test::Instance::Staging -- qa/specs/features/api/ qa/specs/features/login/ qa/specs/features/merge_request/
```
```
# Tab 2: This should take approximately 6 minutes
# Make sure to replace `11.1.0-rc4-ee` with the version exposed at http://staging.gitlab.com/help
› gitlab-qa Test::Instance::Any dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ee:11.1.0-rc4-ee https://staging.gitlab.com -- qa/specs/features/project/
› gitlab-qa Test::Instance::Staging -- qa/specs/features/project/
```
```
# Tab 3: This should take approximately 5 minutes
# Make sure to replace `11.1.0-rc4-ee` with the version exposed at http://staging.gitlab.com/help
› gitlab-qa Test::Instance::Any dev.gitlab.org:5005/gitlab/omnibus-gitlab/gitlab-ee:11.1.0-rc4-ee https://staging.gitlab.com -- qa/specs/features/repository/
› gitlab-qa Test::Instance::Staging -- qa/specs/features/repository/
```
- [ ] Post results and failures logs + screenshots as comments of this issue
- [ ] Create `Automation Triage RELEASE_MAJOR_VERSION RC#` issues for all the
......@@ -85,17 +82,17 @@ PLEASE FOLLOW THESE INSTRUCTIONS AND REMOVE THEM ONCE DONE.
### During the Blackout
- [ ] When the 🔪 Chef-Runner is at the `Ensure that unicorn, etc, has been
restarted on all hosts` step, send a heads-to QA testers in `#gcp_migration`:
- [ ] When the 🔪 Chef-Runner is at the `Ensure that important processes have
been restarted on all hosts` step, send a heads-to QA testers in `#gcp_migration`:
```
@mkozono @bob @fran @ddavison @dgriffith @DaveSmith @Daniel Gruesso @toon @ruben @jedwardsjones @fabio Heads-up, manual QA testing will start soon. Please be ready (test plan: LINK_TO_MANUAL_TESTPLAN)! :)
@mkozono @bob @fran @ddavison @DylanGriffith @DaveSmith @Daniel Gruesso @toon @ruben @jedwardsjones @fabio Heads-up, manual QA testing will start soon. Please be ready (test plan: LINK_TO_MANUAL_TESTPLAN)! :)
```
- [ ] Post in `#gcp_migration`:
```
@mkozono @bob @fran @ddavison @dgriffith @DaveSmith You can start performing your respective "During Blackout" manual QA scenarios on https://staging.gitlab.com. Please find the scenarios and track results at LINK_TO_MANUAL_TESTPLAN. Thanks in advance! :tada:
@mkozono @bob @fran @ddavison @DylanGriffith @DaveSmith You can start performing your respective "During Blackout" manual QA scenarios on https://staging.gitlab.com. Please find the scenarios and track results at LINK_TO_MANUAL_TESTPLAN. Thanks in advance! :tada:
```
- [ ] Create follow-up issues for all the manual QA failures
......
......@@ -174,3 +174,11 @@ Each [team](https://about.gitlab.com/team/chart/) involved in the effort has a l
1. **Automate the lifecycle of environments for GitLab.com**: https://gitlab.com/gitlab-com/environments
1. **GitLab.com Infrastructure**: https://gitlab.com/gitlab-com/infrastructure
1. **GitLab CE**: https://gitlab.com/gitlab-org/gitlab-ce
## Preparing for a Failover Run
1. **Setup `bin/source_vars`**: `cp ./bin/source_vars_template.sh ./bin/source_vars`
1. **Configure `bin/source_vars`**: The variables are explained in the file. Since this contains secrets, this file should not be checked in. (it's `.gitignore`'d)
1. **Setup the workflow issues**": Run `bin/start-failover-procedure.sh`. This will setup several issues in the issue tracker for performing the checks, failover, tests, etc.
* Any variables in the template in the format `__VARIABLE__` will be substituted with their values from the `bin/source_vars` file, saving manual effort.
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck disable=SC1091,SC1090
source "${SCRIPT_DIR}/../../../workflow-script-commons.sh"
# --------------------------------------------------------------
PRODUCTION_ONLY
cat <<EOD
Open https://tweetdeck.twitter.com/ and tweet from @gitlab:
-------------------------------->8-------------------
Reminder: GitLab.com will be undergoing 2 hours maintenance tomorrow, from ${MAINTENANCE_START_TIME} - ${MAINTENANCE_END_TIME} UTC. Follow @gitlabstatus for more details. ${BLOG_POST_URL}
----------------------------------------------------
EOD
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck disable=SC1091,SC1090
source "${SCRIPT_DIR}/../../../workflow-script-commons.sh"
# --------------------------------------------------------------
cat <<EOD
Open https://tweetdeck.twitter.com/ and tweet from @gitlabstatus:
-------------------------------->8-------------------
Reminder: GitLab.com will be undergoing 2 hours maintenance tomorrow. We'll be live on YouTube. Working doc: ${GOOGLE_DOC_URL}, Blog: ${BLOG_POST_URL}
-------------------------------->8-------------------
EOD
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck disable=SC1091,SC1090
source "${SCRIPT_DIR}/../../../workflow-script-commons.sh"
# --------------------------------------------------------------
function send_slack() {
curl --fail --silent -X POST --data-urlencode 'payload={"text": "'"${1}"'"}' "$SLACK_WEBHOOK_URL"
}
case "${FAILOVER_ENVIRONMENT}" in
"prd")
send_slack "GitLab.com is being migrated to GCP at *${MAINTENANCE_START_TIME}* UTC. There is a 2-hour downtime window. We'll be live on YouTube. Notes in ${GOOGLE_DOC_URL}!"
;;
"stg")
send_slack "We're rehearsing the failover of GitLab.com at *${MAINTENANCE_START_TIME}* UTC by migrating staging.gitlab.com to GCP. Come watch us at ${ZOOM_LINK}! Notes in ${GOOGLE_DOC_URL}!"
;;
*)
die "Unknown environment"
;;
esac
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
ISSUE_TEMPLATES_DIR=${ROOT_DIR}/.gitlab/issue_templates
function find_script_ref() {
grep -Eho "\`./bin.*?\`" "${ISSUE_TEMPLATES_DIR}"/*.md|cut -d\` -f2|cut -d" " -f1|uniq
}
find_script_ref | while IFS='' read -r file; do
if ! [[ -f ${ROOT_DIR}/$file ]] || ! [[ -x ${ROOT_DIR}/$file ]]; then
>&2 echo "$file is missing or not executable"
grep -En "${file}" "${ISSUE_TEMPLATES_DIR}/*.md"
exit 1
fi
done
......@@ -3,76 +3,5 @@
set -euo pipefail
IFS=$'\n\t'
function network_owner() {
local ip=$(dig ${1} +short|tail -1)
if [[ -z $ip ]]; then
echo "DOES_NOT_RESOLVE"
else
local who_is=$(whois ${ip}|grep OrgName|sed -E 's/^.*: +//')
if [[ -z $who_is ]]; then
echo "N/A"
else
echo "$who_is"
fi
fi
}
function rev_name() {
local ip=$(dig ${1} +short|tail -1)
if [[ -z $ip ]]; then
echo "DOES_NOT_RESOLVE"
else
local rev_ip=$(dig -x ${ip} +short)
if [[ -z $rev_ip ]]; then
echo $ip
else
echo $rev_ip
fi
fi
}
function ssh_port_open() {
result=$(ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oConnectTimeout=5 -p "$2" "git@$1" 2>/dev/null)
if [[ "$result" =~ "Welcome to GitLab" ]]; then
echo Yes
else
echo No
fi
}
function ssh_port() {
if [[ $1 == altssh* ]]; then
echo 443
else
echo 22
fi
}
function http_status() {
local status=$((curl --insecure --head --connect-timeout 5 --max-time 5 --silent ${1}|head -1|cut -d\ -f2) || echo "Error")
if [[ -z $status ]]; then
echo "Invalid"
else
echo "$status"
fi
}
function redirect() {
if ! curl --insecure --head --connect-timeout 5 --max-time 5 --silent "$1"|grep Location|sed -E 's/^.*: +//'|cut -c 1-40; then
echo "-"
fi
}
function run() {
printf '%s\t%s\t%s\t%s\t%s\t%s\t\t%s\n' "HOST" "NETWORK" "REV" "HTTPS" "SSH" "REDIRECT"
for host in "$@"
do
printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$host" $(network_owner $host) $(rev_name $host) $(http_status "https://$host") $(ssh_port_open $host $(ssh_port $host)) $(redirect "https://$host")
done
}
echo -e "Date: $(date '+%F %T')\n"
run $@ | column -t -s $'\t'
ruby "$(dirname $0)/hostinfo.rb" "$@" | column -t -s $'\t'
#!/usr/bin/env ruby
Thread.abort_on_exception = true
require 'net/https'
class Check
# These are provided or computed
attr_reader :hostname, :https_url, :ssh_port
# These are looked up in parallel
attr_reader :ip, :https_response, :ssh_port_open, :network_owner, :rev_name
def initialize(hostname)
@hostname = hostname
@https_url = "https://#{hostname}"
@ssh_port = hostname.start_with?('altssh') ? 443 : 22
end
def execute
parallel_lookup!
[
hostname,
network_owner,
rev_name,
https_status,
ssh_port_open,
redirect
].join("\t")
end
def https_status
return "Invalid" unless https_response
https_response.code
end
def redirect
return "-" unless https_response && https_response.key?('Location')
https_response['Location'][0..39]
end
private
def parallel_lookup!
# Checking whois and reverse IP lookup depends on the IP being resolved,
# so set it off separately
ip_thread = Thread.new { @ip = resolv(hostname) }
threads = [
Thread.new { @https_response = curl(https_url) },
Thread.new { @ssh_port_open = check_ssh }
]
# Once this join completes, it's safe to reference the IP
ip_thread.join
threads.concat([
Thread.new { @network_owner = check_network_owner },
Thread.new { @rev_name = check_rev_name }
])
threads.map(&:join)
end
def popen(cmd)
cmd.push(err: :close)
result = IO.popen(cmd) { |io| io.read.strip }
if $?.success?
result
else
nil
end
end
def check_ssh
result = popen(%W[
ssh -o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=no \
-oConnectTimeout=5 \
-p #{ssh_port} \
-q -T \
git@#{hostname}
])
if result =~ /Welcome to GitLab/
"Yes"
else
"No"
end
end
def check_network_owner
return "DOES_NOT_RESOLVE" unless ip
whois(ip) || "N/A"
end
def check_rev_name
return "DOES_NOT_RESOLVE" unless ip
resolv(ip, ptr: true) || ip
end
def resolv(hostname, ptr: false)
cmd = ['dig', '+short']
cmd << '-x' if ptr
cmd << hostname
result = popen(cmd)
return nil unless result
result.split("\n")[-1]
end
def whois(thing)
result = popen(%W[whois #{thing}])
return nil unless result
orgname = result.lines.find { |l| l =~ /OrgName:/ }
orgname&.split(":", 2)[1].strip
end
def curl(url)
result = popen(%W[curl --insecure --head --connect-timeout 5 --max-time 5 --silent #{url}])
return nil unless result
lines = result.lines.map(&:chomp)
return nil unless lines.size > 1
version, code, message = lines[0].split(" ")
kls = Net::HTTPResponse::CODE_TO_OBJ[code] || Net::HTTPResponse
out = kls.new(version, code, message)
lines[1..-1].map do |line|
key, value = line.split(": ")
out.add_field(key, value)
end
out
end
end
class Checks
class << self
def header
["HOST", "NETWORK", "REV", "HTTPS", "SSH", "REDIRECT"].join("\t")
end
end
attr_reader :checks
def initialize(hostnames)
@checks = hostnames.map { |hostname| Check.new(hostname) }
end
def execute
threads = checks.map { |check| Thread.new { check.execute } }
threads.map(&:value)
end
end
results = Checks.new(ARGV).execute
STDOUT.puts Checks.header
results.each { |result| STDOUT.puts result }
#!/usr/bin/env bash
# --------------------------------------------------------------------------------------------------------
# NB. Note: do not edit this file. Please copy this file to `source_vars` and override the defaults there
# --------------------------------------------------------------------------------------------------------
export FAILOVER_DATE="__REQUIRED__" # The date of the failover in YYYY-MM-DD format
export FAILOVER_ENVIRONMENT="__REQUIRED__" # Environment: This should be prd or stg
export MAINTENANCE_START_TIME="__REQUIRED__" # Maintenance window start time in UTC. eg 13h00
export MAINTENANCE_END_TIME="__REQUIRED__" # Maintenance window end time in UTC. eg 15h00
export BLOG_POST_URL="https://about.gitlab.com/2018/07/19/gcp-move-update/" # Blog post URL
export GOOGLE_DOC_URL="https://docs.google.com/document/d/18vGk6dQs7L0oGQOb_bNiFa5JhwLq5WBS7oNxQy09ml8/edit" # Google Working Doc for the event
export SLACK_WEBHOOK_URL="__REQUIRED__" # This is available at https://gitlab.slack.com/services/BBZP9QKLZ
export ZOOM_LINK="https://gitlab.zoom.us/j/859814316" # Find this in the calendar
export GITLAB_INSTANCE="https://dev.gitlab.org/" # Issue tracker in which to create the failover issues
export GITLAB_MIGRATION_PROJECT_PATH="gitlab-com/migration" # Migration Project
export GITLAB_TOKEN="__REQUIRED__" # Available at https://gitlab.1password.com/vaults/ljcrnm55wwhnh5ynr7edbxgowm/allitems/dx2qetzqbbcsrlrj3vtunexagy
export TEAM_COORDINATOR="@nick" # Coordinator (backup: @digitalmoksha)
export TEAM_CHEF_RUNNER="@ahmadsherif" # Chef Runner (backup: @alejandro)
export TEAM_COMMS_HANDLER="@dawsmith" # Comms Handler
export TEAM_DATABASE_WRANGLER="@ibaum" # Database Wrangler (backup: @jarv)
export TEAM_CLOUD_CONDUCTOR="@ahmadsherif" # Cloud Conductor (backup: @alejandro)
export TEAM_QUALITY="@meks" # Quality (backup: @remy)
export TEAM_FAILBACK_HANDLER="@ahmadsherif" # Failback Handler (staging only) (backup: @alejandro)
export TEAM_HEAD_HONCHO="@edjdev" # Head Honcho (production only) (backup: @sytses)
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
export SKIP_HOST_CHECK=true
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# shellcheck disable=SC1091,SC1090
source "${SCRIPT_DIR}/workflow-script-commons.sh"
# --------------------------------------------------------------
function createIssue() {
local title
local template
local body
title=$1
template=$2
body=$(cat "${SCRIPT_DIR}/../.gitlab/issue_templates/${template}.md")
echo "Creating ${title} with template ${template}"
while read -r line; do
# shellcheck disable=SC2001
body=$(echo "$body"| sed "s#__${line}__#${!line}#")
done < <(grep -Eho 'export \w+' ./bin/source_vars_template.sh|cut -d" " -f2)
PROJECT_ID=${GITLAB_MIGRATION_PROJECT_PATH/\//%2f}
curl --silent --fail --request POST -H "Private-Token: ${GITLAB_TOKEN}" "${GITLAB_INSTANCE}/api/v4/projects/${PROJECT_ID}/issues" --data-urlencode "title=${title}" --data-urlencode "description=${body}" > /dev/null
}
if [[ ${FAILOVER_ENVIRONMENT} == "stg" ]]; then
name="${FAILOVER_DATE} STAGING failover attempt:"
else
name="${FAILOVER_DATE} PRODUCTION failover attempt:"
fi
createIssue "${name} preflight checks" "preflight_checks"
createIssue "${name} main procedure" "failover"
createIssue "${name} test plan" "test_plan"
createIssue "${name} failback" "failback"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment