Make remote rails runner commands work much better

bin/scripts/02_failover/060_go/p02/010-stop-mailroom.sh

@@ -12,9 +12,9 @@ source "${UNSYMLINKED_SCRIPT_DIR}/../../../../workflow-script-commons.sh"
 # --------------------------------------------------------------
 
 if [[ ${FAILOVER_ENVIRONMENT} == "stg" ]]; then
-  log_command remote_command "role:staging-base-be-mailroom OR role:gstg-base-be-mailroom" 'sudo gitlab-ctl stop mailroom'
+  log_command knife ssh "role:staging-base-be-mailroom OR role:gstg-base-be-mailroom" 'sudo gitlab-ctl stop mailroom'
 elif [[ ${FAILOVER_ENVIRONMENT} == "prd" ]]; then
-  log_command remote_command "role:gitlab-base-be-mailroom OR role:gprd-base-be-mailroom" 'sudo gitlab-ctl stop mailroom'
+  log_command knife ssh "role:gitlab-base-be-mailroom OR role:gprd-base-be-mailroom" 'sudo gitlab-ctl stop mailroom'
 else
   die "Unknown environment"
 fi

bin/scripts/02_failover/060_go/p02/020-stop-sidekiq-pullmirror.sh

@@ -12,4 +12,4 @@ source "${UNSYMLINKED_SCRIPT_DIR}/../../../../workflow-script-commons.sh"
 
 PRODUCTION_ONLY
 
-log_command remote_command roles:gitlab-base-be-sidekiq-pullmirror "sudo gitlab-ctl stop sidekiq-cluster"
+log_command knife ssh roles:gitlab-base-be-sidekiq-pullmirror "sudo gitlab-ctl stop sidekiq-cluster"

bin/scripts/02_failover/060_go/p02/030-await-sidekiq-drain.rb

@@ -56,7 +56,7 @@ begin
     else
       puts "WARNING: This script is in terminator mode. It will hunt down unwanted jobs and kill them off"
     end
-    pending_queues = Sidekiq::Queue.all.select { |q| q.size > 0 }.sort
+    pending_queues = Sidekiq::Queue.all.select { |q| q.size > 0 }
 
     unless pending_queues.empty?
       puts "Queues:"
@@ -92,7 +92,7 @@ begin
     total_enabled_sidekiq_crons = crons.select { |cron| cron.status == "enabled" }.count
 
     ## Disable most cronjobs
-    unless $dry_run do
+    unless $dry_run
       crons.select { |cron| cron.status == "enabled" && cronjob_can_be_disabled(cron.name) }.each(&:disable!)
     end
 

bin/scripts/02_failover/060_go/p02/030-await-sidekiq-drain.sh

@@ -16,4 +16,5 @@ elif [[ ${FAILOVER_ENVIRONMENT} == "prd" ]]; then
   DEPLOY_HOST=deploy.stg.gitlab.com
 fi
 
-log_command knife ssh "fqdn:${DEPLOY_HOST}" sudo gitlab-rails runner "$(cat "${SCRIPT_DIR}"/030-await-sidekiq-drain.rb)"
+# remote_gitlab_runner will log the command internally
+remote_gitlab_runner "${DEPLOY_HOST}" "${SCRIPT_DIR}/030-await-sidekiq-drain.rb"

bin/verify-failover-config

@@ -9,4 +9,13 @@ source "${SCRIPT_DIR}/workflow-script-commons.sh"
 
 # --------------------------------------------------------------
 
+if [[ ${FAILOVER_ENVIRONMENT} == "stg" ]]; then
+  echo Testing ssh to deploy.stg.gitlab.com
+  log_command ssh deploy.stg.gitlab.com date
+elif [[ ${FAILOVER_ENVIRONMENT} == "prd" ]]; then
+  echo Testing ssh to deploy.gitlab.com
+  log_command ssh deploy.stg.gitlab.com date
+fi
+
+
 echo "Success, you're ready to failover."

bin/workflow-script-commons.sh

@@ -83,19 +83,22 @@ function ensure_valid() {
       fi
       ;;
   esac
+
+  if [[ -z "${SSH_AUTH_SOCK:=}" ]]; then
+    >&2 echo "Warning: you appear to not have SSH agent forwarding setup. You may need to add \`ForwardAgent yes\` to your ~/.ssh/config"
+  fi
 }
 
 function log_command() {
   (set -x; "$@")
 }
 
-function remote_command() {
-  if [[ -z "${SSH_AUTH_SOCK:=}" ]]; then
-    >&2 echo "Warning: you appear to not have SSH agent forwarding setup. You may need to add \`ForwardAgent yes\` to your SSH config"
-  fi
-
-  chef_query=$1; shift
-  knife ssh "${chef_query}" "$@"
+function remote_gitlab_runner() {
+  host=$1
+  script=$2
+  temp="$(ssh "${host}" mktemp).rb"
+  scp "$script" "${host}:${temp}"
+  log_command ssh "${host}" bash -c "set -x; sudo gitlab-rails runner \"\$(cat ${temp})\" && rm ${temp}"
 }
 
 function header() {