Merge branch 'tc-fill-gaps' into 'master'

    * You can use `sudo gitlab-rake geo:status` instead if the UI is non-compliant

    * If failures appear, see Rails console commands to resync repos/wikis: https://gitlab.com/snippets/1713152

    * On staging, this may not complete

    * Fill event log gaps manually, by running the steps in:

      * `/opt/gitlab-migration/migration/bin/scripts/02_failover/060_go/p03/fill-event-log-gaps.rb`

1. [ ] 🐺 {+ Coordinator +}: Wait for all repositories and wikis to become verified

    * Staging: https://dashboards.gitlab.net/d/YoKVGxSmk/gcp-failover-gcp?orgId=1&var-environment=gstg

    * Production: https://dashboards.gitlab.net/d/YoKVGxSmk/gcp-failover-gcp?orgId=1&var-environment=gprd

#!/usr/bin/gitlab-rails runner

GEO_EVENT_LOG_GAPS = 'geo:event_log:gaps'.freeze

def with_redis

  ::Gitlab::Redis::SharedState.with { |redis| yield redis }

end

# Get gaps tracked in redis

gap_ids = with_redis { |redis| redis.zrangebyscore(GEO_EVENT_LOG_GAPS, '-inf', '+inf', with_scores: true) };

gap_ids.each { |gap| puts "Found event log gap with id #{gap.first} @ #{Time.now.to_i - gap.second} seconds ago" }; gap_ids.count

# Fetch the events from database, if they exist

gap_events = Geo::EventLog.where(id: gap_ids.map(&:first)); gap_events.count

# Use the daemon to process the gap events

daemon = Gitlab::Geo::LogCursor::Daemon.new; nil

gap_events.each { |gap_event| daemon.send(:handle_single_event, gap_event.first) }; nil

# Delete the gaps from redis

with_redis { |redis| redis.zrem(GEO_EVENT_LOG_GAPS, gap_events.map(&:id)) }

puts "Filled #{gap_events.count} gaps out of #{gaps_ids.count}"