Commit d4a85647 authored by Pablo Carranza's avatar Pablo Carranza
Browse files

Merge branch 'feature/sidekiq' into 'master'

Add Sidekiq monitoring

Closes #3

See merge request !18
parents bc7bdabf ef161f44
......@@ -18,3 +18,6 @@ Style/RaiseArgs:
Metrics/MethodLength:
Max: 15
Metrics/AbcSize:
Enabled: false
......@@ -4,12 +4,16 @@ PATH
gitlab-monitor (0.0.8)
pg (~> 0.18.4)
quantile (~> 0.2.0)
redis-namespace (~> 1.5.2)
sidekiq (~> 4.2)
sinatra (~> 1.4.7)
GEM
remote: https://rubygems.org/
specs:
ast (2.3.0)
concurrent-ruby (1.0.2)
connection_pool (2.2.0)
diff-lcs (1.2.5)
parser (2.3.1.2)
ast (~> 2.2)
......@@ -20,6 +24,9 @@ GEM
rack-protection (1.5.3)
rack
rainbow (2.1.0)
redis (3.3.1)
redis-namespace (1.5.2)
redis (~> 3.0, >= 3.0.4)
rspec (3.5.0)
rspec-core (~> 3.5.0)
rspec-expectations (~> 3.5.0)
......@@ -40,6 +47,11 @@ GEM
ruby-progressbar (~> 1.7)
unicode-display_width (~> 1.0, >= 1.0.1)
ruby-progressbar (1.8.1)
sidekiq (4.2.3)
concurrent-ruby (~> 1.0)
connection_pool (~> 2.2, >= 2.2.0)
rack-protection (>= 1.5.0)
redis (~> 3.2, >= 3.2.1)
sinatra (1.4.7)
rack (~> 1.5)
rack-protection (~> 1.4)
......
......@@ -22,5 +22,6 @@ frequency.
1. Process
* Age
* Memory usage
1. Sidekiq
[Prometheus Web exporter]: https://prometheus.io/docs/instrumenting/exporters/
......@@ -29,3 +29,10 @@ probes:
- pid_or_pattern: "git-upload-pack --stateless-rpc"
name: git_upload_pack
quantiles: true
sidekiq:
methods:
- probe_queues
- probe_workers
- probe_retries
opts:
redis_url: "redis://localhost:6379"
......@@ -23,6 +23,8 @@ Gem::Specification.new do |s|
s.add_runtime_dependency "pg", "~> 0.18.4"
s.add_runtime_dependency "sinatra", "~> 1.4.7"
s.add_runtime_dependency "quantile", "~> 0.2.0"
s.add_runtime_dependency "sidekiq", "~> 4.2"
s.add_runtime_dependency "redis-namespace", "~> 1.5.2"
s.add_development_dependency "rspec", "~> 3.3"
end
......@@ -12,5 +12,6 @@ module GitLab
autoload :ProcessProber, "gitlab_monitor/process"
autoload :WebExporter, "gitlab_monitor/web_exporter"
autoload :Prober, "gitlab_monitor/prober"
autoload :SidekiqProber, "gitlab_monitor/sidekiq"
end
end
......@@ -326,6 +326,50 @@ module GitLab
end
end
# Sidekiq runner.
#
# It will take a Redis connection URL and print results to STDOUT
class SidekiqRunner
COMMAND_NAME = "sidekiq".freeze
def initialize(args)
@options = options(args)
@options.parse!
@target = args.shift || STDOUT
@target = File.open(@target, "a") if @target.is_a?(String)
end
def options(args)
args.options do |opts|
opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
opts.on("--redis-url=\"redis://localhost:6379\"", "Redis URL") do |val|
@redis_url = val
end
end
end
def help
@options.help
end
def run
validate!
::GitLab::Monitor::SidekiqProber.new(redis_url: @redis_url)
.probe_queues
.probe_workers
.probe_retries
.write_to(@target)
end
private
def validate!
fail InvalidCLICommand.new(help) unless @redis_url
end
end
def self.commands
[
GIT,
......@@ -334,6 +378,7 @@ module GitLab
DatabaseSlowQueries,
DatabaseVacuumQueries,
Process,
SidekiqRunner,
Server
].each_with_object({}) do |command_class, commands|
commands[command_class::COMMAND_NAME] = command_class
......
......@@ -25,7 +25,7 @@ module GitLab
self
end
def to_s # rubocop:disable Metrics/AbcSize
def to_s
add_quantiles_to_metrics
buffer = ""
......@@ -44,7 +44,7 @@ module GitLab
private
def add_quantiles_to_metrics # rubocop:disable Metrics/AbcSize
def add_quantiles_to_metrics
@quantiles.each do |data, measurements|
estimator = Quantile::Estimator.new
......
require "sidekiq/api"
module GitLab
module Monitor
# A prober for Sidekiq queues
#
# It takes the Redis URL Sidekiq is connected to
class SidekiqProber
def initialize(opts, metrics: PrometheusMetrics.new)
@opts = opts
@metrics = metrics
Sidekiq.configure_client { |config|
config.redis = { url: opts[:redis_url], namespace: "resque:gitlab" }
}
end
def probe_queues
return self unless connected?
job_stats = Hash.new(0)
Sidekiq::Queue.all.each do |queue|
@metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
@metrics.add("sidekiq_queue_latency", queue.latency, name: queue.name)
queue.each { |job| job_stats[job.klass] += 1 }
end
job_stats.each do |class_name, count|
@metrics.add("sidekiq_enqueued_jobs_count", count, name: class_name)
end
self
end
def probe_workers
return self unless connected?
worker_stats = Hash.new(0)
Sidekiq::Workers.new.map do |_pid, _tid, work|
job_klass = work["payload"]["class"]
worker_stats[job_klass] += 1
end
worker_stats.each do |class_name, count|
@metrics.add("sidekiq_running_jobs_count", count, name: class_name)
end
self
end
def probe_retries
return self unless connected?
retry_stats = Hash.new(0)
Sidekiq::RetrySet.new.map do |job|
retry_stats[job.klass] += 1
end
retry_stats.each do |class_name, count|
@metrics.add("sidekiq_jobs_to_be_retried_count", count, name: class_name)
end
self
end
def write_to(target)
target.write(@metrics.to_s)
end
private
def connected?
@connected ||= begin
Sidekiq.redis do |conn|
conn.get("foo")
end
true
end
rescue Redis::CannotConnectError # rubocop:disable Lint/HandleExceptions
# Maybe we're trying connecting to a slave
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment