Add basic metrics for CI Builds

parent 189194b3
.bundle
config/gitlab-monitor*.yml
......@@ -4,6 +4,7 @@ module GitLab
module Database
autoload :Base, "gitlab_monitor/database/base"
autoload :BlockedQueriesProber, "gitlab_monitor/database/blocked_queries"
autoload :CiBuildsProber, "gitlab_monitor/database/ci_builds"
autoload :SlowQueriesProber, "gitlab_monitor/database/slow_queries"
autoload :DeadTuplesProber, "gitlab_monitor/database/dead_tuples"
autoload :VacuumQueriesProber, "gitlab_monitor/database/vacuum_queries"
......
module GitLab
module Monitor
module Database
# A helper class to collect blocked queries
class CiBuildsCollector < Base
RUNNING_BUILDS_QUERY =
"SELECT COUNT(*) AS count FROM ci_builds AS b WHERE b.type = 'Ci::Build' AND b.status = 'running'".freeze
RUNNING_PER_SHARED_RUNNER_QUERY =
"SELECT b.runner_id, COUNT(*) AS count " \
"FROM ci_builds AS b " \
"WHERE b.type = 'Ci::Build' AND b.status = 'running' AND b.runner_id IN (%s) " \
"GROUP BY b.runner_id".freeze
RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY =
"SELECT b.runner_id, COUNT(*) AS count " \
"FROM ci_builds AS b " \
"JOIN projects AS p " \
"ON p.id = b.gl_project_id AND p.pending_delete = 'f' AND p.mirror = 't' AND p.mirror_trigger_builds = 't' " \
"WHERE b.type = 'Ci::Build' AND b.status = 'running' AND b.runner_id IN (%s) " \
"GROUP BY b.runner_id".freeze
def run
results = {}
results[:all] = get_general(RUNNING_BUILDS_QUERY)
results[:per_runner] = get_per_shared_runner(RUNNING_PER_SHARED_RUNNER_QUERY)
results[:per_runner_mirrors] = get_per_shared_runner(RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY)
results
end
private
def get_per_shared_runner(query)
return {} unless shared_runners_ids.count > 0
query = query % shared_runners_ids.join(", ")
builds = begin
result = connection.exec(query)
result.map { |v| [v["runner_id"].to_i, v["count"].to_i] }.to_h
rescue PG::UndefinedTable, PG::UndefinedColumn
{}
end
results = {}
shared_runners_ids.each do |id|
results[id] = builds[id] || 0
end
results
end
def get_general(query)
connection.exec(query)[0]["count"].to_i
rescue PG::UndefinedTable, PG::UndefinedColumn
0
end
def shared_runners_ids
@shared_runners_ids ||= find_shared_runners_ids
end
def find_shared_runners_ids
shared_runners_result = connection.exec("SELECT id FROM ci_runners AS r WHERE r.is_shared = 't'")
return [] unless shared_runners_result.count > 0
shared_runners_result.map { |v| v["id"].to_i }
end
end
# The prober which is called when gathering metrics
class CiBuildsProber
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = CiBuildsCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
results = @collector.run
@metrics.add("ci_builds_total", results[:all].to_i, status: "running")
results[:per_runner].each do |key, value|
all = value
for_mirrors = results[:per_runner_mirrors][key]
not_for_mirrors = all - for_mirrors
@metrics.add("ci_builds_per_runner", not_for_mirrors, status: "running", runner: key, mirrors: 0)
@metrics.add("ci_builds_per_runner", for_mirrors, status: "running", runner: key, mirrors: 1)
end
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
require "spec_helper"
require "gitlab_monitor/database/ci_builds"
describe GitLab::Monitor::Database do
let(:running_builds_query) { "SELECT RUNNING" }
let(:per_runner_query) { "SELECT ALL RUNNING PER RUNNER %s" }
let(:mirrors_per_runner_query) { "SELECT MIRRORS RUNNING PER RUNNER %s" }
let(:connection) { double("connection") }
before do
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_BUILDS_QUERY", running_builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_PER_SHARED_RUNNER_QUERY", per_runner_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY",
mirrors_per_runner_query)
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection).and_return(connection)
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:shared_runners_ids)
.and_return([1, 2])
allow(connection).to receive(:exec).with(running_builds_query).and_return([{ "count" => 5 }])
allow(connection).to receive(:exec).with(per_runner_query % "1, 2")
.and_return([{ "runner_id" => 2, "count" => 15 }])
allow(connection).to receive(:exec).with(mirrors_per_runner_query % "1, 2")
.and_return([{ "runner_id" => 2, "count" => 10 }])
end
describe GitLab::Monitor::Database::CiBuildsCollector do
let(:collector) { described_class.new(connection_string: "host=localhost") }
it "executes the query" do
expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 }, per_runner_mirrors: { 1 => 0, 2 => 10 }, all: 5)
end
end
describe GitLab::Monitor::Database::CiBuildsProber do
let(:writer) { StringIO.new }
let(:prober) do
described_class.new(opts: { connection_string: "host=localhost" },
metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
end
before do
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connected?).and_return(true)
end
it "responds with Prometheus metrics" do
prober.probe_db
prober.write_to(writer)
expect(writer.string).to match(/ci_builds_total{status="running"} 5/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="0"} 0/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="0"} 5/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="1"} 0/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="1"} 10/)
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment