Commit 11c9071d authored by Ahmad Sherif's avatar Ahmad Sherif
Browse files

Merge branch 'bjk/ci_metrics' into 'master'

Improve CI builds metrics

See merge request !40
parents 80107b6e f65f1bd4
...@@ -3,62 +3,81 @@ module GitLab ...@@ -3,62 +3,81 @@ module GitLab
module Database module Database
# A helper class to collect blocked queries # A helper class to collect blocked queries
class CiBuildsCollector < Base class CiBuildsCollector < Base
TOTAL_BUILDS_QUERY = BUILDS_QUERY =
"SELECT p.shared_runners_enabled, b.status, COUNT(*) AS count FROM ci_builds AS b JOIN projects AS p " \ "SELECT " \
"ON p.id = b.gl_project_id WHERE b.type = 'Ci::Build' AND b.status IN ('created', 'pending', 'running') " \ " projects.namespace_id, " \
"GROUP BY p.shared_runners_enabled, b.status".freeze " ci_builds.status, " \
" projects.shared_runners_enabled, " \
" COUNT(*) AS count " \
" FROM ci_builds " \
" JOIN projects " \
" ON projects.id = ci_builds.gl_project_id " \
" WHERE ci_builds.type = 'Ci::Build' " \
" AND ci_builds.status IN ('created', 'pending') " \
" GROUP BY " \
" projects.namespace_id, " \
" ci_builds.status, " \
" projects.shared_runners_enabled".freeze
NOT_UPDATED_RUNNING_BUILDS_QUERY = NOT_UPDATED_RUNNING_BUILDS_QUERY =
"SELECT COUNT(*) AS count FROM ci_builds AS b WHERE b.type = 'Ci::Build' AND b.status = 'running' " \ "SELECT " \
"AND b.updated_at < NOW() - INTERVAL '1 hour'".freeze " COUNT(*) AS count " \
" FROM ci_builds " \
RUNNING_PER_SHARED_RUNNER_QUERY = " WHERE ci_builds.type = 'Ci::Build' " \
"SELECT b.runner_id, COUNT(*) AS count " \ " AND ci_builds.status = 'running' " \
"FROM ci_builds AS b " \ " AND ci_builds.updated_at < NOW() - INTERVAL '1 hour'".freeze
"WHERE b.type = 'Ci::Build' AND b.status = 'running' AND b.runner_id IN (%s) " \
"GROUP BY b.runner_id".freeze PER_RUNNER_QUERY =
"SELECT " \
RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY = " ci_builds.runner_id, " \
"SELECT b.runner_id, COUNT(*) AS count " \ " ci_runners.is_shared, " \
"FROM ci_builds AS b " \ " projects.mirror, " \
"JOIN projects AS p " \ " projects.pending_delete, " \
"ON p.id = b.gl_project_id AND p.pending_delete = 'f' AND p.mirror = 't' AND p.mirror_trigger_builds = 't' " \ " projects.mirror_trigger_builds, " \
"WHERE b.type = 'Ci::Build' AND b.status = 'running' AND b.runner_id IN (%s) " \ " COUNT(*) AS count " \
"GROUP BY b.runner_id".freeze " FROM ci_builds " \
" JOIN ci_runners " \
" ON ci_runners.id = ci_builds.runner_id " \
" JOIN projects " \
" ON projects.id = ci_builds.gl_project_id " \
" WHERE ci_builds.type = 'Ci::Build' " \
" AND ci_builds.status = 'running' " \
" GROUP BY " \
" ci_builds.runner_id, " \
" projects.mirror, " \
" projects.pending_delete, " \
" projects.mirror_trigger_builds, " \
" ci_runners.is_shared".freeze
def run def run
results = {} results = {}
total = get_shared_enabled_and_status_grouped(TOTAL_BUILDS_QUERY) results.merge!(get_builds(BUILDS_QUERY))
results[:created] = total[:created]
results[:pending] = total[:pending]
results[:running] = total[:running]
results[:not_updated_last_hour] = get_general(NOT_UPDATED_RUNNING_BUILDS_QUERY) results[:not_updated_last_hour] = get_general(NOT_UPDATED_RUNNING_BUILDS_QUERY)
results[:per_runner] = get_per_shared_runner(RUNNING_PER_SHARED_RUNNER_QUERY) results[:per_runner] = get_per_runner(PER_RUNNER_QUERY)
results[:per_runner_mirrors] = get_per_shared_runner(RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY)
results results
end end
private private
def get_per_shared_runner(query) def get_per_runner(query)
return {} unless shared_runners_ids.count > 0 results = []
query = query % shared_runners_ids.join(", ")
builds = begin connection.exec(query).each do |row|
result = connection.exec(query) results.push(
result.map { |v| [v["runner_id"].to_i, v["count"].to_i] }.to_h runner: row["runner_id"].to_s,
rescue PG::UndefinedTable, PG::UndefinedColumn shared_runner: row["is_shared"] == "t" ? "yes" : "no",
{} mirror: row["mirror"] == "t" ? "yes" : "no",
end pending_delete: row["pending_delete"] == "t" ? "yes" : "no",
mirror_trigger_builds: row["mirror_trigger_builds"] == "t" ? "yes" : "no",
results = {} value: row["count"].to_i
shared_runners_ids.each do |id| )
results[id] = builds[id] || 0
end end
results results
rescue PG::UndefinedTable, PG::UndefinedColumn
[]
end end
def get_general(query) def get_general(query)
...@@ -67,34 +86,24 @@ module GitLab ...@@ -67,34 +86,24 @@ module GitLab
0 0
end end
def get_shared_enabled_and_status_grouped(query) def get_builds(query)
results = { created: { shared_enabled: 0, shared_disabled: 0 }, results = { pending_builds: [], created_builds: [] }
pending: { shared_enabled: 0, shared_disabled: 0 },
running: { shared_enabled: 0, shared_disabled: 0 } }
connection.exec(query).each do |row| connection.exec(query).each do |row|
shared_enabled = row["shared_runners_enabled"] == "t" shared_runners = row["shared_runners_enabled"] == "t" ? "yes" : "no"
status = row["status"].to_sym namespace = row["namespace_id"].to_s
value = row["count"].to_i
results[status][:shared_enabled] = row["count"].to_i if shared_enabled
results[status][:shared_disabled] = row["count"].to_i unless shared_enabled if row["status"] == "pending"
results[:pending_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
elsif row["status"] == "created"
results[:created_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
end
end end
results results
rescue PG::UndefinedTable, PG::UndefinedColumn rescue PG::UndefinedTable, PG::UndefinedColumn
{ created: { shared_enabled: 0, shared_disabled: 0 }, {}
pending: { shared_enabled: 0, shared_disabled: 0 },
running: { shared_enabled: 0, shared_disabled: 0 } }
end
def shared_runners_ids
@shared_runners_ids ||= find_shared_runners_ids
end
def find_shared_runners_ids
shared_runners_result = connection.exec("SELECT id FROM ci_runners AS r WHERE r.is_shared = 't'")
return [] unless shared_runners_result.count > 0
shared_runners_result.map { |v| v["id"].to_i }
end end
end end
...@@ -110,7 +119,9 @@ module GitLab ...@@ -110,7 +119,9 @@ module GitLab
@results = @collector.run @results = @collector.run
metrics_total ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
ci_builds_metrics(@results[:created_builds], "ci_created_builds")
ci_stale_builds_metrics
metrics_per_runner metrics_per_runner
self self
...@@ -122,30 +133,37 @@ module GitLab ...@@ -122,30 +133,37 @@ module GitLab
private private
def metrics_total def ci_builds_metrics(results_list, metric_name)
@metrics.add("ci_builds_total", @results[:created][:shared_enabled], status: "created", other_value = { "yes" => 0, "no" => 0 }
shared_runners_enabled_projects: 1) results_list.each do |metric|
@metrics.add("ci_builds_total", @results[:created][:shared_disabled], status: "created", shared_runners = metric[:shared_runners]
shared_runners_enabled_projects: 0) namespace = metric[:namespace]
@metrics.add("ci_builds_total", @results[:pending][:shared_enabled], status: "pending", value = metric[:value]
shared_runners_enabled_projects: 1) # If we have a low value, put the value into an "other" bucket.
@metrics.add("ci_builds_total", @results[:pending][:shared_disabled], status: "pending", if value < 10
shared_runners_enabled_projects: 0) other_value[shared_runners] += value
@metrics.add("ci_builds_total", @results[:running][:shared_enabled], status: "running", else
shared_runners_enabled_projects: 1) @metrics.add(metric_name, value, namespace: namespace, shared_runners: shared_runners)
@metrics.add("ci_builds_total", @results[:running][:shared_disabled], status: "running", end
shared_runners_enabled_projects: 0) end
# Add metrics for the "other" bucket.
@metrics.add(metric_name, other_value["yes"], namespace: "", shared_runners: "yes")
@metrics.add(metric_name, other_value["no"], namespace: "", shared_runners: "no")
end
def ci_stale_builds_metrics
@metrics.add("ci_builds_stale", @results[:not_updated_last_hour], status: "running", when: "last_hour") @metrics.add("ci_builds_stale", @results[:not_updated_last_hour], status: "running", when: "last_hour")
end end
def metrics_per_runner def metrics_per_runner
@results[:per_runner].each do |key, value| @results[:per_runner].each do |metric|
all = value @metrics.add("ci_running_builds",
for_mirrors = @results[:per_runner_mirrors][key] metric[:value],
not_for_mirrors = all - for_mirrors runner: metric[:runner],
@metrics.add("ci_builds_per_runner", not_for_mirrors, status: "running", runner: key, mirrors: 0) shared_runner: metric[:shared_runner],
@metrics.add("ci_builds_per_runner", for_mirrors, status: "running", runner: key, mirrors: 1) mirror: metric[:mirror],
pending_delete: metric[:pending_delete],
mirror_trigger_builds: metric[:mirror_trigger_builds])
end end
end end
end end
......
require "spec_helper" require "spec_helper"
require "gitlab_monitor/database/ci_builds" require "gitlab_monitor/database/ci_builds"
# rubocop:disable Metrics/LineLength
describe GitLab::Monitor::Database do describe GitLab::Monitor::Database do
let(:total_builds_query) { "SELECT TOTAL BUILDS" } let(:builds_query) { "SELECT BUILDS" }
let(:not_updated_running_builds_query) { "SELECT NOT UPDATED RUNNING" } let(:not_updated_running_builds_query) { "SELECT NOT UPDATED RUNNING" }
let(:per_runner_query) { "SELECT ALL RUNNING PER RUNNER %s" } let(:per_runner_query) { "SELECT ALL RUNNING PER RUNNER" }
let(:mirrors_per_runner_query) { "SELECT MIRRORS RUNNING PER RUNNER %s" }
let(:connection) { double("connection") } let(:connection) { double("connection") }
before do before do
stub_const("GitLab::Monitor::Database::CiBuildsCollector::TOTAL_BUILDS_QUERY", total_builds_query) stub_const("GitLab::Monitor::Database::CiBuildsCollector::BUILDS_QUERY", builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::NOT_UPDATED_RUNNING_BUILDS_QUERY", stub_const("GitLab::Monitor::Database::CiBuildsCollector::NOT_UPDATED_RUNNING_BUILDS_QUERY",
not_updated_running_builds_query) not_updated_running_builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_PER_SHARED_RUNNER_QUERY", per_runner_query) stub_const("GitLab::Monitor::Database::CiBuildsCollector::PER_RUNNER_QUERY", per_runner_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY",
mirrors_per_runner_query)
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection).and_return(connection) allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection).and_return(connection)
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:shared_runners_ids)
.and_return([1, 2])
allow(connection).to receive(:exec).with(total_builds_query) allow(connection).to receive(:exec).with(builds_query)
.and_return([{ "shared_runners_enabled" => "f", "status" => "created", "count" => 1 }, .and_return([{ "shared_runners_enabled" => "f", "status" => "created", "namespace_id" => "1", "count" => 10 },
{ "shared_runners_enabled" => "t", "status" => "created", "count" => 4 }, { "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "1", "count" => 30 },
{ "shared_runners_enabled" => "f", "status" => "pending", "count" => 2 }, { "shared_runners_enabled" => "f", "status" => "created", "namespace_id" => "2", "count" => 20 },
{ "shared_runners_enabled" => "t", "status" => "pending", "count" => 5 }, { "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "2", "count" => 50 },
{ "shared_runners_enabled" => "f", "status" => "running", "count" => 2 }, { "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "3", "count" => 1 },
{ "shared_runners_enabled" => "t", "status" => "running", "count" => 3 }]) { "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "4", "count" => 2 }])
allow(connection).to receive(:exec).with(not_updated_running_builds_query).and_return([{ "count" => 2 }]) allow(connection).to receive(:exec).with(not_updated_running_builds_query).and_return([{ "count" => 2 }])
allow(connection).to receive(:exec).with(per_runner_query % "1, 2") allow(connection).to receive(:exec).with(per_runner_query)
.and_return([{ "runner_id" => 2, "count" => 15 }]) .and_return([{ "runner_id" => 1,
allow(connection).to receive(:exec).with(mirrors_per_runner_query % "1, 2") "is_shared" => "t",
.and_return([{ "runner_id" => 2, "count" => 10 }]) "mirror" => "f",
"pending_delete" => "f",
"mirror_trigger_builds" => "f",
"count" => 15 },
{ "runner_id" => 2,
"is_shared" => "f",
"mirror" => "t",
"pending_delete" => "f",
"mirror_trigger_builds" => "t",
"count" => 5 }])
end end
describe GitLab::Monitor::Database::CiBuildsCollector do describe GitLab::Monitor::Database::CiBuildsCollector do
let(:collector) { described_class.new(connection_string: "host=localhost") } let(:collector) { described_class.new(connection_string: "host=localhost") }
it "executes the query" do it "executes the query" do
expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 }, expect(collector.run).to eq(per_runner: [
per_runner_mirrors: { 1 => 0, 2 => 10 }, { runner: "1", shared_runner: "yes", mirror: "no", pending_delete: "no", mirror_trigger_builds: "no", value: 15 },
created: { shared_enabled: 4, shared_disabled: 1 }, { runner: "2", shared_runner: "no", mirror: "yes", pending_delete: "no", mirror_trigger_builds: "yes", value: 5 }
pending: { shared_enabled: 5, shared_disabled: 2 }, ],
running: { shared_enabled: 3, shared_disabled: 2 }, pending_builds: [
{ namespace: "1", shared_runners: "yes", value: 30 },
{ namespace: "2", shared_runners: "yes", value: 50 },
{ namespace: "3", shared_runners: "yes", value: 1 },
{ namespace: "4", shared_runners: "yes", value: 2 }
],
created_builds: [
{ namespace: "1", shared_runners: "no", value: 10 },
{ namespace: "2", shared_runners: "no", value: 20 }
],
not_updated_last_hour: 2) not_updated_last_hour: 2)
end end
end end
...@@ -62,17 +76,17 @@ describe GitLab::Monitor::Database do ...@@ -62,17 +76,17 @@ describe GitLab::Monitor::Database do
prober.probe_db prober.probe_db
prober.write_to(writer) prober.write_to(writer)
expect(writer.string).to match(/ci_builds_total{status="created",shared_runners_enabled_projects="1"} 4/) expect(writer.string).to match(/ci_created_builds{namespace="1",shared_runners="no"} 10/)
expect(writer.string).to match(/ci_builds_total{status="created",shared_runners_enabled_projects="0"} 1/) expect(writer.string).to match(/ci_created_builds{namespace="2",shared_runners="no"} 20/)
expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="1"} 5/) expect(writer.string).to match(/ci_pending_builds{namespace="1",shared_runners="yes"} 30/)
expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="0"} 2/) expect(writer.string).to match(/ci_pending_builds{namespace="2",shared_runners="yes"} 50/)
expect(writer.string).to match(/ci_builds_total{status="running",shared_runners_enabled_projects="1"} 3/) expect(writer.string).to match(/ci_pending_builds{namespace="",shared_runners="yes"} 3/)
expect(writer.string).to match(/ci_builds_total{status="running",shared_runners_enabled_projects="0"} 2/) expect(writer.string).to match(/ci_pending_builds{namespace="",shared_runners="no"} 0/)
expect(writer.string).to match(/ci_created_builds{namespace="",shared_runners="yes"} 0/)
expect(writer.string).to match(/ci_created_builds{namespace="",shared_runners="no"} 0/)
expect(writer.string).to match(/ci_builds_stale{status="running",when="last_hour"} 2/) expect(writer.string).to match(/ci_builds_stale{status="running",when="last_hour"} 2/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="0"} 0/) expect(writer.string).to match(/ci_running_builds{runner="1",shared_runner="yes",mirror="no",pending_delete="no",mirror_trigger_builds="no"} 15/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="0"} 5/) expect(writer.string).to match(/ci_running_builds{runner="2",shared_runner="no",mirror="yes",pending_delete="no",mirror_trigger_builds="yes"} 5/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="1"} 0/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="1"} 10/)
end end
end end
end end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment