Add Ci Build metrics for pending builds, and stale running builds.

parent 336659ae
Pipeline #182288 passed with stage
in 48 seconds
......@@ -6,6 +6,15 @@ module GitLab
RUNNING_BUILDS_QUERY =
"SELECT COUNT(*) AS count FROM ci_builds AS b WHERE b.type = 'Ci::Build' AND b.status = 'running'".freeze
NOT_UPDATED_RUNNING_BUILDS_QUERY =
"SELECT COUNT(*) AS count FROM ci_builds AS b WHERE b.type = 'Ci::Build' AND b.status = 'running' " \
"AND b.updated_at < NOW() - INTERVAL '1 hour'".freeze
PENDING_BUILDS_QUERY =
"SELECT p.shared_runners_enabled, COUNT(*) AS count FROM ci_builds AS b JOIN projects AS p " \
"ON p.id = b.gl_project_id WHERE b.type = 'Ci::Build' AND b.status = 'pending' " \
"GROUP BY p.shared_runners_enabled".freeze
RUNNING_PER_SHARED_RUNNER_QUERY =
"SELECT b.runner_id, COUNT(*) AS count " \
"FROM ci_builds AS b " \
......@@ -23,6 +32,8 @@ module GitLab
def run
results = {}
results[:all] = get_general(RUNNING_BUILDS_QUERY)
results[:pending] = get_shared_enabled_grouped(PENDING_BUILDS_QUERY)
results[:not_updated_last_hour] = get_general(NOT_UPDATED_RUNNING_BUILDS_QUERY)
results[:per_runner] = get_per_shared_runner(RUNNING_PER_SHARED_RUNNER_QUERY)
results[:per_runner_mirrors] = get_per_shared_runner(RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY)
results
......@@ -55,6 +66,18 @@ module GitLab
0
end
def get_shared_enabled_grouped(query)
results = { shared_enabled: 0, shared_disabled: 0 }
connection.exec(query).each do |row|
row["shared_runners_enabled"] = row["shared_runners_enabled"] == "t"
results[:shared_enabled] = row["count"].to_i if row["shared_runners_enabled"]
results[:shared_disabled] = row["count"].to_i unless row["shared_runners_enabled"]
end
results
rescue PG::UndefinedTable, PG::UndefinedColumn
{ shared_enabled: 0, shared_disabled: 0 }
end
def shared_runners_ids
@shared_runners_ids ||= find_shared_runners_ids
end
......@@ -76,17 +99,10 @@ module GitLab
def probe_db
return self unless @collector.connected?
results = @collector.run
@results = @collector.run
@metrics.add("ci_builds_total", results[:all].to_i, status: "running")
results[:per_runner].each do |key, value|
all = value
for_mirrors = results[:per_runner_mirrors][key]
not_for_mirrors = all - for_mirrors
@metrics.add("ci_builds_per_runner", not_for_mirrors, status: "running", runner: key, mirrors: 0)
@metrics.add("ci_builds_per_runner", for_mirrors, status: "running", runner: key, mirrors: 1)
end
metrics_total
metrics_per_runner
self
end
......@@ -94,6 +110,28 @@ module GitLab
def write_to(target)
target.write(@metrics.to_s)
end
private
def metrics_total
@metrics.add("ci_builds_total", @results[:all], status: "running")
@metrics.add("ci_builds_total", @results[:pending][:shared_enabled], status: "pending",
shared_runners_enabled_projects: 1)
@metrics.add("ci_builds_total", @results[:pending][:shared_disabled], status: "pending",
shared_runners_enabled_projects: 0)
@metrics.add("ci_builds_stale", @results[:not_updated_last_hour], status: "running", when: "last_hour")
end
def metrics_per_runner
@results[:per_runner].each do |key, value|
all = value
for_mirrors = @results[:per_runner_mirrors][key]
not_for_mirrors = all - for_mirrors
@metrics.add("ci_builds_per_runner", not_for_mirrors, status: "running", runner: key, mirrors: 0)
@metrics.add("ci_builds_per_runner", for_mirrors, status: "running", runner: key, mirrors: 1)
end
end
end
end
end
......
......@@ -3,12 +3,17 @@ require "gitlab_monitor/database/ci_builds"
describe GitLab::Monitor::Database do
let(:running_builds_query) { "SELECT RUNNING" }
let(:not_updated_running_builds_query) { "SELECT NOT UPDATED RUNNING" }
let(:pending_builds_query) { "SELECT PENDING BUILDS" }
let(:per_runner_query) { "SELECT ALL RUNNING PER RUNNER %s" }
let(:mirrors_per_runner_query) { "SELECT MIRRORS RUNNING PER RUNNER %s" }
let(:connection) { double("connection") }
before do
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_BUILDS_QUERY", running_builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::NOT_UPDATED_RUNNING_BUILDS_QUERY",
not_updated_running_builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::PENDING_BUILDS_QUERY", pending_builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_PER_SHARED_RUNNER_QUERY", per_runner_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY",
mirrors_per_runner_query)
......@@ -16,7 +21,12 @@ describe GitLab::Monitor::Database do
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection).and_return(connection)
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:shared_runners_ids)
.and_return([1, 2])
allow(connection).to receive(:exec).with(running_builds_query).and_return([{ "count" => 5 }])
allow(connection).to receive(:exec).with(not_updated_running_builds_query).and_return([{ "count" => 2 }])
allow(connection).to receive(:exec).with(pending_builds_query)
.and_return([{ "shared_runners_enabled" => "f", "count" => 2 },
{ "shared_runners_enabled" => "t", "count" => 5 }])
allow(connection).to receive(:exec).with(per_runner_query % "1, 2")
.and_return([{ "runner_id" => 2, "count" => 15 }])
allow(connection).to receive(:exec).with(mirrors_per_runner_query % "1, 2")
......@@ -27,14 +37,18 @@ describe GitLab::Monitor::Database do
let(:collector) { described_class.new(connection_string: "host=localhost") }
it "executes the query" do
expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 }, per_runner_mirrors: { 1 => 0, 2 => 10 }, all: 5)
expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 },
per_runner_mirrors: { 1 => 0, 2 => 10 },
all: 5,
not_updated_last_hour: 2,
pending: { shared_enabled: 5, shared_disabled: 2 })
end
end
describe GitLab::Monitor::Database::CiBuildsProber do
let(:writer) { StringIO.new }
let(:prober) do
described_class.new(opts: { connection_string: "host=localhost" },
described_class.new({ connection_string: "host=localhost" },
metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
end
......@@ -47,6 +61,9 @@ describe GitLab::Monitor::Database do
prober.write_to(writer)
expect(writer.string).to match(/ci_builds_total{status="running"} 5/)
expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="1"} 5/)
expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="0"} 2/)
expect(writer.string).to match(/ci_builds_stale{status="running",when="last_hour"} 2/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="0"} 0/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="0"} 5/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="1"} 0/)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment