Skip to content

GitLab

Verified Commit 97986505 authored by Tomasz Maczukin's avatar Tomasz Maczukin
Browse files
Options

Improve CI metrics

parent 90203671
Hide whitespace changes
Inline Side-by-side
Showing with 138 additions and 88 deletions
lib/gitlab_monitor/database/ci_builds.rb
View file @ 97986505
......@@ -4,36 +4,44 @@ module GitLab
# A helper class to collect blocked queries
class CiBuildsCollector < Base
BUILDS_QUERY =
"SELECT " \
" projects.namespace_id, " \
" ci_builds.status, " \
" projects.shared_runners_enabled, " \
" COUNT(*) AS count " \
" FROM ci_builds " \
" JOIN projects " \
" ON projects.id = ci_builds.project_id " \
" WHERE ci_builds.type = 'Ci::Build' " \
" AND ci_builds.status IN ('created', 'pending') " \
" GROUP BY " \
" projects.namespace_id, " \
" ci_builds.status, " \
" projects.shared_runners_enabled".freeze
<<~SQL.freeze
SELECT
projects.namespace_id,
ci_builds.status,
projects.shared_runners_enabled,
COUNT(*) AS count
FROM ci_builds
JOIN projects
ON projects.id = ci_builds.project_id
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('created', 'pending')
AND projects.pending_delete = 'f'
GROUP BY
projects.namespace_id,
ci_builds.status,
projects.shared_runners_enabled
SQL
STALE_BUILDS_QUERY =
"SELECT " \
" COUNT(*) AS count " \
" FROM ci_builds " \
" WHERE ci_builds.type = 'Ci::Build' " \
" AND ci_builds.status = 'running' " \
" AND ci_builds.updated_at < NOW() - INTERVAL '1 hour'".freeze
<<~SQL.freeze
SELECT
COUNT(*) AS count
FROM ci_builds
JOIN projects
ON projects.id = ci_builds.project_id
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status = 'running'
AND ci_builds.updated_at < NOW() - INTERVAL '1 hour'
AND projects.pending_delete = 'f'
SQL
PER_RUNNER_QUERY_EE =
<<~SQL.freeze
SELECT
ci_builds.runner_id,
ci_runners.is_shared,
projects.namespace_id,
projects.mirror,
projects.pending_delete,
projects.mirror_trigger_builds,
ci_pipelines.pipeline_schedule_id,
ci_builds.trigger_request_id,
......@@ -47,6 +55,7 @@ module GitLab
ON ci_pipelines.id = ci_builds.commit_id
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status = 'running'
AND projects.pending_delete = 'f'
GROUP BY
ci_builds.runner_id,
ci_runners.is_shared,
......@@ -59,7 +68,7 @@ module GitLab
SELECT
ci_builds.runner_id,
ci_runners.is_shared,
projects.pending_delete,
projects.namespace_id,
ci_pipelines.pipeline_schedule_id,
ci_builds.trigger_request_id,
COUNT(*) AS count
......@@ -72,10 +81,11 @@ module GitLab
ON ci_pipelines.id = ci_builds.commit_id
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status = 'running'
AND projects.pending_delete = 'f'
GROUP BY
ci_builds.runner_id,
projects.id,
ci_runners.is_shared,
projects.namespace_id,
ci_pipelines.pipeline_schedule_id,
ci_builds.trigger_request_id
SQL
......@@ -84,16 +94,41 @@ module GitLab
results = {}
results.merge!(get_builds(BUILDS_QUERY))
results[:stale_builds] = get_general(STALE_BUILDS_QUERY)
results[:stale_builds] = get_stale_builds(STALE_BUILDS_QUERY)
query = mirror_column? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
results[:per_runner] = get_per_runner(query)
results
end
private
def get_builds(query)
results = { pending_builds: [], created_builds: [] }
connection.exec(query).each do |row|
row_data = { namespace: row["namespace_id"].to_s,
shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
value: row["count"].to_i }
if row["status"] == "pending"
results[:pending_builds].push(row_data)
elsif row["status"] == "created"
results[:created_builds].push(row_data)
end
end
results
rescue PG::UndefinedTable, PG::UndefinedColumn
results
end
def get_stale_builds(query)
connection.exec(query)[0]["count"].to_i
rescue PG::UndefinedTable, PG::UndefinedColumn
0
end
def get_per_runner(query)
results = []
......@@ -106,45 +141,19 @@ module GitLab
[]
end
def transform_row_to_values(row) # rubocop:disable Metrics/CyclomaticComplexity
def transform_row_to_values(row)
{
runner: row["runner_id"].to_s,
shared_runner: row["is_shared"] == "t" ? "yes" : "no",
namespace: row["namespace_id"].to_s,
mirror: row["mirror"] == "t" ? "yes" : "no",
scheduled: row["pipeline_schedule_id"] ? "yes" : "no",
triggered: row["trigger_request_id"] ? "yes" : "no",
pending_delete: row["pending_delete"] == "t" ? "yes" : "no",
mirror_trigger_builds: row["mirror_trigger_builds"] == "t" ? "yes" : "no",
value: row["count"].to_i
}
end
def get_general(query)
connection.exec(query)[0]["count"].to_i
rescue PG::UndefinedTable, PG::UndefinedColumn
0
end
def get_builds(query)
results = { pending_builds: [], created_builds: [] }
connection.exec(query).each do |row|
shared_runners = row["shared_runners_enabled"] == "t" ? "yes" : "no"
namespace = row["namespace_id"].to_s
value = row["count"].to_i
if row["status"] == "pending"
results[:pending_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
elsif row["status"] == "created"
results[:created_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
end
end
results
rescue PG::UndefinedTable, PG::UndefinedColumn
results
end
def mirror_column?
@mirror_column ||=
begin
......@@ -187,24 +196,30 @@ module GitLab
private
def ci_builds_metrics(results_list, metric_name)
other_value = { "yes" => 0, "no" => 0 }
other_values = {}
results_list.each do |metric|
shared_runners = metric[:shared_runners]
namespace = metric[:namespace]
value = metric[:value]
# If we have a low value, put the value into an "other" bucket.
if value < 10
other_value[shared_runners] += value
if metric[:value] < 10
key = { shared_runners: metric[:shared_runners] }
other_values[key] ||= 0
other_values[key] += metric[:value]
else
@metrics.add(metric_name, value, namespace: namespace, shared_runners: shared_runners)
add_ci_created_pending_builds(metric_name, metric[:value], metric)
end
end
# Add metrics for the "other" bucket.
@metrics.add(metric_name, other_value["yes"], namespace: "", shared_runners: "yes")
@metrics.add(metric_name, other_value["no"], namespace: "", shared_runners: "no")
other_values.each do |key, value|
add_ci_created_pending_builds(metric_name, value, key)
end
end
def add_ci_created_pending_builds(metric_name, value, labels)
@metrics.add(metric_name,
value,
namespace: labels[:namespace] ? labels[:namespace] : "",
shared_runners: labels[:shared_runners])
end
def ci_stale_builds_metrics
......@@ -212,18 +227,38 @@ module GitLab
end
def metrics_per_runner
other_values = {}
@results[:per_runner].each do |metric|
@metrics.add("ci_running_builds",
metric[:value],
runner: metric[:runner],
shared_runner: metric[:shared_runner],
mirror: metric[:mirror],
pending_delete: metric[:pending_delete],
mirror_trigger_builds: metric[:mirror_trigger_builds],
scheduled: metric[:scheduled],
triggered: metric[:triggered])
# If we have a low value, put the value into an "other" bucket.
if metric[:value] < 10
key = { runner: metric[:runner], shared_runner: metric[:shared_runner], mirror: metric[:mirror],
mirror_trigger_builds: metric[:mirror_trigger_builds], scheduled: metric[:scheduled],
triggered: metric[:triggered] }
other_values[key] ||= 0
other_values[key] += metric[:value]
else
add_ci_running_builds(metric[:value], metric)
end
end
# Add metrics for the "other" bucket.
other_values.each do |key, value|
add_ci_running_builds(value, key)
end
end
def add_ci_running_builds(value, labels)
@metrics.add("ci_running_builds",
value,
runner: labels[:runner],
shared_runner: labels[:shared_runner],
namespace: labels[:namespace] ? labels[:namespace] : "",
mirror: labels[:mirror],
mirror_trigger_builds: labels[:mirror_trigger_builds],
scheduled: labels[:scheduled],
triggered: labels[:triggered])
end
end
end
end
......
spec/database/ci_builds_spec.rb
View file @ 97986505
......@@ -23,23 +23,40 @@ describe GitLab::Monitor::Database do
{ "shared_runners_enabled" => "f", "status" => "created", "namespace_id" => "2", "count" => 20 },
{ "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "2", "count" => 50 },
{ "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "3", "count" => 1 },
{ "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "4", "count" => 2 }])
{ "shared_runners_enabled" => "t", "status" => "pending", "namespace_id" => "4", "count" => 2 },
{ "shared_runners_enabled" => "f", "status" => "pending", "namespace_id" => "5", "count" => 2 }])
allow(connection).to receive(:exec).with(stale_builds_query).and_return([{ "count" => 2 }])
allow(connection).to receive(:exec).with(per_runner_query)
.and_return([{ "runner_id" => 1,
"is_shared" => "t",
"namespace_id" => 1,
"mirror" => "f",
"pending_delete" => "f",
"mirror_trigger_builds" => "f",
"pipeline_schedule_id" => 1,
"trigger_request_id" => nil,
"count" => 15 },
{ "runner_id" => 2,
"is_shared" => "f",
"namespace_id" => 2,
"mirror" => "t",
"pipeline_schedule_id" => nil,
"trigger_request_id" => 3,
"mirror_trigger_builds" => "t",
"count" => 5 },
{ "runner_id" => 2,
"is_shared" => "f",
"namespace_id" => 3,
"mirror" => "t",
"pipeline_schedule_id" => nil,
"trigger_request_id" => 3,
"mirror_trigger_builds" => "t",
"count" => 5 },
{ "runner_id" => 3,
"is_shared" => "f",
"namespace_id" => 4,
"mirror" => "t",
"pipeline_schedule_id" => nil,
"trigger_request_id" => 3,
"pending_delete" => "f",
"mirror_trigger_builds" => "t",
"count" => 5 }])
end
......@@ -49,14 +66,17 @@ describe GitLab::Monitor::Database do
it "executes the query" do
expect(collector.run).to eq(per_runner: [
{ runner: "1", shared_runner: "yes", mirror: "no", pending_delete: "no", mirror_trigger_builds: "no", scheduled: "yes", triggered: "no", value: 15 },
{ runner: "2", shared_runner: "no", mirror: "yes", pending_delete: "no", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", value: 5 }
{ runner: "1", shared_runner: "yes", namespace: "1", mirror: "no", mirror_trigger_builds: "no", scheduled: "yes", triggered: "no", value: 15 },
{ runner: "2", shared_runner: "no", namespace: "2", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", value: 5 },
{ runner: "2", shared_runner: "no", namespace: "3", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", value: 5 },
{ runner: "3", shared_runner: "no", namespace: "4", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", value: 5 }
],
pending_builds: [
{ namespace: "1", shared_runners: "yes", value: 30 },
{ namespace: "2", shared_runners: "yes", value: 50 },
{ namespace: "3", shared_runners: "yes", value: 1 },
{ namespace: "4", shared_runners: "yes", value: 2 }
{ namespace: "4", shared_runners: "yes", value: 2 },
{ namespace: "5", shared_runners: "no", value: 2 }
],
created_builds: [
{ namespace: "1", shared_runners: "no", value: 10 },
......@@ -86,14 +106,13 @@ describe GitLab::Monitor::Database do
ci_pending_builds{namespace="1",shared_runners="yes"} 30
ci_pending_builds{namespace="2",shared_runners="yes"} 50
ci_pending_builds{namespace="",shared_runners="yes"} 3
ci_pending_builds{namespace="",shared_runners="no"} 0
ci_pending_builds{namespace="",shared_runners="no"} 2
ci_created_builds{namespace="1",shared_runners="no"} 10
ci_created_builds{namespace="2",shared_runners="no"} 20
ci_created_builds{namespace="",shared_runners="yes"} 0
ci_created_builds{namespace="",shared_runners="no"} 0
ci_stale_builds 2
ci_running_builds{runner="1",shared_runner="yes",mirror="no",pending_delete="no",mirror_trigger_builds="no",scheduled="yes",triggered="no"} 15
ci_running_builds{runner="2",shared_runner="no",mirror="yes",pending_delete="no",mirror_trigger_builds="yes",scheduled="no",triggered="yes"} 5
ci_running_builds{runner="1",shared_runner="yes",namespace="1",mirror="no",mirror_trigger_builds="no",scheduled="yes",triggered="no"} 15
ci_running_builds{runner="2",shared_runner="no",namespace="",mirror="yes",mirror_trigger_builds="yes",scheduled="no",triggered="yes"} 10
ci_running_builds{runner="3",shared_runner="no",namespace="",mirror="yes",mirror_trigger_builds="yes",scheduled="no",triggered="yes"} 5
OUTPUT
expect(writer.string).to eq(output)
......@@ -110,10 +129,6 @@ describe GitLab::Monitor::Database do
prober.write_to(writer)
output = <<~OUTPUT
ci_pending_builds{namespace="",shared_runners="yes"} 0
ci_pending_builds{namespace="",shared_runners="no"} 0
ci_created_builds{namespace="",shared_runners="yes"} 0
ci_created_builds{namespace="",shared_runners="no"} 0
ci_stale_builds 0
OUTPUT
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment