Commit 1988413c authored by Ben Kochie's avatar Ben Kochie

Refactor total builds metrics

* Remove `_total` from metric name, this is a gauge.
* Split `running` and `created` builds into separate metrics.
* Add `namespace.path` as new metric label.
* Refactor `metrics_total` function to support new metrics and labeling.
* Reduce cardinality for `pending` and `created` builds by ignoring
  values less than 10.
* Split stale builds metrics to separate function.
* Update specs.
parent 5a44b395
......@@ -3,19 +3,23 @@ module GitLab
module Database
# A helper class to collect blocked queries
class CiBuildsCollector < Base
TOTAL_BUILDS_QUERY =
BUILDS_QUERY =
"SELECT " \
" projects.shared_runners_enabled, " \
" namespaces.path, " \
" ci_builds.status, " \
" projects.shared_runners_enabled, " \
" COUNT(*) AS count " \
" FROM ci_builds " \
" JOIN projects " \
" ON projects.id = ci_builds.gl_project_id " \
" JOIN namespaces " \
" ON namespaces.id = projects.namespace_id " \
" WHERE ci_builds.type = 'Ci::Build' " \
" AND ci_builds.status IN ('created', 'pending', 'running') " \
" AND ci_builds.status IN ('created', 'pending') " \
" GROUP BY " \
" projects.shared_runners_enabled, " \
" ci_builds.status".freeze
" namespaces.path, " \
" ci_builds.status, " \
" projects.shared_runners_enabled".freeze
NOT_UPDATED_RUNNING_BUILDS_QUERY =
"SELECT " \
......@@ -53,10 +57,7 @@ module GitLab
def run
results = {}
total = get_shared_enabled_and_status_grouped(TOTAL_BUILDS_QUERY)
results[:created] = total[:created]
results[:pending] = total[:pending]
results[:running] = total[:running]
results.merge!(get_builds(BUILDS_QUERY))
results[:not_updated_last_hour] = get_general(NOT_UPDATED_RUNNING_BUILDS_QUERY)
results[:per_runner] = get_per_shared_runner(RUNNING_PER_SHARED_RUNNER_QUERY)
......@@ -91,24 +92,24 @@ module GitLab
0
end
def get_shared_enabled_and_status_grouped(query)
results = { created: { shared_enabled: 0, shared_disabled: 0 },
pending: { shared_enabled: 0, shared_disabled: 0 },
running: { shared_enabled: 0, shared_disabled: 0 } }
def get_builds(query)
results = { pending_builds: [], created_builds: [] }
connection.exec(query).each do |row|
shared_enabled = row["shared_runners_enabled"] == "t"
status = row["status"].to_sym
results[status][:shared_enabled] = row["count"].to_i if shared_enabled
results[status][:shared_disabled] = row["count"].to_i unless shared_enabled
shared_runners = row["shared_runners_enabled"] == "t" ? "yes" : "no"
namespace = row["path"]
value = row["count"].to_i
if row["status"] == "pending"
results[:pending_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
elsif row["status"] == "created"
results[:created_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
end
end
results
rescue PG::UndefinedTable, PG::UndefinedColumn
{ created: { shared_enabled: 0, shared_disabled: 0 },
pending: { shared_enabled: 0, shared_disabled: 0 },
running: { shared_enabled: 0, shared_disabled: 0 } }
{}
end
def shared_runners_ids
......@@ -134,7 +135,9 @@ module GitLab
@results = @collector.run
metrics_total
ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
ci_builds_metrics(@results[:created_builds], "ci_created_builds")
ci_stale_builds_metrics
metrics_per_runner
self
......@@ -146,20 +149,25 @@ module GitLab
private
def metrics_total
@metrics.add("ci_builds_total", @results[:created][:shared_enabled], status: "created",
shared_runners_enabled_projects: 1)
@metrics.add("ci_builds_total", @results[:created][:shared_disabled], status: "created",
shared_runners_enabled_projects: 0)
@metrics.add("ci_builds_total", @results[:pending][:shared_enabled], status: "pending",
shared_runners_enabled_projects: 1)
@metrics.add("ci_builds_total", @results[:pending][:shared_disabled], status: "pending",
shared_runners_enabled_projects: 0)
@metrics.add("ci_builds_total", @results[:running][:shared_enabled], status: "running",
shared_runners_enabled_projects: 1)
@metrics.add("ci_builds_total", @results[:running][:shared_disabled], status: "running",
shared_runners_enabled_projects: 0)
def ci_builds_metrics(results_list, metric_name)
other_value = { "yes" => 0, "no" => 0 }
results_list.each do |metric|
shared_runners = metric[:shared_runners]
namespace = metric[:namespace]
value = metric[:value]
if value < 10
other_value[shared_runners] += value
value = other_value[shared_runners]
namespace = ""
end
@metrics.add(metric_name,
value,
namespace: namespace,
shared_runners: shared_runners)
end
end
def ci_stale_builds_metrics
@metrics.add("ci_builds_stale", @results[:not_updated_last_hour], status: "running", when: "last_hour")
end
......
......@@ -2,14 +2,14 @@ require "spec_helper"
require "gitlab_monitor/database/ci_builds"
describe GitLab::Monitor::Database do
let(:total_builds_query) { "SELECT TOTAL BUILDS" }
let(:builds_query) { "SELECT BUILDS" }
let(:not_updated_running_builds_query) { "SELECT NOT UPDATED RUNNING" }
let(:per_runner_query) { "SELECT ALL RUNNING PER RUNNER %s" }
let(:mirrors_per_runner_query) { "SELECT MIRRORS RUNNING PER RUNNER %s" }
let(:connection) { double("connection") }
before do
stub_const("GitLab::Monitor::Database::CiBuildsCollector::TOTAL_BUILDS_QUERY", total_builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::BUILDS_QUERY", builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::NOT_UPDATED_RUNNING_BUILDS_QUERY",
not_updated_running_builds_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_PER_SHARED_RUNNER_QUERY", per_runner_query)
......@@ -20,13 +20,13 @@ describe GitLab::Monitor::Database do
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:shared_runners_ids)
.and_return([1, 2])
allow(connection).to receive(:exec).with(total_builds_query)
.and_return([{ "shared_runners_enabled" => "f", "status" => "created", "count" => 1 },
{ "shared_runners_enabled" => "t", "status" => "created", "count" => 4 },
{ "shared_runners_enabled" => "f", "status" => "pending", "count" => 2 },
{ "shared_runners_enabled" => "t", "status" => "pending", "count" => 5 },
{ "shared_runners_enabled" => "f", "status" => "running", "count" => 2 },
{ "shared_runners_enabled" => "t", "status" => "running", "count" => 3 }])
allow(connection).to receive(:exec).with(builds_query)
.and_return([{ "shared_runners_enabled" => "f", "status" => "created", "path" => "foo", "count" => 10 },
{ "shared_runners_enabled" => "t", "status" => "pending", "path" => "foo", "count" => 30 },
{ "shared_runners_enabled" => "f", "status" => "created", "path" => "bar", "count" => 20 },
{ "shared_runners_enabled" => "t", "status" => "pending", "path" => "bar", "count" => 50 },
{ "shared_runners_enabled" => "t", "status" => "pending", "path" => "baz", "count" => 1 },
{ "shared_runners_enabled" => "t", "status" => "pending", "path" => "bob", "count" => 2 }])
allow(connection).to receive(:exec).with(not_updated_running_builds_query).and_return([{ "count" => 2 }])
allow(connection).to receive(:exec).with(per_runner_query % "1, 2")
.and_return([{ "runner_id" => 2, "count" => 15 }])
......@@ -40,9 +40,16 @@ describe GitLab::Monitor::Database do
it "executes the query" do
expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 },
per_runner_mirrors: { 1 => 0, 2 => 10 },
created: { shared_enabled: 4, shared_disabled: 1 },
pending: { shared_enabled: 5, shared_disabled: 2 },
running: { shared_enabled: 3, shared_disabled: 2 },
pending_builds: [
{ namespace: "foo", shared_runners: "yes", value: 30 },
{ namespace: "bar", shared_runners: "yes", value: 50 },
{ namespace: "baz", shared_runners: "yes", value: 1 },
{ namespace: "bob", shared_runners: "yes", value: 2 }
],
created_builds: [
{ namespace: "foo", shared_runners: "no", value: 10 },
{ namespace: "bar", shared_runners: "no", value: 20 }
],
not_updated_last_hour: 2)
end
end
......@@ -62,12 +69,11 @@ describe GitLab::Monitor::Database do
prober.probe_db
prober.write_to(writer)
expect(writer.string).to match(/ci_builds_total{status="created",shared_runners_enabled_projects="1"} 4/)
expect(writer.string).to match(/ci_builds_total{status="created",shared_runners_enabled_projects="0"} 1/)
expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="1"} 5/)
expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="0"} 2/)
expect(writer.string).to match(/ci_builds_total{status="running",shared_runners_enabled_projects="1"} 3/)
expect(writer.string).to match(/ci_builds_total{status="running",shared_runners_enabled_projects="0"} 2/)
expect(writer.string).to match(/ci_created_builds{namespace="foo",shared_runners="no"} 10/)
expect(writer.string).to match(/ci_created_builds{namespace="bar",shared_runners="no"} 20/)
expect(writer.string).to match(/ci_pending_builds{namespace="foo",shared_runners="yes"} 30/)
expect(writer.string).to match(/ci_pending_builds{namespace="bar",shared_runners="yes"} 50/)
expect(writer.string).to match(/ci_pending_builds{namespace="",shared_runners="yes"} 3/)
expect(writer.string).to match(/ci_builds_stale{status="running",when="last_hour"} 2/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="0"} 0/)
expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="0"} 5/)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment