Add the ci_repeated_commands_builds metric

parent 473503a6
Pipeline #76423 passed with stage
in 25 seconds
......@@ -141,11 +141,75 @@ module GitLab
SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
SQL
REPEATED_COMMANDS_QUERY_EE =
<<~SQL.freeze
SELECT
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.status,
(COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
COUNT(*) AS count
FROM ci_builds
JOIN projects
ON projects.id = ci_builds.project_id
JOIN namespaces
ON namespaces.id = projects.namespace_id
LEFT JOIN namespace_statistics
ON namespace_statistics.namespace_id = namespaces.id
JOIN application_settings
ON application_settings.id = 1
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('running', 'pending')
GROUP BY
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.status,
ci_builds.commands,
namespaces.shared_runners_minutes_limit,
namespace_statistics.shared_runners_seconds,
application_settings.shared_runners_minutes
HAVING COUNT(*) > %d
SQL
REPEATED_COMMANDS_QUERY_CE =
<<~SQL.freeze
SELECT
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.status,
COUNT(*) AS count
FROM ci_builds
JOIN projects
ON projects.id = ci_builds.project_id
JOIN namespaces
ON namespaces.id = projects.namespace_id
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('running', 'pending')
GROUP BY
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.status,
ci_builds.commands,
HAVING COUNT(*) > %d
SQL
def initialize(opts)
super(opts)
@allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
end
def run
results = {}
results.merge!(builds)
results[:stale_builds] = stale_builds
results[:per_runner] = per_runner_builds
results[:repeated_commands] = repeated_commands
results
end
......@@ -208,9 +272,41 @@ module GitLab
include_ee_fields(values, row)
end
def repeated_commands
results = []
query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
exec_query_with_custom_random_page_cost(query).each do |row|
results << transform_repeated_commands_row_to_values(row)
end
results
rescue PG::UndefinedTable, PG::UndefinedColumn
[]
end
def allowed_repeated_commands_count
@allowed_repeated_commands_count ||= 2
end
def transform_repeated_commands_row_to_values(row)
values = { namespace: row["namespace_id"].to_s,
project: row["project_id"].to_s,
shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
status: row["status"].to_s,
value: row["count"].to_i }
include_has_minutes_field(values, row)
end
def include_ee_fields(values, row)
values.merge!(include_bool_if_row_defined(row, :mirror))
values.merge!(include_bool_if_row_defined(row, :mirror_trigger_builds))
include_has_minutes_field(values, row)
end
def include_has_minutes_field(values, row)
values.merge!(include_bool_if_row_defined(row, :has_minutes))
values
end
......@@ -245,7 +341,10 @@ module GitLab
class CiBuildsProber
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = CiBuildsCollector.new(connection_string: opts[:connection_string])
collector_opts = { connection_string: opts[:connection_string],
allowed_repeated_commands_count: opts[:allowed_repeated_commands_count] }
@collector = CiBuildsCollector.new(collector_opts)
end
def probe_db
......@@ -255,6 +354,7 @@ module GitLab
ci_builds_metrics(@results[:created_builds], "ci_created_builds")
ci_stale_builds_metrics
metrics_per_runner
repeated_commands_metrics
self
rescue PG::ConnectionBad
......@@ -337,6 +437,14 @@ module GitLab
selected_labels = labels.select { |k, _| allowed_labels.include?(k) }.sort.to_h
@metrics.add(metric_name, value, selected_labels)
end
def repeated_commands_metrics
@results[:repeated_commands].each do |metric|
value = metric.delete(:value)
@metrics.add("ci_repeated_commands_builds", value, metric)
end
end
end
end
end
......
......@@ -10,8 +10,11 @@ describe GitLab::Monitor::Database do
let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
let(:connection_pool) { double("connection pool") }
let(:connection) { double("connection") }
let(:allowed_repeated_commands_count) { 5 }
def stub_ee
allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
......@@ -55,6 +58,22 @@ describe GitLab::Monitor::Database do
end
# rubocop:enable Metrics/ParameterLists
# rubocop:disable Metrics/ParameterLists
def repeated_commands_query_row_ee(namespace_id, shared_runners_enabled, project_id, status, has_minutes, count)
row = repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
row["has_minutes"] = has_minutes
row
end
# rubocop:enable Metrics/ParameterLists
def repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
{ "namespace_id" => namespace_id,
"shared_runners_enabled" => shared_runners_enabled,
"project_id" => project_id,
"status" => status,
"count" => count }
end
before do
stub_const("GitLab::Monitor::Database::CiBuildsCollector::SET_RANDOM_PAGE_COST", set_random_page_cost_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::BUILDS_QUERY_EE", builds_query_ee)
......@@ -63,6 +82,8 @@ describe GitLab::Monitor::Database do
stub_const("GitLab::Monitor::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
allow(connection_pool).to receive(:with).and_yield(connection)
......@@ -100,10 +121,28 @@ describe GitLab::Monitor::Database do
per_runner_query_row_ce(2, "f", 2, nil, 3, 5),
per_runner_query_row_ce(2, "f", 3, nil, 3, 5),
per_runner_query_row_ce(3, "f", 4, nil, 3, 5)])
# rubocop:disable Style/FormatString
repeated_commands_query_ee_with_limit = repeated_commands_query_ee % [allowed_repeated_commands_count]
repeated_commands_query_ce_with_limit = repeated_commands_query_ce % [allowed_repeated_commands_count]
# rubocop:enable Style/FormatString
allow(connection).to receive(:exec)
.with(repeated_commands_query_ee_with_limit)
.and_return([repeated_commands_query_row_ee(1, "t", 1, "pending", "t", 10),
repeated_commands_query_row_ee(2, "f", 2, "running", "f", 20),
repeated_commands_query_row_ee(1, "f", 3, "pending", "t", 30),
repeated_commands_query_row_ee(2, "t", 4, "running", "f", 40)])
allow(connection).to receive(:exec)
.with(repeated_commands_query_ce_with_limit)
.and_return([repeated_commands_query_row_ce(1, "t", 1, "pending", 10),
repeated_commands_query_row_ce(2, "f", 2, "running", 20),
repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
end
describe GitLab::Monitor::Database::CiBuildsCollector do
let(:collector) { described_class.new(connection_string: "host=localhost") }
let(:collector) { described_class.new(connection_string: "host=localhost", allowed_repeated_commands_count: allowed_repeated_commands_count) }
let(:expected_stale_builds) { 2 }
shared_examples "data collector" do
......@@ -124,6 +163,10 @@ describe GitLab::Monitor::Database do
it "returns raw stale_builds data" do
expect(subject[:stale_builds]).to eq(expected_stale_builds)
end
it "returns raw repeated_commands data" do
expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
end
end
context "when executed on EE" do
......@@ -144,6 +187,12 @@ describe GitLab::Monitor::Database do
{ runner: "2", shared_runner: "no", namespace: "3", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "yes", value: 5 },
{ runner: "3", shared_runner: "no", namespace: "4", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "no", value: 5 }]
end
let(:expected_repeated_commands) do
[{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", has_minutes: "yes", value: 10 },
{ namespace: "2", project: "2", shared_runners: "no", status: "running", has_minutes: "no", value: 20 },
{ namespace: "1", project: "3", shared_runners: "no", status: "pending", has_minutes: "yes", value: 30 },
{ namespace: "2", project: "4", shared_runners: "yes", status: "running", has_minutes: "no", value: 40 }]
end
before do
stub_ee
......@@ -170,6 +219,12 @@ describe GitLab::Monitor::Database do
{ runner: "2", shared_runner: "no", namespace: "3", scheduled: "no", triggered: "yes", value: 5 },
{ runner: "3", shared_runner: "no", namespace: "4", scheduled: "no", triggered: "yes", value: 5 }]
end
let(:expected_repeated_commands) do
[{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", value: 10 },
{ namespace: "2", project: "2", shared_runners: "no", status: "running", value: 20 },
{ namespace: "1", project: "3", shared_runners: "no", status: "pending", value: 30 },
{ namespace: "2", project: "4", shared_runners: "yes", status: "running", value: 40 }]
end
before do
stub_ce
......@@ -182,7 +237,9 @@ describe GitLab::Monitor::Database do
describe GitLab::Monitor::Database::CiBuildsProber do
let(:writer) { StringIO.new }
let(:prober) do
described_class.new({ connection_string: "host=localhost" },
opts = { connection_string: "host=localhost",
allowed_repeated_commands_count: allowed_repeated_commands_count }
described_class.new(opts,
metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
end
......@@ -216,6 +273,12 @@ describe GitLab::Monitor::Database do
end
end
it "responds with repeated commands Prometheus metrics" do
ci_repeated_commands_builds_lines.each do |expected_line|
expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
end
end
it "responds with stale builds Prometheus metrics" do
expect(subject).to match(/^ci_stale_builds 2$/m)
end
......@@ -254,6 +317,12 @@ describe GitLab::Monitor::Database do
'ci_running_builds\{has_minutes="yes",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="2",scheduled="no",shared_runner="no",triggered="yes"\} 5',
'ci_running_builds\{has_minutes="no",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="3",scheduled="no",shared_runner="no",triggered="yes"\} 5']
end
let(:ci_repeated_commands_builds_lines) do
['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending",has_minutes="yes"\} 10',
'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running",has_minutes="no"\} 20',
'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending",has_minutes="yes"\} 30',
'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running",has_minutes="no"\} 40']
end
let(:namespace_out_of_limit) { 2 }
before do
......@@ -279,6 +348,12 @@ describe GitLab::Monitor::Database do
'ci_running_builds\{namespace="",runner="2",scheduled="no",shared_runner="no",triggered="yes"\} 10',
'ci_running_builds\{namespace="",runner="3",scheduled="no",shared_runner="no",triggered="yes"\} 5']
end
let(:ci_repeated_commands_builds_lines) do
['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending"\} 10',
'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running"\} 20',
'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending"\} 30',
'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running"\} 40']
end
let(:namespace_out_of_limit) { 0 }
before do
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment