Skip to content
GitLab
Commit 7c56ce30 authored by Ahmad Sherif's avatar Ahmad Sherif
Browse files

Merge branch 'remove-ci-repeated-commands-metric' into 'master' 

Remove ci_repeated_commands_builds metric

Closes #77

See merge request gitlab-org/gitlab-exporter!136
parents 9b1098d2 6b07aa8e
Pipeline #190699 passed with stage
in 2 minutes and 27 seconds
Hide whitespace changes
Inline Side-by-side
config/gitlab-exporter.yml.example
View file @ 7c56ce30
......@@ -46,7 +46,6 @@ probes:
<<: *db_common
opts:
<<: *db_common_opts
allowed_repeated_commands_count: 2
created_builds_counting_disabled: true
unarchived_traces_offset_minutes: 1440
tuple_stats:
......
lib/gitlab_exporter/database/ci_builds.rb
View file @ 7c56ce30
......@@ -147,100 +147,6 @@ module GitLab
SELECT COUNT(*) FROM licenses
SQL
REPEATED_COMMANDS_QUERY_EE =
<<~SQL.freeze
SELECT
subquery.namespace_id,
subquery.shared_runners_enabled,
subquery.project_id,
subquery.status,
subquery.has_minutes,
MAX(subquery.count) as count
FROM (
SELECT
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.commit_id,
ci_builds.status,
(COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
COUNT(*) AS count
FROM ci_builds
JOIN projects
ON projects.id = ci_builds.project_id
JOIN namespaces
ON namespaces.id = projects.namespace_id
LEFT JOIN namespace_statistics
ON namespace_statistics.namespace_id = namespaces.id
JOIN application_settings ON (TRUE)
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('running', 'pending')
-- The created_at filter has been introduced for performance reasons only
AND ci_builds.created_at > NOW() - INTERVAL '7 days'
GROUP BY
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.commit_id,
ci_builds.status,
ci_builds.commands,
namespaces.shared_runners_minutes_limit,
namespace_statistics.shared_runners_seconds,
application_settings.shared_runners_minutes
HAVING COUNT(*) > %d
) AS subquery
GROUP BY
subquery.namespace_id,
subquery.shared_runners_enabled,
subquery.project_id,
subquery.commit_id,
subquery.status,
subquery.has_minutes
SQL
REPEATED_COMMANDS_QUERY_CE =
<<~SQL.freeze
SELECT
subquery.namespace_id,
subquery.shared_runners_enabled,
subquery.project_id,
subquery.status,
MAX(subquery.count) as count
FROM (
SELECT
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.commit_id,
ci_builds.status,
COUNT(*) AS count
FROM ci_builds
JOIN projects
ON projects.id = ci_builds.project_id
JOIN namespaces
ON namespaces.id = projects.namespace_id
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('running', 'pending')
-- The created_at filter has been introduced for performance reasons only
AND ci_builds.created_at > NOW() - INTERVAL '7 days'
GROUP BY
projects.namespace_id,
projects.shared_runners_enabled,
ci_builds.project_id,
ci_builds.commit_id,
ci_builds.status,
ci_builds.commands
HAVING COUNT(*) > %d
) AS subquery
GROUP BY
subquery.namespace_id,
subquery.shared_runners_enabled,
subquery.project_id,
subquery.commit_id,
subquery.status
SQL
UNARCHIVED_TRACES_QUERY =
<<~SQL.freeze
SELECT
......@@ -265,7 +171,6 @@ module GitLab
def initialize(opts, logger: nil)
super(opts, logger: logger)
@allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
@created_builds_counting_disabled = opts[:created_builds_counting_disabled]
@unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
end
......@@ -276,7 +181,6 @@ module GitLab
results[:pending_builds] = builds(STATUS_PENDING)
results[:stale_builds] = stale_builds
results[:per_runner] = per_runner_builds
results[:repeated_commands] = repeated_commands
results[:unarchived_traces] = unarchived_traces
results
end
......@@ -335,34 +239,6 @@ module GitLab
include_ee_fields(values, row)
end
def repeated_commands
results = []
query = ee? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
exec_query_with_custom_random_page_cost(query).each do |row|
results << transform_repeated_commands_row_to_values(row)
end
results
rescue PG::UndefinedTable, PG::UndefinedColumn
[]
end
def allowed_repeated_commands_count
@allowed_repeated_commands_count ||= 2
end
def transform_repeated_commands_row_to_values(row)
values = { namespace: row["namespace_id"].to_s,
project: row["project_id"].to_s,
shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
status: row["status"].to_s,
value: row["count"].to_i }
include_has_minutes_field(values, row)
end
def unarchived_traces
time = Time.now - (unarchived_traces_offset_minutes * 60)
query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString
......@@ -422,7 +298,6 @@ module GitLab
@metrics = metrics
collector_opts = { connection_string: opts[:connection_string],
allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
created_builds_counting_disabled: opts[:created_builds_counting_disabled],
unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
@collector = CiBuildsCollector.new(collector_opts, logger: logger)
......@@ -435,7 +310,6 @@ module GitLab
ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
ci_stale_builds_metrics
metrics_per_runner
repeated_commands_metrics
unarchived_traces_metrics
self
......@@ -520,14 +394,6 @@ module GitLab
@metrics.add(metric_name, value.to_f, selected_labels)
end
def repeated_commands_metrics
@results[:repeated_commands].each do |metric|
value = metric.delete(:value)
@metrics.add("ci_repeated_commands_builds", value.to_f, metric)
end
end
def unarchived_traces_metrics
@metrics.add("ci_unarchived_traces", @results[:unarchived_traces].to_f)
end
......
spec/database/ci_builds_spec.rb
View file @ 7c56ce30
......@@ -10,12 +10,9 @@ describe GitLab::Exporter::Database do
let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
let(:ee_check_query) { "SELECT COUNT(*) FROM licenses" }
let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
let(:connection_pool) { double("connection pool") }
let(:connection) { double("connection") }
let(:allowed_repeated_commands_count) { 5 }
let(:created_builds_counting_disabled) { true }
let(:time_now) { Time.new(2019, 4, 9, 6, 30, 0) }
let(:unarchived_traces_query_time) { "2019-04-09 05:30:00" }
......@@ -63,22 +60,6 @@ describe GitLab::Exporter::Database do
end
# rubocop:enable Metrics/ParameterLists
# rubocop:disable Metrics/ParameterLists
def repeated_commands_query_row_ee(namespace_id, shared_runners_enabled, project_id, status, has_minutes, count)
row = repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
row["has_minutes"] = has_minutes
row
end
# rubocop:enable Metrics/ParameterLists
def repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
{ "namespace_id" => namespace_id,
"shared_runners_enabled" => shared_runners_enabled,
"project_id" => project_id,
"status" => status,
"count" => count }
end
before do
stub_const("GitLab::Exporter::Database::CiBuildsCollector::SET_RANDOM_PAGE_COST", set_random_page_cost_query)
stub_const("GitLab::Exporter::Database::CiBuildsCollector::BUILDS_QUERY_EE", builds_query_ee)
......@@ -87,8 +68,6 @@ describe GitLab::Exporter::Database do
stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
stub_const("GitLab::Exporter::Database::CiBuildsCollector::EE_CHECK_QUERY", ee_check_query)
stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
stub_const("GitLab::Exporter::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)
allow_any_instance_of(GitLab::Exporter::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
......@@ -130,24 +109,6 @@ describe GitLab::Exporter::Database do
per_runner_query_row_ce(2, "project_type", 3, nil, 3, 5),
per_runner_query_row_ce(3, "project_type", 4, nil, 3, 5)])
# rubocop:disable Style/FormatString
repeated_commands_query_ee_with_limit = repeated_commands_query_ee % [allowed_repeated_commands_count]
repeated_commands_query_ce_with_limit = repeated_commands_query_ce % [allowed_repeated_commands_count]
# rubocop:enable Style/FormatString
allow(connection).to receive(:exec)
.with(repeated_commands_query_ee_with_limit)
.and_return([repeated_commands_query_row_ee(1, "t", 1, "pending", "t", 10),
repeated_commands_query_row_ee(2, "f", 2, "running", "f", 20),
repeated_commands_query_row_ee(1, "f", 3, "pending", "t", 30),
repeated_commands_query_row_ee(2, "t", 4, "running", "f", 40)])
allow(connection).to receive(:exec)
.with(repeated_commands_query_ce_with_limit)
.and_return([repeated_commands_query_row_ce(1, "t", 1, "pending", 10),
repeated_commands_query_row_ce(2, "f", 2, "running", 20),
repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
unarchived_traces_query_with_time = unarchived_traces_query % [unarchived_traces_query_time] # rubocop:disable Style/FormatString
allow(connection).to receive(:exec).with(unarchived_traces_query_with_time).and_return([{ "count" => 10 }])
......@@ -156,7 +117,6 @@ describe GitLab::Exporter::Database do
describe GitLab::Exporter::Database::CiBuildsCollector do
let(:collector) do
described_class.new(connection_string: "host=localhost",
allowed_repeated_commands_count: allowed_repeated_commands_count,
created_builds_counting_disabled: created_builds_counting_disabled,
unarchived_traces_offset_minutes: unarchived_traces_offset_minutes)
end
......@@ -195,10 +155,6 @@ describe GitLab::Exporter::Database do
expect(subject[:stale_builds]).to eq(expected_stale_builds)
end
it "returns raw repeated_commands data" do
expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
end
it "returns raw unarchived_traces data" do
expect(subject[:unarchived_traces]).to eq(expected_unarchived_traces)
end
......@@ -222,12 +178,6 @@ describe GitLab::Exporter::Database do
{ runner: "2", runner_type: "project_type", namespace: "3", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "yes", value: 5.0 },
{ runner: "3", runner_type: "project_type", namespace: "4", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "no", value: 5.0 }]
end
let(:expected_repeated_commands) do
[{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", has_minutes: "yes", value: 10.0 },
{ namespace: "2", project: "2", shared_runners: "no", status: "running", has_minutes: "no", value: 20.0 },
{ namespace: "1", project: "3", shared_runners: "no", status: "pending", has_minutes: "yes", value: 30.0 },
{ namespace: "2", project: "4", shared_runners: "yes", status: "running", has_minutes: "no", value: 40.0 }]
end
before do
stub_ee
......@@ -254,12 +204,6 @@ describe GitLab::Exporter::Database do
{ runner: "2", runner_type: "project_type", namespace: "3", scheduled: "no", triggered: "yes", value: 5 },
{ runner: "3", runner_type: "project_type", namespace: "4", scheduled: "no", triggered: "yes", value: 5 }]
end
let(:expected_repeated_commands) do
[{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", value: 10 },
{ namespace: "2", project: "2", shared_runners: "no", status: "running", value: 20 },
{ namespace: "1", project: "3", shared_runners: "no", status: "pending", value: 30 },
{ namespace: "2", project: "4", shared_runners: "yes", status: "running", value: 40 }]
end
before do
stub_ce
......@@ -273,7 +217,6 @@ describe GitLab::Exporter::Database do
let(:writer) { StringIO.new }
let(:prober) do
opts = { connection_string: "host=localhost",
allowed_repeated_commands_count: allowed_repeated_commands_count,
created_builds_counting_disabled: created_builds_counting_disabled,
unarchived_traces_offset_minutes: unarchived_traces_offset_minutes }
described_class.new(opts,
......@@ -324,12 +267,6 @@ describe GitLab::Exporter::Database do
end
end
it "responds with repeated commands Prometheus metrics" do
ci_repeated_commands_builds_lines.each do |expected_line|
expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
end
end
it "responds with stale builds Prometheus metrics" do
expect(subject).to match(/^ci_stale_builds 2.0$/m)
end
......@@ -372,12 +309,6 @@ describe GitLab::Exporter::Database do
'ci_running_builds\{has_minutes="yes",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="2",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0',
'ci_running_builds\{has_minutes="no",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="3",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0']
end
let(:ci_repeated_commands_builds_lines) do
['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending",has_minutes="yes"\} 10.0',
'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running",has_minutes="no"\} 20.0',
'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending",has_minutes="yes"\} 30.0',
'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running",has_minutes="no"\} 40.0']
end
let(:namespace_out_of_limit) { 2 }
before do
......@@ -403,12 +334,6 @@ describe GitLab::Exporter::Database do
'ci_running_builds\{namespace="",runner="2",runner_type="project_type",scheduled="no",triggered="yes"\} 10.0',
'ci_running_builds\{namespace="",runner="3",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0']
end
let(:ci_repeated_commands_builds_lines) do
['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending"\} 10.0',
'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running"\} 20.0',
'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending"\} 30.0',
'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running"\} 40.0']
end
let(:namespace_out_of_limit) { 0 }
before do
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment