Commit 37d63449 authored by Ahmad Sherif's avatar Ahmad Sherif

Merge branch 'dont-count-created-jobs' into 'master'

Make it possible to disable gathering number of created jobs

See merge request gitlab-org/gitlab-monitor!88
parents 01e38af8 d466a6f1
...@@ -46,6 +46,7 @@ probes: ...@@ -46,6 +46,7 @@ probes:
opts: opts:
<<: *db_common_opts <<: *db_common_opts
allowed_repeated_commands_count: 2 allowed_repeated_commands_count: 2
created_builds_counting_disabled: true
tuple_stats: tuple_stats:
class_name: Database::TuplesProber class_name: Database::TuplesProber
<<: *db_common <<: *db_common
...@@ -101,4 +102,4 @@ probes: ...@@ -101,4 +102,4 @@ probes:
rows_count: rows_count:
class_name: Database::RowCountProber class_name: Database::RowCountProber
<<: *db_common <<: *db_common
...@@ -24,7 +24,7 @@ module GitLab ...@@ -24,7 +24,7 @@ module GitLab
JOIN application_settings JOIN application_settings
ON application_settings.id = 1 ON application_settings.id = 1
WHERE ci_builds.type = 'Ci::Build' WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('created', 'pending') AND ci_builds.status = '%s'
AND projects.pending_delete = 'f' AND projects.pending_delete = 'f'
GROUP BY GROUP BY
projects.namespace_id, projects.namespace_id,
...@@ -46,7 +46,7 @@ module GitLab ...@@ -46,7 +46,7 @@ module GitLab
JOIN projects JOIN projects
ON projects.id = ci_builds.project_id ON projects.id = ci_builds.project_id
WHERE ci_builds.type = 'Ci::Build' WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('created', 'pending') AND ci_builds.status = '%s'
AND projects.pending_delete = 'f' AND projects.pending_delete = 'f'
GROUP BY GROUP BY
projects.namespace_id, projects.namespace_id,
...@@ -232,15 +232,20 @@ module GitLab ...@@ -232,15 +232,20 @@ module GitLab
subquery.status subquery.status
SQL SQL
STATUS_CREATED = "created".freeze
STATUS_PENDING = "pending".freeze
def initialize(opts) def initialize(opts)
super(opts) super(opts)
@allowed_repeated_commands_count = opts[:allowed_repeated_commands_count] @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
@created_builds_counting_disabled = opts[:created_builds_counting_disabled]
end end
def run def run
results = {} results = {}
results.merge!(builds) results[:created_builds] = builds(STATUS_CREATED) unless @created_builds_counting_disabled
results[:pending_builds] = builds(STATUS_PENDING)
results[:stale_builds] = stale_builds results[:stale_builds] = stale_builds
results[:per_runner] = per_runner_builds results[:per_runner] = per_runner_builds
results[:repeated_commands] = repeated_commands results[:repeated_commands] = repeated_commands
...@@ -249,18 +254,13 @@ module GitLab ...@@ -249,18 +254,13 @@ module GitLab
private private
def builds def builds(status)
results = { pending_builds: [], created_builds: [] } results = []
query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
query = query % [status] # rubocop:disable Style/FormatString
exec_query_with_custom_random_page_cost(query).each do |row| exec_query_with_custom_random_page_cost(query).each do |row|
row_data = transform_builds_row_to_values(row) results << transform_builds_row_to_values(row)
if row["status"] == "pending"
results[:pending_builds].push(row_data)
elsif row["status"] == "created"
results[:created_builds].push(row_data)
end
end end
results results
...@@ -377,15 +377,16 @@ module GitLab ...@@ -377,15 +377,16 @@ module GitLab
@metrics = metrics @metrics = metrics
collector_opts = { connection_string: opts[:connection_string], collector_opts = { connection_string: opts[:connection_string],
allowed_repeated_commands_count: opts[:allowed_repeated_commands_count] } allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
created_builds_counting_disabled: opts[:created_builds_counting_disabled] }
@collector = CiBuildsCollector.new(collector_opts) @collector = CiBuildsCollector.new(collector_opts)
end end
def probe_db def probe_db
@results = @collector.run @results = @collector.run
ci_builds_metrics(@results[:created_builds], "ci_created_builds") if @results[:created_builds]
ci_builds_metrics(@results[:pending_builds], "ci_pending_builds") ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
ci_builds_metrics(@results[:created_builds], "ci_created_builds")
ci_stale_builds_metrics ci_stale_builds_metrics
metrics_per_runner metrics_per_runner
repeated_commands_metrics repeated_commands_metrics
......
...@@ -15,6 +15,7 @@ describe GitLab::Monitor::Database do ...@@ -15,6 +15,7 @@ describe GitLab::Monitor::Database do
let(:connection_pool) { double("connection pool") } let(:connection_pool) { double("connection pool") }
let(:connection) { double("connection") } let(:connection) { double("connection") }
let(:allowed_repeated_commands_count) { 5 } let(:allowed_repeated_commands_count) { 5 }
let(:created_builds_counting_disabled) { true }
def stub_ee def stub_ee
allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }]) allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
...@@ -142,7 +143,11 @@ describe GitLab::Monitor::Database do ...@@ -142,7 +143,11 @@ describe GitLab::Monitor::Database do
end end
describe GitLab::Monitor::Database::CiBuildsCollector do describe GitLab::Monitor::Database::CiBuildsCollector do
let(:collector) { described_class.new(connection_string: "host=localhost", allowed_repeated_commands_count: allowed_repeated_commands_count) } let(:collector) do
described_class.new(connection_string: "host=localhost",
allowed_repeated_commands_count: allowed_repeated_commands_count,
created_builds_counting_disabled: created_builds_counting_disabled)
end
let(:expected_stale_builds) { 2 } let(:expected_stale_builds) { 2 }
shared_examples "data collector" do shared_examples "data collector" do
...@@ -156,8 +161,21 @@ describe GitLab::Monitor::Database do ...@@ -156,8 +161,21 @@ describe GitLab::Monitor::Database do
expect(subject[:pending_builds]).to include(*expected_pending_builds) expect(subject[:pending_builds]).to include(*expected_pending_builds)
end end
it "returns raw created_builds data" do context "when created_builds_counting_disabled is set to false" do
expect(subject[:created_builds]).to include(*expected_created_builds) let(:created_builds_counting_disabled) { false }
it "returns raw created_builds data" do
expect(subject).to have_key(:created_builds)
expect(subject[:created_builds]).to include(*expected_created_builds)
end
end
context "when created_builds_counting_disabled is set to true" do
let(:created_builds_counting_disabled) { true }
it "doesn't return raw created_builds data" do
expect(subject).not_to have_key(:created_builds)
end
end end
it "returns raw stale_builds data" do it "returns raw stale_builds data" do
...@@ -238,7 +256,8 @@ describe GitLab::Monitor::Database do ...@@ -238,7 +256,8 @@ describe GitLab::Monitor::Database do
let(:writer) { StringIO.new } let(:writer) { StringIO.new }
let(:prober) do let(:prober) do
opts = { connection_string: "host=localhost", opts = { connection_string: "host=localhost",
allowed_repeated_commands_count: allowed_repeated_commands_count } allowed_repeated_commands_count: allowed_repeated_commands_count,
created_builds_counting_disabled: created_builds_counting_disabled }
described_class.new(opts, described_class.new(opts,
metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false)) metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
end end
...@@ -255,9 +274,23 @@ describe GitLab::Monitor::Database do ...@@ -255,9 +274,23 @@ describe GitLab::Monitor::Database do
end end
context "when PG exceptions aren't raised" do context "when PG exceptions aren't raised" do
it "responds with created builds Prometheus metrics" do context "when created_builds_counting_disabled is set to false" do
ci_created_builds_expected_lines.each do |expected_line| let(:created_builds_counting_disabled) { false }
expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
it "responds with created builds Prometheus metrics" do
ci_created_builds_expected_lines.each do |expected_line|
expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
end
end
end
context "when created_builds_counting_disabled is set to true" do
let(:created_builds_counting_disabled) { true }
it "doesn't respond with created builds Prometheus metrics" do
ci_created_builds_expected_lines.each do |expected_line|
expect(subject).not_to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
end
end end
end end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment