Merge branch 'dont-count-created-jobs' into 'master'

Make it possible to disable gathering number of created jobs

See merge request gitlab-org/gitlab-monitor!88

config/gitlab-monitor.yml.example

@@ -46,6 +46,7 @@ probes:
       opts:
         <<: *db_common_opts
         allowed_repeated_commands_count: 2
+        created_builds_counting_disabled: true
     tuple_stats:
       class_name: Database::TuplesProber
       <<: *db_common

lib/gitlab_monitor/database/ci_builds.rb

@@ -24,7 +24,7 @@ module GitLab
             JOIN application_settings
               ON application_settings.id = 1
             WHERE ci_builds.type = 'Ci::Build'
-              AND ci_builds.status IN ('created', 'pending')
+              AND ci_builds.status = '%s'
               AND projects.pending_delete = 'f'
             GROUP BY
               projects.namespace_id,
@@ -46,7 +46,7 @@ module GitLab
             JOIN projects
               ON projects.id = ci_builds.project_id
             WHERE ci_builds.type = 'Ci::Build'
-              AND ci_builds.status IN ('created', 'pending')
+              AND ci_builds.status = '%s'
               AND projects.pending_delete = 'f'
             GROUP BY
               projects.namespace_id,
@@ -232,15 +232,20 @@ module GitLab
               subquery.status
           SQL
 
+        STATUS_CREATED = "created".freeze
+        STATUS_PENDING = "pending".freeze
+
         def initialize(opts)
           super(opts)
 
           @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
+          @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
         end
 
         def run
           results = {}
-          results.merge!(builds)
+          results[:created_builds] = builds(STATUS_CREATED) unless @created_builds_counting_disabled
+          results[:pending_builds] = builds(STATUS_PENDING)
           results[:stale_builds] = stale_builds
           results[:per_runner] = per_runner_builds
           results[:repeated_commands] = repeated_commands
@@ -249,18 +254,13 @@ module GitLab
 
         private
 
-        def builds
-          results = { pending_builds: [], created_builds: [] }
+        def builds(status)
+          results = []
 
           query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
+          query = query % [status] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
-            row_data = transform_builds_row_to_values(row)
-
-            if row["status"] == "pending"
-              results[:pending_builds].push(row_data)
-            elsif row["status"] == "created"
-              results[:created_builds].push(row_data)
-            end
+            results << transform_builds_row_to_values(row)
           end
 
           results
@@ -377,15 +377,16 @@ module GitLab
           @metrics = metrics
 
           collector_opts = { connection_string: opts[:connection_string],
-                             allowed_repeated_commands_count: opts[:allowed_repeated_commands_count] }
+                             allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
+                             created_builds_counting_disabled: opts[:created_builds_counting_disabled] }
           @collector = CiBuildsCollector.new(collector_opts)
         end
 
         def probe_db
           @results = @collector.run
 
+          ci_builds_metrics(@results[:created_builds], "ci_created_builds") if @results[:created_builds]
           ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
-          ci_builds_metrics(@results[:created_builds], "ci_created_builds")
           ci_stale_builds_metrics
           metrics_per_runner
           repeated_commands_metrics

spec/database/ci_builds_spec.rb

@@ -15,6 +15,7 @@ describe GitLab::Monitor::Database do
   let(:connection_pool) { double("connection pool") }
   let(:connection) { double("connection") }
   let(:allowed_repeated_commands_count) { 5 }
+  let(:created_builds_counting_disabled) { true }
 
   def stub_ee
     allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
@@ -142,7 +143,11 @@ describe GitLab::Monitor::Database do
   end
 
   describe GitLab::Monitor::Database::CiBuildsCollector do
-    let(:collector) { described_class.new(connection_string: "host=localhost", allowed_repeated_commands_count: allowed_repeated_commands_count) }
+    let(:collector) do
+      described_class.new(connection_string: "host=localhost",
+                          allowed_repeated_commands_count: allowed_repeated_commands_count,
+                          created_builds_counting_disabled: created_builds_counting_disabled)
+    end
     let(:expected_stale_builds) { 2 }
 
     shared_examples "data collector" do
@@ -156,9 +161,22 @@ describe GitLab::Monitor::Database do
         expect(subject[:pending_builds]).to include(*expected_pending_builds)
       end
 
+      context "when created_builds_counting_disabled is set to false" do
+        let(:created_builds_counting_disabled) { false }
+
         it "returns raw created_builds data" do
+          expect(subject).to have_key(:created_builds)
           expect(subject[:created_builds]).to include(*expected_created_builds)
         end
+      end
+
+      context "when created_builds_counting_disabled is set to true" do
+        let(:created_builds_counting_disabled) { true }
+
+        it "doesn't return raw created_builds data" do
+          expect(subject).not_to have_key(:created_builds)
+        end
+      end
 
       it "returns raw stale_builds data" do
         expect(subject[:stale_builds]).to eq(expected_stale_builds)
@@ -238,7 +256,8 @@ describe GitLab::Monitor::Database do
     let(:writer) { StringIO.new }
     let(:prober) do
       opts = { connection_string: "host=localhost",
-               allowed_repeated_commands_count: allowed_repeated_commands_count }
+               allowed_repeated_commands_count: allowed_repeated_commands_count,
+               created_builds_counting_disabled: created_builds_counting_disabled }
       described_class.new(opts,
                           metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
     end
@@ -255,11 +274,25 @@ describe GitLab::Monitor::Database do
       end
 
       context "when PG exceptions aren't raised" do
+        context "when created_builds_counting_disabled is set to false" do
+          let(:created_builds_counting_disabled) { false }
+
           it "responds with created builds Prometheus metrics" do
             ci_created_builds_expected_lines.each do |expected_line|
               expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
             end
           end
+        end
+
+        context "when created_builds_counting_disabled is set to true" do
+          let(:created_builds_counting_disabled) { true }
+
+          it "doesn't respond with created builds Prometheus metrics" do
+            ci_created_builds_expected_lines.each do |expected_line|
+              expect(subject).not_to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
+            end
+          end
+        end
 
         it "responds with pending builds Prometheus metrics" do
           ci_pending_builds_expected_lines.each do |expected_line|