Add the ci_repeated_commands_builds metric

lib/gitlab_monitor/database/ci_builds.rb

@@ -141,11 +141,75 @@ module GitLab
             SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
           SQL
 
+        REPEATED_COMMANDS_QUERY_EE =
+          <<~SQL.freeze
+            SELECT
+              projects.namespace_id,
+              projects.shared_runners_enabled,
+              ci_builds.project_id,
+              ci_builds.status,
+              (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
+                 COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
+              COUNT(*) AS count
+            FROM ci_builds
+            JOIN projects
+              ON projects.id = ci_builds.project_id
+            JOIN namespaces
+              ON namespaces.id = projects.namespace_id
+            LEFT JOIN namespace_statistics
+              ON namespace_statistics.namespace_id = namespaces.id
+            JOIN application_settings
+              ON application_settings.id = 1
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status IN ('running', 'pending')
+            GROUP BY
+              projects.namespace_id,
+              projects.shared_runners_enabled,
+              ci_builds.project_id,
+              ci_builds.status,
+              ci_builds.commands,
+              namespaces.shared_runners_minutes_limit,
+              namespace_statistics.shared_runners_seconds,
+              application_settings.shared_runners_minutes
+            HAVING COUNT(*) > %d
+          SQL
+
+        REPEATED_COMMANDS_QUERY_CE =
+          <<~SQL.freeze
+            SELECT
+              projects.namespace_id,
+              projects.shared_runners_enabled,
+              ci_builds.project_id,
+              ci_builds.status,
+              COUNT(*) AS count
+            FROM ci_builds
+            JOIN projects
+              ON projects.id = ci_builds.project_id
+            JOIN namespaces
+              ON namespaces.id = projects.namespace_id
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status IN ('running', 'pending')
+            GROUP BY
+              projects.namespace_id,
+              projects.shared_runners_enabled,
+              ci_builds.project_id,
+              ci_builds.status,
+              ci_builds.commands,
+            HAVING COUNT(*) > %d
+          SQL
+
+        def initialize(opts)
+          super(opts)
+
+          @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
+        end
+
         def run
           results = {}
           results.merge!(builds)
           results[:stale_builds] = stale_builds
           results[:per_runner] = per_runner_builds
+          results[:repeated_commands] = repeated_commands
           results
         end
 
@@ -208,9 +272,41 @@ module GitLab
           include_ee_fields(values, row)
         end
 
+        def repeated_commands
+          results = []
+
+          query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
+          query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
+          exec_query_with_custom_random_page_cost(query).each do |row|
+            results << transform_repeated_commands_row_to_values(row)
+          end
+
+          results
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          []
+        end
+
+        def allowed_repeated_commands_count
+          @allowed_repeated_commands_count ||= 2
+        end
+
+        def transform_repeated_commands_row_to_values(row)
+          values = { namespace: row["namespace_id"].to_s,
+                     project: row["project_id"].to_s,
+                     shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
+                     status: row["status"].to_s,
+                     value: row["count"].to_i }
+
+          include_has_minutes_field(values, row)
+        end
+
         def include_ee_fields(values, row)
           values.merge!(include_bool_if_row_defined(row, :mirror))
           values.merge!(include_bool_if_row_defined(row, :mirror_trigger_builds))
+          include_has_minutes_field(values, row)
+        end
+
+        def include_has_minutes_field(values, row)
           values.merge!(include_bool_if_row_defined(row, :has_minutes))
           values
         end
@@ -245,7 +341,10 @@ module GitLab
       class CiBuildsProber
         def initialize(opts, metrics: PrometheusMetrics.new)
           @metrics = metrics
-          @collector = CiBuildsCollector.new(connection_string: opts[:connection_string])
+
+          collector_opts = { connection_string: opts[:connection_string],
+                             allowed_repeated_commands_count: opts[:allowed_repeated_commands_count] }
+          @collector = CiBuildsCollector.new(collector_opts)
         end
 
         def probe_db
@@ -255,6 +354,7 @@ module GitLab
           ci_builds_metrics(@results[:created_builds], "ci_created_builds")
           ci_stale_builds_metrics
           metrics_per_runner
+          repeated_commands_metrics
 
           self
         rescue PG::ConnectionBad
@@ -337,6 +437,14 @@ module GitLab
           selected_labels = labels.select { |k, _| allowed_labels.include?(k) }.sort.to_h
           @metrics.add(metric_name, value, selected_labels)
         end
+
+        def repeated_commands_metrics
+          @results[:repeated_commands].each do |metric|
+            value = metric.delete(:value)
+
+            @metrics.add("ci_repeated_commands_builds", value, metric)
+          end
+        end
       end
     end
   end

spec/database/ci_builds_spec.rb

@@ -10,8 +10,11 @@ describe GitLab::Monitor::Database do
   let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
   let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
   let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
+  let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
+  let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
   let(:connection_pool) { double("connection pool") }
   let(:connection) { double("connection") }
+  let(:allowed_repeated_commands_count) { 5 }
 
   def stub_ee
     allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
@@ -55,6 +58,22 @@ describe GitLab::Monitor::Database do
   end
   # rubocop:enable Metrics/ParameterLists
 
+  # rubocop:disable Metrics/ParameterLists
+  def repeated_commands_query_row_ee(namespace_id, shared_runners_enabled, project_id, status, has_minutes, count)
+    row = repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
+    row["has_minutes"] = has_minutes
+    row
+  end
+  # rubocop:enable Metrics/ParameterLists
+
+  def repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
+    { "namespace_id" => namespace_id,
+      "shared_runners_enabled" => shared_runners_enabled,
+      "project_id" => project_id,
+      "status" => status,
+      "count" => count }
+  end
+
   before do
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::SET_RANDOM_PAGE_COST", set_random_page_cost_query)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::BUILDS_QUERY_EE", builds_query_ee)
@@ -63,6 +82,8 @@ describe GitLab::Monitor::Database do
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
+    stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
+    stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
 
     allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
     allow(connection_pool).to receive(:with).and_yield(connection)
@@ -100,10 +121,28 @@ describe GitLab::Monitor::Database do
                    per_runner_query_row_ce(2, "f", 2, nil, 3, 5),
                    per_runner_query_row_ce(2, "f", 3, nil, 3, 5),
                    per_runner_query_row_ce(3, "f", 4, nil, 3, 5)])
+
+    # rubocop:disable Style/FormatString
+    repeated_commands_query_ee_with_limit = repeated_commands_query_ee % [allowed_repeated_commands_count]
+    repeated_commands_query_ce_with_limit = repeated_commands_query_ce % [allowed_repeated_commands_count]
+    # rubocop:enable Style/FormatString
+
+    allow(connection).to receive(:exec)
+      .with(repeated_commands_query_ee_with_limit)
+      .and_return([repeated_commands_query_row_ee(1, "t", 1, "pending", "t", 10),
+                   repeated_commands_query_row_ee(2, "f", 2, "running", "f", 20),
+                   repeated_commands_query_row_ee(1, "f", 3, "pending", "t", 30),
+                   repeated_commands_query_row_ee(2, "t", 4, "running", "f", 40)])
+    allow(connection).to receive(:exec)
+      .with(repeated_commands_query_ce_with_limit)
+      .and_return([repeated_commands_query_row_ce(1, "t", 1, "pending", 10),
+                   repeated_commands_query_row_ce(2, "f", 2, "running", 20),
+                   repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
+                   repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
   end
 
   describe GitLab::Monitor::Database::CiBuildsCollector do
-    let(:collector) { described_class.new(connection_string: "host=localhost") }
+    let(:collector) { described_class.new(connection_string: "host=localhost", allowed_repeated_commands_count: allowed_repeated_commands_count) }
     let(:expected_stale_builds) { 2 }
 
     shared_examples "data collector" do
@@ -124,6 +163,10 @@ describe GitLab::Monitor::Database do
       it "returns raw stale_builds data" do
         expect(subject[:stale_builds]).to eq(expected_stale_builds)
       end
+
+      it "returns raw repeated_commands data" do
+        expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
+      end
     end
 
     context "when executed on EE" do
@@ -144,6 +187,12 @@ describe GitLab::Monitor::Database do
          { runner: "2", shared_runner: "no", namespace: "3", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "yes", value: 5 },
          { runner: "3", shared_runner: "no", namespace: "4", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "no", value: 5 }]
       end
+      let(:expected_repeated_commands) do
+        [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", has_minutes: "yes", value: 10 },
+         { namespace: "2", project: "2", shared_runners: "no", status: "running", has_minutes: "no", value: 20 },
+         { namespace: "1", project: "3", shared_runners: "no", status: "pending", has_minutes: "yes", value: 30 },
+         { namespace: "2", project: "4", shared_runners: "yes", status: "running", has_minutes: "no", value: 40 }]
+      end
 
       before do
         stub_ee
@@ -170,6 +219,12 @@ describe GitLab::Monitor::Database do
          { runner: "2", shared_runner: "no", namespace: "3", scheduled: "no", triggered: "yes", value: 5 },
          { runner: "3", shared_runner: "no", namespace: "4", scheduled: "no", triggered: "yes", value: 5 }]
       end
+      let(:expected_repeated_commands) do
+        [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", value: 10 },
+         { namespace: "2", project: "2", shared_runners: "no", status: "running", value: 20 },
+         { namespace: "1", project: "3", shared_runners: "no", status: "pending", value: 30 },
+         { namespace: "2", project: "4", shared_runners: "yes", status: "running", value: 40 }]
+      end
 
       before do
         stub_ce
@@ -182,7 +237,9 @@ describe GitLab::Monitor::Database do
   describe GitLab::Monitor::Database::CiBuildsProber do
     let(:writer) { StringIO.new }
     let(:prober) do
-      described_class.new({ connection_string: "host=localhost" },
+      opts = { connection_string: "host=localhost",
+               allowed_repeated_commands_count: allowed_repeated_commands_count }
+      described_class.new(opts,
                           metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
     end
 
@@ -216,6 +273,12 @@ describe GitLab::Monitor::Database do
           end
         end
 
+        it "responds with repeated commands Prometheus metrics" do
+          ci_repeated_commands_builds_lines.each do |expected_line|
+            expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
+          end
+        end
+
         it "responds with stale builds Prometheus metrics" do
           expect(subject).to match(/^ci_stale_builds 2$/m)
         end
@@ -254,6 +317,12 @@ describe GitLab::Monitor::Database do
          'ci_running_builds\{has_minutes="yes",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="2",scheduled="no",shared_runner="no",triggered="yes"\} 5',
          'ci_running_builds\{has_minutes="no",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="3",scheduled="no",shared_runner="no",triggered="yes"\} 5']
       end
+      let(:ci_repeated_commands_builds_lines) do
+        ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending",has_minutes="yes"\} 10',
+         'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running",has_minutes="no"\} 20',
+         'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending",has_minutes="yes"\} 30',
+         'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running",has_minutes="no"\} 40']
+      end
       let(:namespace_out_of_limit) { 2 }
 
       before do
@@ -279,6 +348,12 @@ describe GitLab::Monitor::Database do
          'ci_running_builds\{namespace="",runner="2",scheduled="no",shared_runner="no",triggered="yes"\} 10',
          'ci_running_builds\{namespace="",runner="3",scheduled="no",shared_runner="no",triggered="yes"\} 5']
       end
+      let(:ci_repeated_commands_builds_lines) do
+        ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending"\} 10',
+         'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running"\} 20',
+         'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending"\} 30',
+         'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running"\} 40']
+      end
       let(:namespace_out_of_limit) { 0 }
 
       before do