Refactor total builds metrics

* Remove `_total` from metric name, this is a gauge.
* Split `running` and `created` builds into separate metrics.
* Add `namespace.path` as new metric label.
* Refactor `metrics_total` function to support new metrics and labeling.
* Reduce cardinality for `pending` and `created` builds by ignoring
  values less than 10.
* Split stale builds metrics to separate function.
* Update specs.

lib/gitlab_monitor/database/ci_builds.rb

@@ -3,19 +3,23 @@ module GitLab
     module Database
       # A helper class to collect blocked queries
       class CiBuildsCollector < Base
-        TOTAL_BUILDS_QUERY =
+        BUILDS_QUERY =
           "SELECT " \
-          "  projects.shared_runners_enabled, " \
+          "  namespaces.path, " \
           "  ci_builds.status, " \
+          "  projects.shared_runners_enabled, " \
           "  COUNT(*) AS count " \
           "  FROM ci_builds " \
           "  JOIN projects " \
           "    ON projects.id = ci_builds.gl_project_id " \
+          "  JOIN namespaces " \
+          "    ON namespaces.id = projects.namespace_id " \
           "  WHERE ci_builds.type = 'Ci::Build' " \
-          "    AND ci_builds.status IN ('created', 'pending', 'running') " \
+          "    AND ci_builds.status IN ('created', 'pending') " \
           "  GROUP BY " \
-          "    projects.shared_runners_enabled, " \
-          "    ci_builds.status".freeze
+          "    namespaces.path, " \
+          "    ci_builds.status, " \
+          "    projects.shared_runners_enabled".freeze
 
         NOT_UPDATED_RUNNING_BUILDS_QUERY =
           "SELECT " \
@@ -53,10 +57,7 @@ module GitLab
         def run
           results = {}
 
-          total = get_shared_enabled_and_status_grouped(TOTAL_BUILDS_QUERY)
-          results[:created] = total[:created]
-          results[:pending] = total[:pending]
-          results[:running] = total[:running]
+          results.merge!(get_builds(BUILDS_QUERY))
 
           results[:not_updated_last_hour] = get_general(NOT_UPDATED_RUNNING_BUILDS_QUERY)
           results[:per_runner] = get_per_shared_runner(RUNNING_PER_SHARED_RUNNER_QUERY)
@@ -91,24 +92,24 @@ module GitLab
           0
         end
 
-        def get_shared_enabled_and_status_grouped(query)
-          results = { created: { shared_enabled: 0, shared_disabled: 0 },
-                      pending: { shared_enabled: 0, shared_disabled: 0 },
-                      running: { shared_enabled: 0, shared_disabled: 0 } }
+        def get_builds(query)
+          results = { pending_builds: [], created_builds: [] }
 
           connection.exec(query).each do |row|
-            shared_enabled = row["shared_runners_enabled"] == "t"
-            status = row["status"].to_sym
-
-            results[status][:shared_enabled] = row["count"].to_i if shared_enabled
-            results[status][:shared_disabled] = row["count"].to_i unless shared_enabled
+            shared_runners = row["shared_runners_enabled"] == "t" ? "yes" : "no"
+            namespace = row["path"]
+            value = row["count"].to_i
+
+            if row["status"] == "pending"
+              results[:pending_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
+            elsif row["status"] == "created"
+              results[:created_builds].push(namespace: namespace, shared_runners: shared_runners, value: value)
+            end
           end
 
           results
         rescue PG::UndefinedTable, PG::UndefinedColumn
-          { created: { shared_enabled: 0, shared_disabled: 0 },
-            pending: { shared_enabled: 0, shared_disabled: 0 },
-            running: { shared_enabled: 0, shared_disabled: 0 } }
+          {}
         end
 
         def shared_runners_ids
@@ -134,7 +135,9 @@ module GitLab
 
           @results = @collector.run
 
-          metrics_total
+          ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
+          ci_builds_metrics(@results[:created_builds], "ci_created_builds")
+          ci_stale_builds_metrics
           metrics_per_runner
 
           self
@@ -146,20 +149,25 @@ module GitLab
 
         private
 
-        def metrics_total
-          @metrics.add("ci_builds_total", @results[:created][:shared_enabled], status: "created",
-                                                                               shared_runners_enabled_projects: 1)
-          @metrics.add("ci_builds_total", @results[:created][:shared_disabled], status: "created",
-                                                                                shared_runners_enabled_projects: 0)
-          @metrics.add("ci_builds_total", @results[:pending][:shared_enabled], status: "pending",
-                                                                               shared_runners_enabled_projects: 1)
-          @metrics.add("ci_builds_total", @results[:pending][:shared_disabled], status: "pending",
-                                                                                shared_runners_enabled_projects: 0)
-          @metrics.add("ci_builds_total", @results[:running][:shared_enabled], status: "running",
-                                                                               shared_runners_enabled_projects: 1)
-          @metrics.add("ci_builds_total", @results[:running][:shared_disabled], status: "running",
-                                                                                shared_runners_enabled_projects: 0)
+        def ci_builds_metrics(results_list, metric_name)
+          other_value = { "yes" => 0, "no" => 0 }
+          results_list.each do |metric|
+            shared_runners = metric[:shared_runners]
+            namespace = metric[:namespace]
+            value = metric[:value]
+            if value < 10
+              other_value[shared_runners] += value
+              value = other_value[shared_runners]
+              namespace = ""
+            end
+            @metrics.add(metric_name,
+                         value,
+                         namespace: namespace,
+                         shared_runners: shared_runners)
+          end
+        end
 
+        def ci_stale_builds_metrics
           @metrics.add("ci_builds_stale", @results[:not_updated_last_hour], status: "running", when: "last_hour")
         end
 

spec/database/ci_builds_spec.rb

@@ -2,14 +2,14 @@ require "spec_helper"
 require "gitlab_monitor/database/ci_builds"
 
 describe GitLab::Monitor::Database do
-  let(:total_builds_query) { "SELECT TOTAL BUILDS" }
+  let(:builds_query) { "SELECT BUILDS" }
   let(:not_updated_running_builds_query) { "SELECT NOT UPDATED RUNNING" }
   let(:per_runner_query) { "SELECT ALL RUNNING PER RUNNER %s" }
   let(:mirrors_per_runner_query) { "SELECT MIRRORS RUNNING PER RUNNER %s" }
   let(:connection) { double("connection") }
 
   before do
-    stub_const("GitLab::Monitor::Database::CiBuildsCollector::TOTAL_BUILDS_QUERY", total_builds_query)
+    stub_const("GitLab::Monitor::Database::CiBuildsCollector::BUILDS_QUERY", builds_query)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::NOT_UPDATED_RUNNING_BUILDS_QUERY",
                not_updated_running_builds_query)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_PER_SHARED_RUNNER_QUERY", per_runner_query)
@@ -20,13 +20,13 @@ describe GitLab::Monitor::Database do
     allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:shared_runners_ids)
       .and_return([1, 2])
 
-    allow(connection).to receive(:exec).with(total_builds_query)
-      .and_return([{ "shared_runners_enabled" => "f", "status" => "created", "count" => 1 },
-                   { "shared_runners_enabled" => "t", "status" => "created", "count" => 4 },
-                   { "shared_runners_enabled" => "f", "status" => "pending", "count" => 2 },
-                   { "shared_runners_enabled" => "t", "status" => "pending", "count" => 5 },
-                   { "shared_runners_enabled" => "f", "status" => "running", "count" => 2 },
-                   { "shared_runners_enabled" => "t", "status" => "running", "count" => 3 }])
+    allow(connection).to receive(:exec).with(builds_query)
+      .and_return([{ "shared_runners_enabled" => "f", "status" => "created", "path" => "foo", "count" => 10 },
+                   { "shared_runners_enabled" => "t", "status" => "pending", "path" => "foo", "count" => 30 },
+                   { "shared_runners_enabled" => "f", "status" => "created", "path" => "bar", "count" => 20 },
+                   { "shared_runners_enabled" => "t", "status" => "pending", "path" => "bar", "count" => 50 },
+                   { "shared_runners_enabled" => "t", "status" => "pending", "path" => "baz", "count" => 1 },
+                   { "shared_runners_enabled" => "t", "status" => "pending", "path" => "bob", "count" => 2 }])
     allow(connection).to receive(:exec).with(not_updated_running_builds_query).and_return([{ "count" => 2 }])
     allow(connection).to receive(:exec).with(per_runner_query % "1, 2")
       .and_return([{ "runner_id" => 2, "count" => 15 }])
@@ -40,9 +40,16 @@ describe GitLab::Monitor::Database do
     it "executes the query" do
       expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 },
                                   per_runner_mirrors: { 1 => 0, 2 => 10 },
-                                  created: { shared_enabled: 4, shared_disabled: 1 },
-                                  pending: { shared_enabled: 5, shared_disabled: 2 },
-                                  running: { shared_enabled: 3, shared_disabled: 2 },
+                                  pending_builds: [
+                                    { namespace: "foo", shared_runners: "yes", value: 30 },
+                                    { namespace: "bar", shared_runners: "yes", value: 50 },
+                                    { namespace: "baz", shared_runners: "yes", value: 1 },
+                                    { namespace: "bob", shared_runners: "yes", value: 2 }
+                                  ],
+                                  created_builds: [
+                                    { namespace: "foo", shared_runners: "no", value: 10 },
+                                    { namespace: "bar", shared_runners: "no", value: 20 }
+                                  ],
                                   not_updated_last_hour: 2)
     end
   end
@@ -62,12 +69,11 @@ describe GitLab::Monitor::Database do
       prober.probe_db
       prober.write_to(writer)
 
-      expect(writer.string).to match(/ci_builds_total{status="created",shared_runners_enabled_projects="1"} 4/)
-      expect(writer.string).to match(/ci_builds_total{status="created",shared_runners_enabled_projects="0"} 1/)
-      expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="1"} 5/)
-      expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="0"} 2/)
-      expect(writer.string).to match(/ci_builds_total{status="running",shared_runners_enabled_projects="1"} 3/)
-      expect(writer.string).to match(/ci_builds_total{status="running",shared_runners_enabled_projects="0"} 2/)
+      expect(writer.string).to match(/ci_created_builds{namespace="foo",shared_runners="no"} 10/)
+      expect(writer.string).to match(/ci_created_builds{namespace="bar",shared_runners="no"} 20/)
+      expect(writer.string).to match(/ci_pending_builds{namespace="foo",shared_runners="yes"} 30/)
+      expect(writer.string).to match(/ci_pending_builds{namespace="bar",shared_runners="yes"} 50/)
+      expect(writer.string).to match(/ci_pending_builds{namespace="",shared_runners="yes"} 3/)
       expect(writer.string).to match(/ci_builds_stale{status="running",when="last_hour"} 2/)
       expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="0"} 0/)
       expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="0"} 5/)