Add Ci Build metrics for pending builds, and stale running builds.

lib/gitlab_monitor/database/ci_builds.rb

@@ -6,6 +6,15 @@ module GitLab
         RUNNING_BUILDS_QUERY =
           "SELECT COUNT(*) AS count FROM ci_builds AS b WHERE b.type = 'Ci::Build' AND b.status = 'running'".freeze
 
+        NOT_UPDATED_RUNNING_BUILDS_QUERY =
+          "SELECT COUNT(*) AS count FROM ci_builds AS b WHERE b.type = 'Ci::Build' AND b.status = 'running' " \
+          "AND b.updated_at < NOW() - INTERVAL '1 hour'".freeze
+
+        PENDING_BUILDS_QUERY =
+          "SELECT p.shared_runners_enabled, COUNT(*) AS count FROM ci_builds AS b JOIN projects AS p " \
+          "ON p.id = b.gl_project_id WHERE b.type = 'Ci::Build' AND b.status = 'pending' " \
+          "GROUP BY p.shared_runners_enabled".freeze
+
         RUNNING_PER_SHARED_RUNNER_QUERY =
           "SELECT b.runner_id, COUNT(*) AS count " \
           "FROM ci_builds AS b " \
@@ -23,6 +32,8 @@ module GitLab
         def run
           results = {}
           results[:all] = get_general(RUNNING_BUILDS_QUERY)
+          results[:pending] = get_shared_enabled_grouped(PENDING_BUILDS_QUERY)
+          results[:not_updated_last_hour] = get_general(NOT_UPDATED_RUNNING_BUILDS_QUERY)
           results[:per_runner] = get_per_shared_runner(RUNNING_PER_SHARED_RUNNER_QUERY)
           results[:per_runner_mirrors] = get_per_shared_runner(RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY)
           results
@@ -55,6 +66,18 @@ module GitLab
           0
         end
 
+        def get_shared_enabled_grouped(query)
+          results = { shared_enabled: 0, shared_disabled: 0 }
+          connection.exec(query).each do |row|
+            row["shared_runners_enabled"] = row["shared_runners_enabled"] == "t"
+            results[:shared_enabled] = row["count"].to_i if row["shared_runners_enabled"]
+            results[:shared_disabled] = row["count"].to_i unless row["shared_runners_enabled"]
+          end
+          results
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          { shared_enabled: 0, shared_disabled: 0 }
+        end
+
         def shared_runners_ids
           @shared_runners_ids ||= find_shared_runners_ids
         end
@@ -76,17 +99,10 @@ module GitLab
         def probe_db
           return self unless @collector.connected?
 
-          results = @collector.run
+          @results = @collector.run
 
-          @metrics.add("ci_builds_total", results[:all].to_i, status: "running")
-
-          results[:per_runner].each do |key, value|
-            all = value
-            for_mirrors = results[:per_runner_mirrors][key]
-            not_for_mirrors = all - for_mirrors
-            @metrics.add("ci_builds_per_runner", not_for_mirrors, status: "running", runner: key, mirrors: 0)
-            @metrics.add("ci_builds_per_runner", for_mirrors, status: "running", runner: key, mirrors: 1)
-          end
+          metrics_total
+          metrics_per_runner
 
           self
         end
@@ -94,6 +110,28 @@ module GitLab
         def write_to(target)
           target.write(@metrics.to_s)
         end
+
+        private
+
+        def metrics_total
+          @metrics.add("ci_builds_total", @results[:all], status: "running")
+          @metrics.add("ci_builds_total", @results[:pending][:shared_enabled], status: "pending",
+                                                                               shared_runners_enabled_projects: 1)
+          @metrics.add("ci_builds_total", @results[:pending][:shared_disabled], status: "pending",
+                                                                                shared_runners_enabled_projects: 0)
+
+          @metrics.add("ci_builds_stale", @results[:not_updated_last_hour], status: "running", when: "last_hour")
+        end
+
+        def metrics_per_runner
+          @results[:per_runner].each do |key, value|
+            all = value
+            for_mirrors = @results[:per_runner_mirrors][key]
+            not_for_mirrors = all - for_mirrors
+            @metrics.add("ci_builds_per_runner", not_for_mirrors, status: "running", runner: key, mirrors: 0)
+            @metrics.add("ci_builds_per_runner", for_mirrors, status: "running", runner: key, mirrors: 1)
+          end
+        end
       end
     end
   end

spec/database/ci_builds_spec.rb

@@ -3,12 +3,17 @@ require "gitlab_monitor/database/ci_builds"
 
 describe GitLab::Monitor::Database do
   let(:running_builds_query) { "SELECT RUNNING" }
+  let(:not_updated_running_builds_query) { "SELECT NOT UPDATED RUNNING" }
+  let(:pending_builds_query) { "SELECT PENDING BUILDS" }
   let(:per_runner_query) { "SELECT ALL RUNNING PER RUNNER %s" }
   let(:mirrors_per_runner_query) { "SELECT MIRRORS RUNNING PER RUNNER %s" }
   let(:connection) { double("connection") }
 
   before do
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_BUILDS_QUERY", running_builds_query)
+    stub_const("GitLab::Monitor::Database::CiBuildsCollector::NOT_UPDATED_RUNNING_BUILDS_QUERY",
+               not_updated_running_builds_query)
+    stub_const("GitLab::Monitor::Database::CiBuildsCollector::PENDING_BUILDS_QUERY", pending_builds_query)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_PER_SHARED_RUNNER_QUERY", per_runner_query)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::RUNNING_FOR_MIRRORS_PER_SHARED_RUNNER_QUERY",
                mirrors_per_runner_query)
@@ -16,7 +21,12 @@ describe GitLab::Monitor::Database do
     allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection).and_return(connection)
     allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:shared_runners_ids)
       .and_return([1, 2])
+
     allow(connection).to receive(:exec).with(running_builds_query).and_return([{ "count" => 5 }])
+    allow(connection).to receive(:exec).with(not_updated_running_builds_query).and_return([{ "count" => 2 }])
+    allow(connection).to receive(:exec).with(pending_builds_query)
+      .and_return([{ "shared_runners_enabled" => "f", "count" => 2 },
+                   { "shared_runners_enabled" => "t", "count" => 5 }])
     allow(connection).to receive(:exec).with(per_runner_query % "1, 2")
       .and_return([{ "runner_id" => 2, "count" => 15 }])
     allow(connection).to receive(:exec).with(mirrors_per_runner_query % "1, 2")
@@ -27,14 +37,18 @@ describe GitLab::Monitor::Database do
     let(:collector) { described_class.new(connection_string: "host=localhost") }
 
     it "executes the query" do
-      expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 }, per_runner_mirrors: { 1 => 0, 2 => 10 }, all: 5)
+      expect(collector.run).to eq(per_runner: { 1 => 0, 2 => 15 },
+                                  per_runner_mirrors: { 1 => 0, 2 => 10 },
+                                  all: 5,
+                                  not_updated_last_hour: 2,
+                                  pending: { shared_enabled: 5, shared_disabled: 2 })
     end
   end
 
   describe GitLab::Monitor::Database::CiBuildsProber do
     let(:writer) { StringIO.new }
     let(:prober) do
-      described_class.new(opts: { connection_string: "host=localhost" },
+      described_class.new({ connection_string: "host=localhost" },
                           metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
     end
 
@@ -47,6 +61,9 @@ describe GitLab::Monitor::Database do
       prober.write_to(writer)
 
       expect(writer.string).to match(/ci_builds_total{status="running"} 5/)
+      expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="1"} 5/)
+      expect(writer.string).to match(/ci_builds_total{status="pending",shared_runners_enabled_projects="0"} 2/)
+      expect(writer.string).to match(/ci_builds_stale{status="running",when="last_hour"} 2/)
       expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="0"} 0/)
       expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="2",mirrors="0"} 5/)
       expect(writer.string).to match(/ci_builds_per_runner{status="running",runner="1",mirrors="1"} 0/)