Track number of unarchived traces

config/gitlab-monitor.yml.example

@@ -47,6 +47,7 @@ probes:
         <<: *db_common_opts
         allowed_repeated_commands_count: 2
         created_builds_counting_disabled: true
+        unarchived_traces_offset_minutes: 1440
     tuple_stats:
       class_name: Database::TuplesProber
       <<: *db_common

lib/gitlab_monitor/database/ci_builds.rb

@@ -232,14 +232,33 @@ module GitLab
               subquery.status
           SQL
 
+        UNARCHIVED_TRACES_QUERY =
+          <<~SQL.freeze
+            SELECT
+              COUNT(*) as count
+            FROM ci_builds
+            JOIN ci_build_trace_chunks
+              ON ci_build_trace_chunks.build_id = ci_builds.id
+            LEFT JOIN ci_job_artifacts
+              ON ci_job_artifacts.job_id = ci_builds.id
+              AND ci_job_artifacts.file_type = 3
+            WHERE ci_builds.type = 'Ci::Build'
+              AND ci_builds.status IN ('success', 'failed', 'canceled')
+              AND ci_builds.finished_at < '%s'
+              AND ci_job_artifacts.job_id IS NULL
+          SQL
+
         STATUS_CREATED = "created".freeze
         STATUS_PENDING = "pending".freeze
 
+        DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
+
         def initialize(opts)
           super(opts)
 
           @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
           @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
+          @unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
         end
 
         def run
@@ -249,6 +268,7 @@ module GitLab
           results[:stale_builds] = stale_builds
           results[:per_runner] = per_runner_builds
           results[:repeated_commands] = repeated_commands
+          results[:unarchived_traces] = unarchived_traces
           results
         end
 
@@ -334,6 +354,21 @@ module GitLab
           include_has_minutes_field(values, row)
         end
 
+        def unarchived_traces
+          time = Time.now - (unarchived_traces_offset_minutes * 60)
+          query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString
+
+          with_connection_pool do |conn|
+            conn.exec(query)[0]["count"].to_i
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          0
+        end
+
+        def unarchived_traces_offset_minutes
+          @unarchived_traces_offset_minutes ||= DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES
+        end
+
         def include_ee_fields(values, row)
           values.merge!(include_bool_if_row_defined(row, :mirror))
           values.merge!(include_bool_if_row_defined(row, :mirror_trigger_builds))
@@ -378,7 +413,8 @@ module GitLab
 
           collector_opts = { connection_string: opts[:connection_string],
                              allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
-                             created_builds_counting_disabled: opts[:created_builds_counting_disabled] }
+                             created_builds_counting_disabled: opts[:created_builds_counting_disabled],
+                             unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
           @collector = CiBuildsCollector.new(collector_opts)
         end
 
@@ -390,6 +426,7 @@ module GitLab
           ci_stale_builds_metrics
           metrics_per_runner
           repeated_commands_metrics
+          unarchived_traces_metrics
 
           self
         rescue PG::ConnectionBad
@@ -480,6 +517,10 @@ module GitLab
             @metrics.add("ci_repeated_commands_builds", value, metric)
           end
         end
+
+        def unarchived_traces_metrics
+          @metrics.add("ci_unarchived_traces", @results[:unarchived_traces])
+        end
       end
     end
   end

spec/database/ci_builds_spec.rb

@@ -12,10 +12,14 @@ describe GitLab::Monitor::Database do
   let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
   let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
   let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
+  let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
   let(:connection_pool) { double("connection pool") }
   let(:connection) { double("connection") }
   let(:allowed_repeated_commands_count) { 5 }
   let(:created_builds_counting_disabled) { true }
+  let(:time_now) { Time.new(2019, 4, 9, 6, 30, 0) }
+  let(:unarchived_traces_query_time) { "2019-04-09 05:30:00" }
+  let(:unarchived_traces_offset_minutes) { 60 }
 
   def stub_ee
     allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
@@ -85,6 +89,7 @@ describe GitLab::Monitor::Database do
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
     stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
+    stub_const("GitLab::Monitor::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)
 
     allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
     allow(connection_pool).to receive(:with).and_yield(connection)
@@ -92,6 +97,8 @@ describe GitLab::Monitor::Database do
     allow(connection).to receive(:transaction).and_yield(connection)
     allow(connection).to receive(:exec).with(set_random_page_cost_query)
 
+    allow(Time).to receive(:now).and_return(time_now)
+
     allow(connection).to receive(:exec).with(builds_query_ee)
       .and_return([builds_query_row_ee("f", "created", "1", "f", 10),
                    builds_query_row_ee("t", "pending", "1", "t", 30),
@@ -140,15 +147,21 @@ describe GitLab::Monitor::Database do
                    repeated_commands_query_row_ce(2, "f", 2, "running", 20),
                    repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
                    repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
+
+    unarchived_traces_query_with_time = unarchived_traces_query % [unarchived_traces_query_time] # rubocop:disable Style/FormatString
+
+    allow(connection).to receive(:exec).with(unarchived_traces_query_with_time).and_return([{ "count" => 10 }])
   end
 
   describe GitLab::Monitor::Database::CiBuildsCollector do
     let(:collector) do
       described_class.new(connection_string: "host=localhost",
                           allowed_repeated_commands_count: allowed_repeated_commands_count,
-                          created_builds_counting_disabled: created_builds_counting_disabled)
+                          created_builds_counting_disabled: created_builds_counting_disabled,
+                          unarchived_traces_offset_minutes: unarchived_traces_offset_minutes)
     end
     let(:expected_stale_builds) { 2 }
+    let(:expected_unarchived_traces) { 10 }
 
     shared_examples "data collector" do
       subject { collector.run }
@@ -185,6 +198,10 @@ describe GitLab::Monitor::Database do
       it "returns raw repeated_commands data" do
         expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
       end
+
+      it "returns raw unarchived_traces data" do
+        expect(subject[:unarchived_traces]).to eq(expected_unarchived_traces)
+      end
     end
 
     context "when executed on EE" do
@@ -257,7 +274,8 @@ describe GitLab::Monitor::Database do
     let(:prober) do
       opts = { connection_string: "host=localhost",
                allowed_repeated_commands_count: allowed_repeated_commands_count,
-               created_builds_counting_disabled: created_builds_counting_disabled }
+               created_builds_counting_disabled: created_builds_counting_disabled,
+               unarchived_traces_offset_minutes: unarchived_traces_offset_minutes }
       described_class.new(opts,
                           metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
     end
@@ -315,6 +333,10 @@ describe GitLab::Monitor::Database do
         it "responds with stale builds Prometheus metrics" do
           expect(subject).to match(/^ci_stale_builds 2$/m)
         end
+
+        it "responds with unarchived traces Prometheus metrics" do
+          expect(subject).to match(/^ci_unarchived_traces 10$/m)
+        end
       end
 
       context "when PG exceptions are raised" do