ci_builds.rb 20.2 KB
Newer Older
1
module GitLab
2
  module Exporter
3
    module Database
Ben Kochie's avatar
Ben Kochie committed
4
      # A helper class to collect CI builds metrics.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
      class CiBuildsCollector < Base # rubocop:disable Metrics/ClassLength
        SET_RANDOM_PAGE_COST = "SET LOCAL random_page_cost TO 1".freeze

        BUILDS_QUERY_EE =
          <<~SQL.freeze
            SELECT
              projects.namespace_id,
              ci_builds.status,
              projects.shared_runners_enabled,
              (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
                 COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
              COUNT(*) AS count
            FROM ci_builds
            JOIN projects
              ON projects.id = ci_builds.project_id
            JOIN namespaces
              ON namespaces.id = projects.namespace_id
            LEFT JOIN namespace_statistics
              ON namespace_statistics.namespace_id = namespaces.id
            JOIN application_settings
              ON application_settings.id = 1
            WHERE ci_builds.type = 'Ci::Build'
27
              AND ci_builds.status = '%s'
28 29
              -- The created_at filter has been introduced for performance reasons only
              AND ci_builds.created_at > NOW() - INTERVAL '7 days'
30 31 32 33 34 35 36 37 38 39 40
              AND projects.pending_delete = 'f'
            GROUP BY
              projects.namespace_id,
              ci_builds.status,
              projects.shared_runners_enabled,
              namespaces.shared_runners_minutes_limit,
              namespace_statistics.shared_runners_seconds,
              application_settings.shared_runners_minutes
          SQL

        BUILDS_QUERY_CE =
Tomasz Maczukin's avatar
Tomasz Maczukin committed
41 42 43 44 45 46
          <<~SQL.freeze
            SELECT
              projects.namespace_id,
              ci_builds.status,
              projects.shared_runners_enabled,
              COUNT(*) AS count
47 48 49 50
            FROM ci_builds
            JOIN projects
              ON projects.id = ci_builds.project_id
            WHERE ci_builds.type = 'Ci::Build'
51
              AND ci_builds.status = '%s'
52 53
              -- The created_at filter has been introduced for performance reasons only
              AND ci_builds.created_at > NOW() - INTERVAL '7 days'
54 55 56 57 58
              AND projects.pending_delete = 'f'
            GROUP BY
              projects.namespace_id,
              ci_builds.status,
              projects.shared_runners_enabled
Tomasz Maczukin's avatar
Tomasz Maczukin committed
59
          SQL
60

Ben Kochie's avatar
Ben Kochie committed
61
        STALE_BUILDS_QUERY =
Tomasz Maczukin's avatar
Tomasz Maczukin committed
62 63 64
          <<~SQL.freeze
            SELECT
              COUNT(*) AS count
65 66 67 68 69 70 71
            FROM ci_builds
            JOIN projects
              ON projects.id = ci_builds.project_id
            WHERE ci_builds.type = 'Ci::Build'
              AND ci_builds.status = 'running'
              AND ci_builds.updated_at < NOW() - INTERVAL '1 hour'
              AND projects.pending_delete = 'f'
Tomasz Maczukin's avatar
Tomasz Maczukin committed
72
          SQL
73

74
        PER_RUNNER_QUERY_EE =
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
75 76 77 78
          <<~SQL.freeze
            SELECT
              ci_builds.runner_id,
              ci_runners.is_shared,
Tomasz Maczukin's avatar
Tomasz Maczukin committed
79
              projects.namespace_id,
80 81 82 83
              projects.mirror,
              projects.mirror_trigger_builds,
              ci_pipelines.pipeline_schedule_id,
              ci_builds.trigger_request_id,
84 85
              (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
                 COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
86 87 88 89 90 91 92 93
              COUNT(*) AS count
            FROM ci_builds
            JOIN ci_runners
              ON ci_runners.id = ci_builds.runner_id
            JOIN projects
              ON projects.id = ci_builds.project_id
            JOIN ci_pipelines
              ON ci_pipelines.id = ci_builds.commit_id
94 95 96 97 98 99
            JOIN namespaces
              ON namespaces.id = projects.namespace_id
            LEFT JOIN namespace_statistics
              ON namespace_statistics.namespace_id = namespaces.id
            JOIN application_settings
              ON application_settings.id = 1
100 101
            WHERE ci_builds.type = 'Ci::Build'
              AND ci_builds.status = 'running'
102 103
              -- The created_at filter has been introduced for performance reasons only
              AND ci_builds.created_at > NOW() - INTERVAL '7 days'
Tomasz Maczukin's avatar
Tomasz Maczukin committed
104
              AND projects.pending_delete = 'f'
105 106 107
            GROUP BY
              ci_builds.runner_id,
              ci_runners.is_shared,
108 109 110
              projects.namespace_id,
              projects.mirror,
              projects.mirror_trigger_builds,
111
              ci_pipelines.pipeline_schedule_id,
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
112
              ci_builds.trigger_request_id,
113 114 115
              namespaces.shared_runners_minutes_limit,
              namespace_statistics.shared_runners_seconds,
              application_settings.shared_runners_minutes
116 117 118 119 120 121 122
          SQL

        PER_RUNNER_QUERY_CE =
          <<~SQL.freeze
            SELECT
              ci_builds.runner_id,
              ci_runners.is_shared,
Tomasz Maczukin's avatar
Tomasz Maczukin committed
123
              projects.namespace_id,
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
124 125 126 127 128 129 130 131 132 133 134 135
              ci_pipelines.pipeline_schedule_id,
              ci_builds.trigger_request_id,
              COUNT(*) AS count
            FROM ci_builds
            JOIN ci_runners
              ON ci_runners.id = ci_builds.runner_id
            JOIN projects
              ON projects.id = ci_builds.project_id
            JOIN ci_pipelines
              ON ci_pipelines.id = ci_builds.commit_id
            WHERE ci_builds.type = 'Ci::Build'
              AND ci_builds.status = 'running'
136 137
              -- The created_at filter has been introduced for performance reasons only
              AND ci_builds.created_at > NOW() - INTERVAL '7 days'
Tomasz Maczukin's avatar
Tomasz Maczukin committed
138
              AND projects.pending_delete = 'f'
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
139 140 141
            GROUP BY
              ci_builds.runner_id,
              ci_runners.is_shared,
Tomasz Maczukin's avatar
Tomasz Maczukin committed
142
              projects.namespace_id,
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
143 144 145
              ci_pipelines.pipeline_schedule_id,
              ci_builds.trigger_request_id
          SQL
146

Tomasz Maczukin's avatar
Tomasz Maczukin committed
147 148 149
        MIRROR_COLUMN_QUERY =
          <<~SQL.freeze
            SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
Tomasz Maczukin's avatar
Tomasz Maczukin committed
150
          SQL
Tomasz Maczukin's avatar
Tomasz Maczukin committed
151

152 153 154
        REPEATED_COMMANDS_QUERY_EE =
          <<~SQL.freeze
            SELECT
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
              subquery.namespace_id,
              subquery.shared_runners_enabled,
              subquery.project_id,
              subquery.status,
              subquery.has_minutes,
              MAX(subquery.count) as count
            FROM (
              SELECT
                projects.namespace_id,
                projects.shared_runners_enabled,
                ci_builds.project_id,
                ci_builds.commit_id,
                ci_builds.status,
                (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
                   COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
                COUNT(*) AS count
              FROM ci_builds
              JOIN projects
                ON projects.id = ci_builds.project_id
              JOIN namespaces
                ON namespaces.id = projects.namespace_id
              LEFT JOIN namespace_statistics
                ON namespace_statistics.namespace_id = namespaces.id
              JOIN application_settings
                ON application_settings.id = 1
              WHERE ci_builds.type = 'Ci::Build'
                AND ci_builds.status IN ('running', 'pending')
182 183
                -- The created_at filter has been introduced for performance reasons only
                AND ci_builds.created_at > NOW() - INTERVAL '7 days'
184 185 186 187 188 189 190 191 192 193 194 195
              GROUP BY
                projects.namespace_id,
                projects.shared_runners_enabled,
                ci_builds.project_id,
                ci_builds.commit_id,
                ci_builds.status,
                ci_builds.commands,
                namespaces.shared_runners_minutes_limit,
                namespace_statistics.shared_runners_seconds,
                application_settings.shared_runners_minutes
              HAVING COUNT(*) > %d
            ) AS subquery
196
            GROUP BY
197 198 199 200 201 202
              subquery.namespace_id,
              subquery.shared_runners_enabled,
              subquery.project_id,
              subquery.commit_id,
              subquery.status,
              subquery.has_minutes
203 204 205 206 207
          SQL

        REPEATED_COMMANDS_QUERY_CE =
          <<~SQL.freeze
            SELECT
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
              subquery.namespace_id,
              subquery.shared_runners_enabled,
              subquery.project_id,
              subquery.status,
              MAX(subquery.count) as count
            FROM (
              SELECT
                projects.namespace_id,
                projects.shared_runners_enabled,
                ci_builds.project_id,
                ci_builds.commit_id,
                ci_builds.status,
                COUNT(*) AS count
              FROM ci_builds
              JOIN projects
                ON projects.id = ci_builds.project_id
              JOIN namespaces
                ON namespaces.id = projects.namespace_id
              WHERE ci_builds.type = 'Ci::Build'
                AND ci_builds.status IN ('running', 'pending')
228 229
              -- The created_at filter has been introduced for performance reasons only
              AND ci_builds.created_at > NOW() - INTERVAL '7 days'
230 231 232 233 234 235 236 237 238
              GROUP BY
                projects.namespace_id,
                projects.shared_runners_enabled,
                ci_builds.project_id,
                ci_builds.commit_id,
                ci_builds.status,
                ci_builds.commands
              HAVING COUNT(*) > %d
            ) AS subquery
239
            GROUP BY
240 241 242 243 244
              subquery.namespace_id,
              subquery.shared_runners_enabled,
              subquery.project_id,
              subquery.commit_id,
              subquery.status
245 246
          SQL

247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
        UNARCHIVED_TRACES_QUERY =
          <<~SQL.freeze
            SELECT
              COUNT(*) as count
            FROM ci_builds
            JOIN ci_build_trace_chunks
              ON ci_build_trace_chunks.build_id = ci_builds.id
            LEFT JOIN ci_job_artifacts
              ON ci_job_artifacts.job_id = ci_builds.id
              AND ci_job_artifacts.file_type = 3
            WHERE ci_builds.type = 'Ci::Build'
              AND ci_builds.status IN ('success', 'failed', 'canceled')
              AND ci_builds.finished_at < '%s'
              AND ci_job_artifacts.job_id IS NULL
          SQL

263 264 265
        STATUS_CREATED = "created".freeze
        STATUS_PENDING = "pending".freeze

266 267
        DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440

268 269 270 271
        def initialize(opts)
          super(opts)

          @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
272
          @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
273
          @unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
274 275
        end

276 277
        def run
          results = {}
278
          results[:created_builds] = builds(STATUS_CREATED) unless @created_builds_counting_disabled
279
          results[:pending_builds] = builds(STATUS_PENDING)
280 281
          results[:stale_builds] = stale_builds
          results[:per_runner] = per_runner_builds
282
          results[:repeated_commands] = repeated_commands
283
          results[:unarchived_traces] = unarchived_traces
284 285 286 287 288
          results
        end

        private

289 290
        def builds(status)
          results = []
Tomasz Maczukin's avatar
Tomasz Maczukin committed
291

292
          query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
293
          query = query % [status] # rubocop:disable Style/FormatString
294
          exec_query_with_custom_random_page_cost(query).each do |row|
295
            results << transform_builds_row_to_values(row)
Tomasz Maczukin's avatar
Tomasz Maczukin committed
296 297 298 299 300 301 302
          end

          results
        rescue PG::UndefinedTable, PG::UndefinedColumn
          results
        end

303 304 305 306 307 308 309 310
        def transform_builds_row_to_values(row)
          values = { namespace: row["namespace_id"].to_s,
                     shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
                     value: row["count"].to_i }
          include_ee_fields(values, row)
        end

        def stale_builds
311
          with_connection_pool do |conn|
312 313
            conn.exec(STALE_BUILDS_QUERY)[0]["count"].to_i
          end
Tomasz Maczukin's avatar
Tomasz Maczukin committed
314 315 316 317
        rescue PG::UndefinedTable, PG::UndefinedColumn
          0
        end

318
        def per_runner_builds
Ben Kochie's avatar
Ben Kochie committed
319
          results = []
320

321 322 323
          query = mirror_column? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
          exec_query_with_custom_random_page_cost(query).each do |row|
            results << transform_per_runners_builds_row_to_values(row)
324 325 326
          end

          results
Ben Kochie's avatar
Ben Kochie committed
327 328
        rescue PG::UndefinedTable, PG::UndefinedColumn
          []
329 330
        end

331
        def transform_per_runners_builds_row_to_values(row)
Tomasz Maczukin's avatar
Tomasz Maczukin committed
332 333 334 335 336 337
          values = { runner: row["runner_id"].to_s,
                     shared_runner: row["is_shared"] == "t" ? "yes" : "no",
                     namespace: row["namespace_id"].to_s,
                     scheduled: row["pipeline_schedule_id"] ? "yes" : "no",
                     triggered: row["trigger_request_id"] ? "yes" : "no",
                     value: row["count"].to_i }
338
          include_ee_fields(values, row)
Tomasz Maczukin's avatar
Tomasz Maczukin committed
339 340
        end

341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
        def repeated_commands
          results = []

          query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
          query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
          exec_query_with_custom_random_page_cost(query).each do |row|
            results << transform_repeated_commands_row_to_values(row)
          end

          results
        rescue PG::UndefinedTable, PG::UndefinedColumn
          []
        end

        def allowed_repeated_commands_count
          @allowed_repeated_commands_count ||= 2
        end

        def transform_repeated_commands_row_to_values(row)
          values = { namespace: row["namespace_id"].to_s,
                     project: row["project_id"].to_s,
                     shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
                     status: row["status"].to_s,
                     value: row["count"].to_i }

          include_has_minutes_field(values, row)
        end

369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
        def unarchived_traces
          time = Time.now - (unarchived_traces_offset_minutes * 60)
          query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString

          with_connection_pool do |conn|
            conn.exec(query)[0]["count"].to_i
          end
        rescue PG::UndefinedTable, PG::UndefinedColumn
          0
        end

        def unarchived_traces_offset_minutes
          @unarchived_traces_offset_minutes ||= DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES
        end

384 385 386
        def include_ee_fields(values, row)
          values.merge!(include_bool_if_row_defined(row, :mirror))
          values.merge!(include_bool_if_row_defined(row, :mirror_trigger_builds))
387 388 389 390
          include_has_minutes_field(values, row)
        end

        def include_has_minutes_field(values, row)
391
          values.merge!(include_bool_if_row_defined(row, :has_minutes))
Tomasz Maczukin's avatar
Tomasz Maczukin committed
392
          values
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
393 394
        end

395 396 397 398 399 400
        def include_bool_if_row_defined(row, field)
          return {} unless row[field.to_s]
          { field => row[field.to_s] == "t" ? "yes" : "no" }
        end

        def exec_query_with_custom_random_page_cost(query)
401
          with_connection_pool do |conn|
402 403 404 405
            conn.transaction do |trans|
              trans.exec(SET_RANDOM_PAGE_COST)
              trans.exec(query)
            end
406 407 408
          end
        end

409 410 411
        def mirror_column?
          @mirror_column ||=
            begin
412
              with_connection_pool do |conn|
413 414
                conn.exec(MIRROR_COLUMN_QUERY)[0]["exists"] == "t"
              end
415 416 417 418
            rescue PG::UndefinedColumn
              false
            end
        end
419 420 421 422 423 424
      end

      # The prober which is called when gathering metrics
      class CiBuildsProber
        def initialize(opts, metrics: PrometheusMetrics.new)
          @metrics = metrics
425 426

          collector_opts = { connection_string: opts[:connection_string],
427
                             allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
428 429
                             created_builds_counting_disabled: opts[:created_builds_counting_disabled],
                             unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
430
          @collector = CiBuildsCollector.new(collector_opts)
431 432 433
        end

        def probe_db
434
          @results = @collector.run
435

436
          ci_builds_metrics(@results[:created_builds], "ci_created_builds") if @results[:created_builds]
Ben Kochie's avatar
Ben Kochie committed
437 438
          ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
          ci_stale_builds_metrics
439
          metrics_per_runner
440
          repeated_commands_metrics
441
          unarchived_traces_metrics
442

443 444
          self
        rescue PG::ConnectionBad
445 446 447 448 449 450
          self
        end

        def write_to(target)
          target.write(@metrics.to_s)
        end
451 452 453

        private

Ben Kochie's avatar
Ben Kochie committed
454
        def ci_builds_metrics(results_list, metric_name)
Tomasz Maczukin's avatar
Tomasz Maczukin committed
455
          other_values = {}
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
456

Ben Kochie's avatar
Ben Kochie committed
457
          results_list.each do |metric|
Ben Kochie's avatar
Ben Kochie committed
458
            # If we have a low value, put the value into an "other" bucket.
Tomasz Maczukin's avatar
Tomasz Maczukin committed
459 460
            if metric[:value] < 10
              key = { shared_runners: metric[:shared_runners] }
461 462
              key[:has_minutes] = metric[:has_minutes] if metric[:has_minutes]

Tomasz Maczukin's avatar
Tomasz Maczukin committed
463 464
              other_values[key] ||= 0
              other_values[key] += metric[:value]
Ben Kochie's avatar
Ben Kochie committed
465
            else
Tomasz Maczukin's avatar
Tomasz Maczukin committed
466
              add_ci_created_pending_builds(metric_name, metric[:value], metric)
Ben Kochie's avatar
Ben Kochie committed
467 468
            end
          end
Zeger-Jan van de Weg's avatar
Zeger-Jan van de Weg committed
469

Ben Kochie's avatar
Ben Kochie committed
470
          # Add metrics for the "other" bucket.
Tomasz Maczukin's avatar
Tomasz Maczukin committed
471
          other_values.each { |key, value| add_ci_created_pending_builds(metric_name, value, key) }
Tomasz Maczukin's avatar
Tomasz Maczukin committed
472 473 474
        end

        def add_ci_created_pending_builds(metric_name, value, labels)
Tomasz Maczukin's avatar
Tomasz Maczukin committed
475
          add_metric_with_namespace_label(metric_name,
476
                                          [:namespace, :shared_runners, :has_minutes],
Tomasz Maczukin's avatar
Tomasz Maczukin committed
477 478
                                          value,
                                          labels)
Ben Kochie's avatar
Ben Kochie committed
479
        end
480

Ben Kochie's avatar
Ben Kochie committed
481
        def ci_stale_builds_metrics
Ben Kochie's avatar
Ben Kochie committed
482
          @metrics.add("ci_stale_builds", @results[:stale_builds])
483 484 485
        end

        def metrics_per_runner
Tomasz Maczukin's avatar
Tomasz Maczukin committed
486 487
          other_values = {}

Ben Kochie's avatar
Ben Kochie committed
488
          @results[:per_runner].each do |metric|
Tomasz Maczukin's avatar
Tomasz Maczukin committed
489 490
            # If we have a low value, put the value into an "other" bucket.
            if metric[:value] < 10
Tomasz Maczukin's avatar
Tomasz Maczukin committed
491 492 493 494
              key = { runner: metric[:runner], shared_runner: metric[:shared_runner],
                      scheduled: metric[:scheduled], triggered: metric[:triggered] }
              key[:mirror] = metric[:mirror] if metric[:mirror]
              key[:mirror_trigger_builds] = metric[:mirror_trigger_builds] if metric[:mirror_trigger_builds]
495
              key[:has_minutes] = metric[:has_minutes] if metric[:has_minutes]
Tomasz Maczukin's avatar
Tomasz Maczukin committed
496

Tomasz Maczukin's avatar
Tomasz Maczukin committed
497 498 499 500 501 502 503 504
              other_values[key] ||= 0
              other_values[key] += metric[:value]
            else
              add_ci_running_builds(metric[:value], metric)
            end
          end

          # Add metrics for the "other" bucket.
Tomasz Maczukin's avatar
Tomasz Maczukin committed
505
          other_values.each { |key, value| add_ci_running_builds(value, key) }
506
        end
Tomasz Maczukin's avatar
Tomasz Maczukin committed
507 508

        def add_ci_running_builds(value, labels)
Tomasz Maczukin's avatar
Tomasz Maczukin committed
509 510
          add_metric_with_namespace_label(
            "ci_running_builds",
511 512
            [:runner, :namespace, :shared_runner, :scheduled,
             :triggered, :mirror, :mirror_trigger_builds, :has_minutes],
Tomasz Maczukin's avatar
Tomasz Maczukin committed
513 514 515
            value,
            labels
          )
Tomasz Maczukin's avatar
Tomasz Maczukin committed
516 517 518 519
        end

        def add_metric_with_namespace_label(metric_name, allowed_labels, value, labels)
          labels[:namespace] = "" unless labels[:namespace]
Tomasz Maczukin's avatar
Tomasz Maczukin committed
520 521 522

          selected_labels = labels.select { |k, _| allowed_labels.include?(k) }.sort.to_h
          @metrics.add(metric_name, value, selected_labels)
Tomasz Maczukin's avatar
Tomasz Maczukin committed
523
        end
524 525 526 527 528 529 530 531

        def repeated_commands_metrics
          @results[:repeated_commands].each do |metric|
            value = metric.delete(:value)

            @metrics.add("ci_repeated_commands_builds", value, metric)
          end
        end
532 533 534 535

        def unarchived_traces_metrics
          @metrics.add("ci_unarchived_traces", @results[:unarchived_traces])
        end
536 537 538 539
      end
    end
  end
end