Commit 54e3aed8 authored by Ahmad Sherif's avatar Ahmad Sherif
Browse files

Merge branch 'unarchived-traces-metric' into 'master'

Track number of unarchived traces

Closes #49

See merge request gitlab-org/gitlab-monitor!89
parents 05311f6d 781e72c7
Pipeline #109594 passed with stage
in 1 minute and 2 seconds
......@@ -47,6 +47,7 @@ probes:
<<: *db_common_opts
allowed_repeated_commands_count: 2
created_builds_counting_disabled: true
unarchived_traces_offset_minutes: 1440
tuple_stats:
class_name: Database::TuplesProber
<<: *db_common
......
......@@ -232,14 +232,33 @@ module GitLab
subquery.status
SQL
UNARCHIVED_TRACES_QUERY =
<<~SQL.freeze
SELECT
COUNT(*) as count
FROM ci_builds
JOIN ci_build_trace_chunks
ON ci_build_trace_chunks.build_id = ci_builds.id
LEFT JOIN ci_job_artifacts
ON ci_job_artifacts.job_id = ci_builds.id
AND ci_job_artifacts.file_type = 3
WHERE ci_builds.type = 'Ci::Build'
AND ci_builds.status IN ('success', 'failed', 'canceled')
AND ci_builds.finished_at < '%s'
AND ci_job_artifacts.job_id IS NULL
SQL
STATUS_CREATED = "created".freeze
STATUS_PENDING = "pending".freeze
DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
def initialize(opts)
super(opts)
@allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
@created_builds_counting_disabled = opts[:created_builds_counting_disabled]
@unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
end
def run
......@@ -249,6 +268,7 @@ module GitLab
results[:stale_builds] = stale_builds
results[:per_runner] = per_runner_builds
results[:repeated_commands] = repeated_commands
results[:unarchived_traces] = unarchived_traces
results
end
......@@ -334,6 +354,21 @@ module GitLab
include_has_minutes_field(values, row)
end
def unarchived_traces
time = Time.now - (unarchived_traces_offset_minutes * 60)
query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString
with_connection_pool do |conn|
conn.exec(query)[0]["count"].to_i
end
rescue PG::UndefinedTable, PG::UndefinedColumn
0
end
def unarchived_traces_offset_minutes
@unarchived_traces_offset_minutes ||= DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES
end
def include_ee_fields(values, row)
values.merge!(include_bool_if_row_defined(row, :mirror))
values.merge!(include_bool_if_row_defined(row, :mirror_trigger_builds))
......@@ -378,7 +413,8 @@ module GitLab
collector_opts = { connection_string: opts[:connection_string],
allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
created_builds_counting_disabled: opts[:created_builds_counting_disabled] }
created_builds_counting_disabled: opts[:created_builds_counting_disabled],
unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
@collector = CiBuildsCollector.new(collector_opts)
end
......@@ -390,6 +426,7 @@ module GitLab
ci_stale_builds_metrics
metrics_per_runner
repeated_commands_metrics
unarchived_traces_metrics
self
rescue PG::ConnectionBad
......@@ -480,6 +517,10 @@ module GitLab
@metrics.add("ci_repeated_commands_builds", value, metric)
end
end
def unarchived_traces_metrics
@metrics.add("ci_unarchived_traces", @results[:unarchived_traces])
end
end
end
end
......
......@@ -12,10 +12,14 @@ describe GitLab::Monitor::Database do
let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
let(:connection_pool) { double("connection pool") }
let(:connection) { double("connection") }
let(:allowed_repeated_commands_count) { 5 }
let(:created_builds_counting_disabled) { true }
let(:time_now) { Time.new(2019, 4, 9, 6, 30, 0) }
let(:unarchived_traces_query_time) { "2019-04-09 05:30:00" }
let(:unarchived_traces_offset_minutes) { 60 }
def stub_ee
allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
......@@ -85,6 +89,7 @@ describe GitLab::Monitor::Database do
stub_const("GitLab::Monitor::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
stub_const("GitLab::Monitor::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)
allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
allow(connection_pool).to receive(:with).and_yield(connection)
......@@ -92,6 +97,8 @@ describe GitLab::Monitor::Database do
allow(connection).to receive(:transaction).and_yield(connection)
allow(connection).to receive(:exec).with(set_random_page_cost_query)
allow(Time).to receive(:now).and_return(time_now)
allow(connection).to receive(:exec).with(builds_query_ee)
.and_return([builds_query_row_ee("f", "created", "1", "f", 10),
builds_query_row_ee("t", "pending", "1", "t", 30),
......@@ -140,15 +147,21 @@ describe GitLab::Monitor::Database do
repeated_commands_query_row_ce(2, "f", 2, "running", 20),
repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
unarchived_traces_query_with_time = unarchived_traces_query % [unarchived_traces_query_time] # rubocop:disable Style/FormatString
allow(connection).to receive(:exec).with(unarchived_traces_query_with_time).and_return([{ "count" => 10 }])
end
describe GitLab::Monitor::Database::CiBuildsCollector do
let(:collector) do
described_class.new(connection_string: "host=localhost",
allowed_repeated_commands_count: allowed_repeated_commands_count,
created_builds_counting_disabled: created_builds_counting_disabled)
created_builds_counting_disabled: created_builds_counting_disabled,
unarchived_traces_offset_minutes: unarchived_traces_offset_minutes)
end
let(:expected_stale_builds) { 2 }
let(:expected_unarchived_traces) { 10 }
shared_examples "data collector" do
subject { collector.run }
......@@ -185,6 +198,10 @@ describe GitLab::Monitor::Database do
it "returns raw repeated_commands data" do
expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
end
it "returns raw unarchived_traces data" do
expect(subject[:unarchived_traces]).to eq(expected_unarchived_traces)
end
end
context "when executed on EE" do
......@@ -257,7 +274,8 @@ describe GitLab::Monitor::Database do
let(:prober) do
opts = { connection_string: "host=localhost",
allowed_repeated_commands_count: allowed_repeated_commands_count,
created_builds_counting_disabled: created_builds_counting_disabled }
created_builds_counting_disabled: created_builds_counting_disabled,
unarchived_traces_offset_minutes: unarchived_traces_offset_minutes }
described_class.new(opts,
metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
end
......@@ -315,6 +333,10 @@ describe GitLab::Monitor::Database do
it "responds with stale builds Prometheus metrics" do
expect(subject).to match(/^ci_stale_builds 2$/m)
end
it "responds with unarchived traces Prometheus metrics" do
expect(subject).to match(/^ci_unarchived_traces 10$/m)
end
end
context "when PG exceptions are raised" do
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment