diff --git a/config/gitlab-monitor.yml.example b/config/gitlab-monitor.yml.example index 782d289f4a8ee730455aec7b64ba800854124f96..c1ba143f8ed92f94d64da1b1da37496fd3d80508 100644 --- a/config/gitlab-monitor.yml.example +++ b/config/gitlab-monitor.yml.example @@ -47,6 +47,7 @@ probes: <<: *db_common_opts allowed_repeated_commands_count: 2 created_builds_counting_disabled: true + unarchived_traces_offset_minutes: 1440 tuple_stats: class_name: Database::TuplesProber <<: *db_common diff --git a/lib/gitlab_monitor/database/ci_builds.rb b/lib/gitlab_monitor/database/ci_builds.rb index acae76bde2ea7e094e29a7a4ea7e3785e286b2ba..2a701094201955c81f0b356053750bdcb3143bcb 100644 --- a/lib/gitlab_monitor/database/ci_builds.rb +++ b/lib/gitlab_monitor/database/ci_builds.rb @@ -232,14 +232,33 @@ module GitLab subquery.status SQL + UNARCHIVED_TRACES_QUERY = + <<~SQL.freeze + SELECT + COUNT(*) as count + FROM ci_builds + JOIN ci_build_trace_chunks + ON ci_build_trace_chunks.build_id = ci_builds.id + LEFT JOIN ci_job_artifacts + ON ci_job_artifacts.job_id = ci_builds.id + AND ci_job_artifacts.file_type = 3 + WHERE ci_builds.type = 'Ci::Build' + AND ci_builds.status IN ('success', 'failed', 'canceled') + AND ci_builds.finished_at < '%s' + AND ci_job_artifacts.job_id IS NULL + SQL + STATUS_CREATED = "created".freeze STATUS_PENDING = "pending".freeze + DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440 + def initialize(opts) super(opts) @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count] @created_builds_counting_disabled = opts[:created_builds_counting_disabled] + @unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes] end def run @@ -249,6 +268,7 @@ module GitLab results[:stale_builds] = stale_builds results[:per_runner] = per_runner_builds results[:repeated_commands] = repeated_commands + results[:unarchived_traces] = unarchived_traces results end @@ -334,6 +354,21 @@ module GitLab include_has_minutes_field(values, row) end + def unarchived_traces + time = Time.now - (unarchived_traces_offset_minutes * 60) + query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString + + with_connection_pool do |conn| + conn.exec(query)[0]["count"].to_i + end + rescue PG::UndefinedTable, PG::UndefinedColumn + 0 + end + + def unarchived_traces_offset_minutes + @unarchived_traces_offset_minutes ||= DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES + end + def include_ee_fields(values, row) values.merge!(include_bool_if_row_defined(row, :mirror)) values.merge!(include_bool_if_row_defined(row, :mirror_trigger_builds)) @@ -378,7 +413,8 @@ module GitLab collector_opts = { connection_string: opts[:connection_string], allowed_repeated_commands_count: opts[:allowed_repeated_commands_count], - created_builds_counting_disabled: opts[:created_builds_counting_disabled] } + created_builds_counting_disabled: opts[:created_builds_counting_disabled], + unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] } @collector = CiBuildsCollector.new(collector_opts) end @@ -390,6 +426,7 @@ module GitLab ci_stale_builds_metrics metrics_per_runner repeated_commands_metrics + unarchived_traces_metrics self rescue PG::ConnectionBad @@ -480,6 +517,10 @@ module GitLab @metrics.add("ci_repeated_commands_builds", value, metric) end end + + def unarchived_traces_metrics + @metrics.add("ci_unarchived_traces", @results[:unarchived_traces]) + end end end end diff --git a/spec/database/ci_builds_spec.rb b/spec/database/ci_builds_spec.rb index b76c1eeee3bc1e7b3fd626fafc1b845bce1ef563..cda2b2fa436becf6202558420596cd0bd5897514 100644 --- a/spec/database/ci_builds_spec.rb +++ b/spec/database/ci_builds_spec.rb @@ -12,10 +12,14 @@ describe GitLab::Monitor::Database do let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" } let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" } let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" } + let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" } let(:connection_pool) { double("connection pool") } let(:connection) { double("connection") } let(:allowed_repeated_commands_count) { 5 } let(:created_builds_counting_disabled) { true } + let(:time_now) { Time.new(2019, 4, 9, 6, 30, 0) } + let(:unarchived_traces_query_time) { "2019-04-09 05:30:00" } + let(:unarchived_traces_offset_minutes) { 60 } def stub_ee allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }]) @@ -85,6 +89,7 @@ describe GitLab::Monitor::Database do stub_const("GitLab::Monitor::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query) stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee) stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce) + stub_const("GitLab::Monitor::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query) allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool) allow(connection_pool).to receive(:with).and_yield(connection) @@ -92,6 +97,8 @@ describe GitLab::Monitor::Database do allow(connection).to receive(:transaction).and_yield(connection) allow(connection).to receive(:exec).with(set_random_page_cost_query) + allow(Time).to receive(:now).and_return(time_now) + allow(connection).to receive(:exec).with(builds_query_ee) .and_return([builds_query_row_ee("f", "created", "1", "f", 10), builds_query_row_ee("t", "pending", "1", "t", 30), @@ -140,15 +147,21 @@ describe GitLab::Monitor::Database do repeated_commands_query_row_ce(2, "f", 2, "running", 20), repeated_commands_query_row_ce(1, "f", 3, "pending", 30), repeated_commands_query_row_ce(2, "t", 4, "running", 40)]) + + unarchived_traces_query_with_time = unarchived_traces_query % [unarchived_traces_query_time] # rubocop:disable Style/FormatString + + allow(connection).to receive(:exec).with(unarchived_traces_query_with_time).and_return([{ "count" => 10 }]) end describe GitLab::Monitor::Database::CiBuildsCollector do let(:collector) do described_class.new(connection_string: "host=localhost", allowed_repeated_commands_count: allowed_repeated_commands_count, - created_builds_counting_disabled: created_builds_counting_disabled) + created_builds_counting_disabled: created_builds_counting_disabled, + unarchived_traces_offset_minutes: unarchived_traces_offset_minutes) end let(:expected_stale_builds) { 2 } + let(:expected_unarchived_traces) { 10 } shared_examples "data collector" do subject { collector.run } @@ -185,6 +198,10 @@ describe GitLab::Monitor::Database do it "returns raw repeated_commands data" do expect(subject[:repeated_commands]).to include(*expected_repeated_commands) end + + it "returns raw unarchived_traces data" do + expect(subject[:unarchived_traces]).to eq(expected_unarchived_traces) + end end context "when executed on EE" do @@ -257,7 +274,8 @@ describe GitLab::Monitor::Database do let(:prober) do opts = { connection_string: "host=localhost", allowed_repeated_commands_count: allowed_repeated_commands_count, - created_builds_counting_disabled: created_builds_counting_disabled } + created_builds_counting_disabled: created_builds_counting_disabled, + unarchived_traces_offset_minutes: unarchived_traces_offset_minutes } described_class.new(opts, metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false)) end @@ -315,6 +333,10 @@ describe GitLab::Monitor::Database do it "responds with stale builds Prometheus metrics" do expect(subject).to match(/^ci_stale_builds 2$/m) end + + it "responds with unarchived traces Prometheus metrics" do + expect(subject).to match(/^ci_unarchived_traces 10$/m) + end end context "when PG exceptions are raised" do