Skip to content

GitLab

Commit baa17eab authored by Ben Kochie's avatar Ben Kochie
Browse files
Options

Cleanup database probes 

Remove obsolete database probes that have been moved to the
`postgres_exporter`.
* `blocked_queries`
* `slow_queries:`
* `stuck_idle_in_transactions`
* `vacuum_queries`
parent 5f58d138
Pipeline #54414 passed with stage
    in 1 minute and 21 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 1 addition and 344 deletions
    README.md
    View file @ baa17eab
    ......@@ -19,15 +19,6 @@ metrics.
    1. Database
    * [Per-table tuple stats](lib/gitlab_monitor/database/tuple_stats.rb) --
    `pg_stat_table_*`
    * [Slow queries](lib/gitlab_monitor/database/slow_queries.rb) --
    `pg_slow_queries_total`
    * [Stuck queries in idle in transaction](lib/gitlab_monitor/database/stuck_idle_in_transactions.rb) --
    `pg_stuck_idle_in_transactions_total`
    * [Blocked queries](lib/gitlab_monitor/database/blocked_queries.rb) --
    `pg_blocked_queries_total`
    * [Vacuum queries](lib/gitlab_monitor/database/vacuum_queries.rb) --
    `pg_vacuum_count`, `pg_vacuum_age_in_seconds`, `pg_vacuum_analyze_count`,
    `pg_vacuum_analyze_age_in_seconds`
    * [Row count queries](lib/gitlab_monitor/database/row_count.rb) --
    `db_rows_count`
    * [CI builds](lib/gitlab_monitor/database/ci_builds.rb) --
    ......
    config/gitlab-monitor.yml.example
    View file @ baa17eab
    ......@@ -34,9 +34,6 @@ probes:
    # by probe definitions as usual.
    database:
    multiple: true
    blocked_queries:
    class_name: Database::BlockedQueriesProber
    <<: *db_common
    ci_builds:
    class_name: Database::CiBuildsProber
    <<: *db_common
    ......@@ -46,15 +43,6 @@ probes:
    rows_count:
    class_name: Database::RowCountProber
    <<: *db_common
    slow_queries:
    class_name: Database::SlowQueriesProber
    <<: *db_common
    stuck_idle_in_transactions:
    class_name: Database::StuckIdleInTransactionsProber
    <<: *db_common
    vacuum_queries:
    class_name: Database::VacuumQueriesProber
    <<: *db_common
    process:
    methods:
    ......
    lib/gitlab_monitor/cli.rb
    View file @ baa17eab
    ......@@ -118,132 +118,6 @@ module GitLab
    end
    end
    # Database blocked queries runner.
    #
    # It will take a database connection string and print results to STDOUT
    class DatabaseBlockedQueries
    COMMAND_NAME = "db-blocked-queries".freeze
    def initialize(args)
    @options = options(args)
    @options.parse!
    @target = args.shift || STDOUT
    @target = File.open(@target, "a") if @target.is_a?(String)
    end
    def options(args)
    args.options do |opts|
    opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
    opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
    @db_connection_string = val
    end
    end
    end
    def help
    @options.help
    end
    def run
    validate!
    ::GitLab::Monitor::Database::BlockedQueriesProber.new(connection_string: @db_connection_string)
    .probe_db
    .write_to(@target)
    end
    private
    def validate!
    fail InvalidCLICommand.new(help) unless @db_connection_string
    end
    end
    # Database slow queries runner.
    #
    # It will take a database connection string and print results to STDOUT
    class DatabaseSlowQueries
    COMMAND_NAME = "db-slow-queries".freeze
    def initialize(args)
    @options = options(args)
    @options.parse!
    @target = args.shift || STDOUT
    @target = File.open(@target, "a") if @target.is_a?(String)
    end
    def options(args)
    args.options do |opts|
    opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
    opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
    @db_connection_string = val
    end
    end
    end
    def help
    @options.help
    end
    def run
    validate!
    ::GitLab::Monitor::Database::SlowQueriesProber.new(connection_string: @db_connection_string)
    .probe_db
    .write_to(@target)
    end
    private
    def validate!
    fail InvalidCLICommand.new(help) unless @db_connection_string
    end
    end
    # Database vacuum queries runner.
    #
    # It will take a database connection string and print results to STDOUT
    class DatabaseVacuumQueries
    COMMAND_NAME = "db-vacuum-queries".freeze
    def initialize(args)
    @options = options(args)
    @options.parse!
    @target = args.shift || STDOUT
    @target = File.open(@target, "a") if @target.is_a?(String)
    end
    def options(args)
    args.options do |opts|
    opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
    opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
    @db_connection_string = val
    end
    end
    end
    def help
    @options.help
    end
    def run
    validate!
    ::GitLab::Monitor::Database::VacuumQueriesProber.new(connection_string: @db_connection_string)
    .probe_db
    .write_to(@target)
    end
    private
    def validate!
    fail InvalidCLICommand.new(help) unless @db_connection_string
    end
    end
    # Database row counts query runner.
    #
    # This will take the database connection and print the result to STDOUT
    ......@@ -452,9 +326,6 @@ module GitLab
    [
    GIT,
    DatabaseTupleStats,
    DatabaseBlockedQueries,
    DatabaseSlowQueries,
    DatabaseVacuumQueries,
    DatabaseRowCounts,
    Process,
    GitProcess,
    ......
    lib/gitlab_monitor/database.rb
    View file @ baa17eab
    ......@@ -3,12 +3,8 @@ module GitLab
    # Database-related classes
    module Database
    autoload :Base, "gitlab_monitor/database/base"
    autoload :BlockedQueriesProber, "gitlab_monitor/database/blocked_queries"
    autoload :CiBuildsProber, "gitlab_monitor/database/ci_builds"
    autoload :SlowQueriesProber, "gitlab_monitor/database/slow_queries"
    autoload :StuckIdleInTransactionsProber, "gitlab_monitor/database/stuck_idle_in_transactions"
    autoload :TuplesProber, "gitlab_monitor/database/tuple_stats"
    autoload :VacuumQueriesProber, "gitlab_monitor/database/vacuum_queries"
    autoload :RowCountProber, "gitlab_monitor/database/row_count"
    end
    end
    ......
    lib/gitlab_monitor/database/blocked_queries.rb deleted 100644 → 0
    View file @ 5f58d138
    module GitLab
    module Monitor
    module Database
    # A helper class to collect blocked queries
    #
    # It takes a connection string (e.g. "dbname=test port=5432")
    class BlockedQueriesCollector < Base
    def run
    stats = Hash.new(0)
    result = connection.exec(
    <<-SQL
    SELECT blocked.relation::regclass
    FROM pg_catalog.pg_locks blocked
    WHERE NOT blocked.granted
    SQL
    ).to_a
    result.each do |row|
    stats[row["relation"] || "__transaction__"] += 1
    end
    stats
    end
    end
    # Probes the DB specified by opts[:connection_string] for blocked queries, then converts them to metrics
    class BlockedQueriesProber
    def initialize(opts, metrics: PrometheusMetrics.new)
    @metrics = metrics
    @collector = BlockedQueriesCollector.new(connection_string: opts[:connection_string])
    end
    def probe_db
    return self unless @collector.connected?
    result = @collector.run
    result.each do |table_name, blocked_count|
    @metrics.add("pg_blocked_queries_total", blocked_count, table_name: table_name)
    end
    self
    end
    def write_to(target)
    target.write(@metrics.to_s)
    end
    end
    end
    end
    end
    lib/gitlab_monitor/database/ci_builds.rb
    View file @ baa17eab
    module GitLab
    module Monitor
    module Database
    # A helper class to collect blocked queries
    # A helper class to collect CI builds metrics.
    class CiBuildsCollector < Base # rubocop:disable ClassLength
    BUILDS_QUERY =
    <<~SQL.freeze
    ......
    lib/gitlab_monitor/database/slow_queries.rb deleted 100644 → 0
    View file @ 5f58d138
    module GitLab
    module Monitor
    module Database
    # A helper class to collect slow queries
    #
    # It takes a connection string (e.g. "dbname=test port=5432")
    class SlowQueriesCollector < Base
    def run
    result = connection.exec(
    <<-SQL
    SELECT COUNT(*) AS slow_count
    FROM pg_stat_activity
    WHERE state = 'active' AND (now() - query_start) > '1 seconds'::interval;
    SQL
    )
    result[0]["slow_count"].to_i
    end
    end
    # Probes the DB specified by opts[:connection_string] for slow queries, then converts them to metrics
    class SlowQueriesProber
    def initialize(opts, metrics: PrometheusMetrics.new)
    @metrics = metrics
    @collector = SlowQueriesCollector.new(connection_string: opts[:connection_string])
    end
    def probe_db
    return self unless @collector.connected?
    @metrics.add("pg_slow_queries_total", @collector.run)
    self
    end
    def write_to(target)
    target.write(@metrics.to_s)
    end
    end
    end
    end
    end
    lib/gitlab_monitor/database/stuck_idle_in_transactions.rb deleted 100644 → 0
    View file @ 5f58d138
    module GitLab
    module Monitor
    module Database
    # A helper class to collect queries stuck in idle in transaction
    #
    # It takes a connection string (e.g. "dbname=test port=5432")
    class StuckIdleInTransactionsCollector < Base
    def run
    result = connection.exec(
    <<-SQL
    SELECT COUNT(*) AS stuck_count
    FROM pg_stat_activity
    WHERE state = 'idle in transaction' AND (now() - query_start) > '10 minutes'::interval;
    SQL
    )
    result[0]["stuck_count"].to_i
    end
    end
    # Probes the DB specified by opts[:connection_string] for slow queries, then converts them to metrics
    class StuckIdleInTransactionsProber
    def initialize(opts, metrics: PrometheusMetrics.new)
    @metrics = metrics
    @collector = StuckIdleInTransactionsCollector.new(connection_string: opts[:connection_string])
    end
    def probe_db
    return self unless @collector.connected?
    @metrics.add("pg_stuck_idle_in_transactions_total", @collector.run)
    self
    end
    def write_to(target)
    target.write(@metrics.to_s)
    end
    end
    end
    end
    end
    lib/gitlab_monitor/database/vacuum_queries.rb deleted 100644 → 0
    View file @ 5f58d138
    module GitLab
    module Monitor
    module Database
    # A helper class to collect vacuum and vacuum analyze information
    #
    # It takes a connection string (e.g. "dbname=test port=5432")
    class VacuumQueriesCollector < Base
    def run
    stats = {}
    { vacuum: "VACUUM (?!ANALYZE)", vacuum_analyze: "VACUUM ANALYZE" }.each do |query_name, condition|
    result = connection.exec(
    <<-SQL
    SELECT COUNT(*) AS query_count, MAX(EXTRACT(EPOCH FROM (clock_timestamp() - query_start))) AS query_duration
    FROM pg_catalog.pg_stat_activity
    WHERE state = 'active' AND trim(query) ~* '\\A#{condition}';
    SQL
    )
    stats[query_name] = { "total" => result[0]["query_count"].to_i,
    "max_age" => result[0]["query_duration"].to_f }
    end
    stats
    end
    end
    # Probes the DB specified by opts[:connection_string] for vacuum and vacuum analyze information,
    # then converts them to metrics
    class VacuumQueriesProber
    def initialize(opts, metrics: PrometheusMetrics.new)
    @metrics = metrics
    @collector = VacuumQueriesCollector.new(connection_string: opts[:connection_string])
    end
    def probe_db
    return self unless @collector.connected?
    result = @collector.run
    @metrics.add("pg_vacuum_count", result[:vacuum]["total"])
    @metrics.add("pg_vacuum_age_in_seconds", result[:vacuum]["max_age"])
    @metrics.add("pg_vacuum_analyze_count", result[:vacuum_analyze]["total"])
    @metrics.add("pg_vacuum_analyze_age_in_seconds", result[:vacuum_analyze]["max_age"])
    self
    end
    def write_to(target)
    target.write(@metrics.to_s)
    end
    end
    end
    end
    end
    Markdown is supported
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment