Commit baa17eab authored by Ben Kochie's avatar Ben Kochie

Cleanup database probes

Remove obsolete database probes that have been moved to the
`postgres_exporter`.
* `blocked_queries`
* `slow_queries:`
* `stuck_idle_in_transactions`
* `vacuum_queries`
parent 5f58d138
Pipeline #54414 passed with stage
in 1 minute and 21 seconds
......@@ -19,15 +19,6 @@ metrics.
1. Database
* [Per-table tuple stats](lib/gitlab_monitor/database/tuple_stats.rb) --
`pg_stat_table_*`
* [Slow queries](lib/gitlab_monitor/database/slow_queries.rb) --
`pg_slow_queries_total`
* [Stuck queries in idle in transaction](lib/gitlab_monitor/database/stuck_idle_in_transactions.rb) --
`pg_stuck_idle_in_transactions_total`
* [Blocked queries](lib/gitlab_monitor/database/blocked_queries.rb) --
`pg_blocked_queries_total`
* [Vacuum queries](lib/gitlab_monitor/database/vacuum_queries.rb) --
`pg_vacuum_count`, `pg_vacuum_age_in_seconds`, `pg_vacuum_analyze_count`,
`pg_vacuum_analyze_age_in_seconds`
* [Row count queries](lib/gitlab_monitor/database/row_count.rb) --
`db_rows_count`
* [CI builds](lib/gitlab_monitor/database/ci_builds.rb) --
......
......@@ -34,9 +34,6 @@ probes:
# by probe definitions as usual.
database:
multiple: true
blocked_queries:
class_name: Database::BlockedQueriesProber
<<: *db_common
ci_builds:
class_name: Database::CiBuildsProber
<<: *db_common
......@@ -46,15 +43,6 @@ probes:
rows_count:
class_name: Database::RowCountProber
<<: *db_common
slow_queries:
class_name: Database::SlowQueriesProber
<<: *db_common
stuck_idle_in_transactions:
class_name: Database::StuckIdleInTransactionsProber
<<: *db_common
vacuum_queries:
class_name: Database::VacuumQueriesProber
<<: *db_common
process:
methods:
......
......@@ -118,132 +118,6 @@ module GitLab
end
end
# Database blocked queries runner.
#
# It will take a database connection string and print results to STDOUT
class DatabaseBlockedQueries
COMMAND_NAME = "db-blocked-queries".freeze
def initialize(args)
@options = options(args)
@options.parse!
@target = args.shift || STDOUT
@target = File.open(@target, "a") if @target.is_a?(String)
end
def options(args)
args.options do |opts|
opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
@db_connection_string = val
end
end
end
def help
@options.help
end
def run
validate!
::GitLab::Monitor::Database::BlockedQueriesProber.new(connection_string: @db_connection_string)
.probe_db
.write_to(@target)
end
private
def validate!
fail InvalidCLICommand.new(help) unless @db_connection_string
end
end
# Database slow queries runner.
#
# It will take a database connection string and print results to STDOUT
class DatabaseSlowQueries
COMMAND_NAME = "db-slow-queries".freeze
def initialize(args)
@options = options(args)
@options.parse!
@target = args.shift || STDOUT
@target = File.open(@target, "a") if @target.is_a?(String)
end
def options(args)
args.options do |opts|
opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
@db_connection_string = val
end
end
end
def help
@options.help
end
def run
validate!
::GitLab::Monitor::Database::SlowQueriesProber.new(connection_string: @db_connection_string)
.probe_db
.write_to(@target)
end
private
def validate!
fail InvalidCLICommand.new(help) unless @db_connection_string
end
end
# Database vacuum queries runner.
#
# It will take a database connection string and print results to STDOUT
class DatabaseVacuumQueries
COMMAND_NAME = "db-vacuum-queries".freeze
def initialize(args)
@options = options(args)
@options.parse!
@target = args.shift || STDOUT
@target = File.open(@target, "a") if @target.is_a?(String)
end
def options(args)
args.options do |opts|
opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
@db_connection_string = val
end
end
end
def help
@options.help
end
def run
validate!
::GitLab::Monitor::Database::VacuumQueriesProber.new(connection_string: @db_connection_string)
.probe_db
.write_to(@target)
end
private
def validate!
fail InvalidCLICommand.new(help) unless @db_connection_string
end
end
# Database row counts query runner.
#
# This will take the database connection and print the result to STDOUT
......@@ -452,9 +326,6 @@ module GitLab
[
GIT,
DatabaseTupleStats,
DatabaseBlockedQueries,
DatabaseSlowQueries,
DatabaseVacuumQueries,
DatabaseRowCounts,
Process,
GitProcess,
......
......@@ -3,12 +3,8 @@ module GitLab
# Database-related classes
module Database
autoload :Base, "gitlab_monitor/database/base"
autoload :BlockedQueriesProber, "gitlab_monitor/database/blocked_queries"
autoload :CiBuildsProber, "gitlab_monitor/database/ci_builds"
autoload :SlowQueriesProber, "gitlab_monitor/database/slow_queries"
autoload :StuckIdleInTransactionsProber, "gitlab_monitor/database/stuck_idle_in_transactions"
autoload :TuplesProber, "gitlab_monitor/database/tuple_stats"
autoload :VacuumQueriesProber, "gitlab_monitor/database/vacuum_queries"
autoload :RowCountProber, "gitlab_monitor/database/row_count"
end
end
......
module GitLab
module Monitor
module Database
# A helper class to collect blocked queries
#
# It takes a connection string (e.g. "dbname=test port=5432")
class BlockedQueriesCollector < Base
def run
stats = Hash.new(0)
result = connection.exec(
<<-SQL
SELECT blocked.relation::regclass
FROM pg_catalog.pg_locks blocked
WHERE NOT blocked.granted
SQL
).to_a
result.each do |row|
stats[row["relation"] || "__transaction__"] += 1
end
stats
end
end
# Probes the DB specified by opts[:connection_string] for blocked queries, then converts them to metrics
class BlockedQueriesProber
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = BlockedQueriesCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
result = @collector.run
result.each do |table_name, blocked_count|
@metrics.add("pg_blocked_queries_total", blocked_count, table_name: table_name)
end
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
module GitLab
module Monitor
module Database
# A helper class to collect blocked queries
# A helper class to collect CI builds metrics.
class CiBuildsCollector < Base # rubocop:disable ClassLength
BUILDS_QUERY =
<<~SQL.freeze
......
module GitLab
module Monitor
module Database
# A helper class to collect slow queries
#
# It takes a connection string (e.g. "dbname=test port=5432")
class SlowQueriesCollector < Base
def run
result = connection.exec(
<<-SQL
SELECT COUNT(*) AS slow_count
FROM pg_stat_activity
WHERE state = 'active' AND (now() - query_start) > '1 seconds'::interval;
SQL
)
result[0]["slow_count"].to_i
end
end
# Probes the DB specified by opts[:connection_string] for slow queries, then converts them to metrics
class SlowQueriesProber
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = SlowQueriesCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
@metrics.add("pg_slow_queries_total", @collector.run)
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
module GitLab
module Monitor
module Database
# A helper class to collect queries stuck in idle in transaction
#
# It takes a connection string (e.g. "dbname=test port=5432")
class StuckIdleInTransactionsCollector < Base
def run
result = connection.exec(
<<-SQL
SELECT COUNT(*) AS stuck_count
FROM pg_stat_activity
WHERE state = 'idle in transaction' AND (now() - query_start) > '10 minutes'::interval;
SQL
)
result[0]["stuck_count"].to_i
end
end
# Probes the DB specified by opts[:connection_string] for slow queries, then converts them to metrics
class StuckIdleInTransactionsProber
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = StuckIdleInTransactionsCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
@metrics.add("pg_stuck_idle_in_transactions_total", @collector.run)
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
module GitLab
module Monitor
module Database
# A helper class to collect vacuum and vacuum analyze information
#
# It takes a connection string (e.g. "dbname=test port=5432")
class VacuumQueriesCollector < Base
def run
stats = {}
{ vacuum: "VACUUM (?!ANALYZE)", vacuum_analyze: "VACUUM ANALYZE" }.each do |query_name, condition|
result = connection.exec(
<<-SQL
SELECT COUNT(*) AS query_count, MAX(EXTRACT(EPOCH FROM (clock_timestamp() - query_start))) AS query_duration
FROM pg_catalog.pg_stat_activity
WHERE state = 'active' AND trim(query) ~* '\\A#{condition}';
SQL
)
stats[query_name] = { "total" => result[0]["query_count"].to_i,
"max_age" => result[0]["query_duration"].to_f }
end
stats
end
end
# Probes the DB specified by opts[:connection_string] for vacuum and vacuum analyze information,
# then converts them to metrics
class VacuumQueriesProber
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = VacuumQueriesCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
result = @collector.run
@metrics.add("pg_vacuum_count", result[:vacuum]["total"])
@metrics.add("pg_vacuum_age_in_seconds", result[:vacuum]["max_age"])
@metrics.add("pg_vacuum_analyze_count", result[:vacuum_analyze]["total"])
@metrics.add("pg_vacuum_analyze_age_in_seconds", result[:vacuum_analyze]["max_age"])
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment