Commit 4468f2b9 authored by Ahmad Sherif's avatar Ahmad Sherif
Browse files

Merge branch 'sh-add-monitoring-for-stuck-queries' into 'master'

Add monitoring for queries that are idle in transaction state too long

See merge request !48
parents ee9b5486 7d8d4a58
......@@ -21,6 +21,8 @@ metrics.
`pg_stat_table_*`
* [Slow queries](lib/gitlab_monitor/database/slow_queries.rb) --
`pg_slow_queries_total`
* [Stuck queries in idle in transaction](lib/gitlab_monitor/database/stuck_idle_in_transactions.rb) --
`pg_stuck_idle_in_transactions_total`
* [Blocked queries](lib/gitlab_monitor/database/blocked_queries.rb) --
`pg_blocked_queries_total`
* [Vacuum queries](lib/gitlab_monitor/database/vacuum_queries.rb) --
......
......@@ -49,6 +49,9 @@ probes:
slow_queries:
class_name: Database::SlowQueriesProber
<<: *db_common
stuck_idle_in_transactions:
class_name: Database::StuckIdleInTransactionsProber
<<: *db_common
vacuum_queries:
class_name: Database::VacuumQueriesProber
<<: *db_common
......
......@@ -6,6 +6,7 @@ module GitLab
autoload :BlockedQueriesProber, "gitlab_monitor/database/blocked_queries"
autoload :CiBuildsProber, "gitlab_monitor/database/ci_builds"
autoload :SlowQueriesProber, "gitlab_monitor/database/slow_queries"
autoload :StuckIdleInTransactionsProber, "gitlab_monitor/database/stuck_idle_in_transactions"
autoload :TuplesProber, "gitlab_monitor/database/tuple_stats"
autoload :VacuumQueriesProber, "gitlab_monitor/database/vacuum_queries"
autoload :RowCountProber, "gitlab_monitor/database/row_count"
......
module GitLab
module Monitor
module Database
# A helper class to collect queries stuck in idle in transaction
#
# It takes a connection string (e.g. "dbname=test port=5432")
class StuckIdleInTransactionsCollector < Base
def run
result = connection.exec(
<<-SQL
SELECT COUNT(*) AS stuck_count
FROM pg_stat_activity
WHERE state = 'idle in transaction' AND (now() - query_start) > '10 minutes'::interval;
SQL
)
result[0]["stuck_count"].to_i
end
end
# Probes the DB specified by opts[:connection_string] for slow queries, then converts them to metrics
class StuckIdleInTransactionsProber
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = StuckIdleInTransactionsCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
@metrics.add("pg_stuck_idle_in_transactions_total", @collector.run)
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment