Commit 024b6f42 authored by Pablo Carranza's avatar Pablo Carranza
Browse files

Merge branch 'feature/monitor-queries-used-in-runbook' into 'master'

Monitor queries used in runbooks



See merge request !7
parents 27d81152 c6e254ef
......@@ -6,7 +6,6 @@ module GitLab
autoload :PrometheusMetrics, "gitlab_monitor/prometheus"
autoload :Git, "gitlab_monitor/git"
autoload :GitProber, "gitlab_monitor/git"
autoload :DatabaseDeadTuplesCollector, "gitlab_monitor/database_dead_tuples"
autoload :DatabaseDeadTuplesProber, "gitlab_monitor/database_dead_tuples"
autoload :Database, "gitlab_monitor/database"
end
end
......@@ -104,9 +104,93 @@ module GitLab
def run
validate!
::GitLab::Monitor::DatabaseDeadTuplesProber.new(connection_string: @db_connection_string)
.probe_db
.write_to(@target)
::GitLab::Monitor::Database::DeadTuplesProber.new(connection_string: @db_connection_string)
.probe_db
.write_to(@target)
end
private
def validate!
fail InvalidCLICommand.new(help) unless @db_connection_string
end
end
# Database blocked queries runner.
#
# It will take a database connection string and print results to STDOUT
class DatabaseBlockedQueries
COMMAND_NAME = "db-blocked-queries".freeze
def initialize(args)
@options = options(args)
@options.parse!
@target = args.shift || STDOUT
@target = File.open(@target, "a") if @target.is_a?(String)
end
def options(args)
args.options do |opts|
opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
@db_connection_string = val
end
end
end
def help
@options.help
end
def run
validate!
::GitLab::Monitor::Database::BlockedQueriesProber.new(connection_string: @db_connection_string)
.probe_db
.write_to(@target)
end
private
def validate!
fail InvalidCLICommand.new(help) unless @db_connection_string
end
end
# Database slow queries runner.
#
# It will take a database connection string and print results to STDOUT
class DatabaseSlowQueries
COMMAND_NAME = "db-slow-queries".freeze
def initialize(args)
@options = options(args)
@options.parse!
@target = args.shift || STDOUT
@target = File.open(@target, "a") if @target.is_a?(String)
end
def options(args)
args.options do |opts|
opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
opts.on("--db-conn=\"dbname=test port=5432\"", "Database connection string") do |val|
@db_connection_string = val
end
end
end
def help
@options.help
end
def run
validate!
::GitLab::Monitor::Database::SlowQueriesProber.new(connection_string: @db_connection_string)
.probe_db
.write_to(@target)
end
private
......@@ -117,7 +201,12 @@ module GitLab
end
def self.commands
[GIT, DatabaseDeadTuples].each_with_object({}) do |command_class, commands|
[
GIT,
DatabaseDeadTuples,
DatabaseBlockedQueries,
DatabaseSlowQueries
].each_with_object({}) do |command_class, commands|
commands[command_class::COMMAND_NAME] = command_class
commands
end
......
module GitLab
module Monitor
# Database-related classes
module Database
autoload :Base, "gitlab_monitor/database/base"
autoload :BlockedQueriesProber, "gitlab_monitor/database/blocked_queries"
autoload :SlowQueriesProber, "gitlab_monitor/database/slow_queries"
autoload :DeadTuplesProber, "gitlab_monitor/database/dead_tuples"
end
end
end
require "pg"
module GitLab
module Monitor
module Database
# An abstract class for interacting with DB
#
# It takes a connection string (e.g. "dbname=test port=5432")
class Base
def initialize(args)
@connection_string = args[:connection_string]
end
def run
fail NotImplemented
end
def connected?
!connection.nil?
end
private
def connection
@connection ||= PG.connect(@connection_string)
rescue PG::ConnectionBad # rubocop:disable Lint/HandleExceptions
# Do nothing, we could be on the slave machine
end
end
end
end
end
module GitLab
module Monitor
module Database
# A helper class to collect blocked queries
#
# It takes a connection string (e.g. "dbname=test port=5432")
class BlockedQueriesCollector < Base
def run
stats = Hash.new(0)
result = connection.exec(
<<-SQL
SELECT blocked.relation::regclass
FROM pg_catalog.pg_locks blocked
WHERE NOT blocked.granted
SQL
).to_a
result.each do |row|
stats[row["relation"] || "__transaction__"] += 1
end
stats
end
end
# Probes the DB specified by opts[:connection_string] for blocked queries, then converts them to metrics
class BlockedQueriesProber
def initialize(opts, metrics = PrometheusMetrics.new)
@metrics = metrics
@collector = BlockedQueriesCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
result = @collector.run
result.each do |table_name, blocked_count|
@metrics.add("pg_blocked_queries_total", blocked_count, table_name: table_name)
end
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
module GitLab
module Monitor
module Database
# A helper class to collect dead tuples stats from the database
#
# It takes a connection string (e.g. "dbname=test port=5432")
class DeadTuplesCollector < Base
def run
table_names = connection.exec("SELECT tablename FROM pg_tables WHERE tableowner = 'gitlab'").values.flatten
stats = {}
table_names.each do |table_name|
result = connection.exec("SELECT n_dead_tup FROM pg_stat_user_tables WHERE relname = '#{table_name}'")
stats[table_name] = result[0]["n_dead_tup"].to_i
end
stats
end
end
# Probes the DB specified by opts[:connection_string] for dead tubles stats, then converts them to metrics
class DeadTuplesProber
def initialize(opts, metrics = PrometheusMetrics.new)
@metrics = metrics
@collector = DeadTuplesCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
result = @collector.run
result.each do |table_name, dead_tuples_count|
@metrics.add("pg_dead_tuples", dead_tuples_count, table_name: table_name)
end
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
module GitLab
module Monitor
module Database
# A helper class to collect slow queries
#
# It takes a connection string (e.g. "dbname=test port=5432")
class SlowQueriesCollector < Base
def run
result = connection.exec(
<<-SQL
SELECT COUNT(*) AS slow_count
FROM pg_stat_activity
WHERE state = 'active' AND (now() - query_start) > '1 seconds'::interval;
SQL
)
result[0]["slow_count"].to_i
end
end
# Probes the DB specified by opts[:connection_string] for slow queries, then converts them to metrics
class SlowQueriesProber
def initialize(opts, metrics = PrometheusMetrics.new)
@metrics = metrics
@collector = SlowQueriesCollector.new(connection_string: opts[:connection_string])
end
def probe_db
return self unless @collector.connected?
@metrics.add("pg_slow_queries_total", @collector.run)
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
end
require "pg"
module GitLab
module Monitor
# A helper class to collect dead tuples stats from the database
#
# It takes a connection string (e.g. "dbname=test port=5432")
class DatabaseDeadTuplesCollector
attr_reader :result
def initialize(args)
@connection_string = args[:connection_string]
end
def run
table_names = connection.exec("SELECT tablename FROM pg_tables WHERE tableowner = 'gitlab'").values.flatten
stats = {}
table_names.each do |table_name|
result = connection.exec("SELECT n_dead_tup FROM pg_stat_user_tables WHERE relname = '#{table_name}'")
stats[table_name] = result[0]["n_dead_tup"].to_i
end
stats
end
private
def connection
@connection ||= PG.connect(@connection_string)
end
end
# Probes the DB specified by opts[:connection_string] for dead tubles stats, then converts them to metrics
class DatabaseDeadTuplesProber
def initialize(opts, metrics = PrometheusMetrics.new)
@metrics = metrics
@collector = DatabaseDeadTuplesCollector.new(connection_string: opts[:connection_string])
end
def probe_db
result = @collector.run
result.each do |table_name, dead_tuples_count|
@metrics.add("pg_dead_tuples", dead_tuples_count, table_name: table_name)
end
self
end
def write_to(target)
target.write(@metrics.to_s)
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment