Commit ac634db8 authored by Pablo Carranza's avatar Pablo Carranza
Browse files

Merge branch 'feature/web-scraping' into 'master'

Add a web server to allow web scraping



See merge request !6
parents 024b6f42 e9dc3cd6
......@@ -15,3 +15,6 @@ Style/SignalException:
Style/RaiseArgs:
EnforcedStyle: compact
Metrics/MethodLength:
Max: 15
source "https://rubygems.org"
gemspec
group :test do
gem "rspec", "~>3.5"
gem "rubocop", "~>0.42"
......
......@@ -3,6 +3,7 @@ PATH
specs:
gitlab-monitor (0.0.1)
pg (~> 0.18.4)
sinatra (~> 1.4.7)
GEM
remote: https://rubygems.org/
......@@ -13,6 +14,9 @@ GEM
ast (~> 2.2)
pg (0.18.4)
powerpack (0.1.1)
rack (1.6.4)
rack-protection (1.5.3)
rack
rainbow (2.1.0)
rspec (3.5.0)
rspec-core (~> 3.5.0)
......@@ -34,6 +38,11 @@ GEM
ruby-progressbar (~> 1.7)
unicode-display_width (~> 1.0, >= 1.0.1)
ruby-progressbar (1.8.1)
sinatra (1.4.7)
rack (~> 1.5)
rack-protection (~> 1.4)
tilt (>= 1.3, < 3)
tilt (2.0.5)
unicode-display_width (1.1.0)
PLATFORMS
......
probes:
dead_tuples_count:
class_name: DatabaseDeadTuplesProber
methods:
- probe_db
opts:
connection_string: dbname=gitlabhq_development user=postgres
git:
methods:
- probe_pull
- probe_push
opts:
source: /home/git/repo
......@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
s.license = "MIT"
s.add_runtime_dependency "pg", "~> 0.18.4"
s.add_runtime_dependency "sinatra", "~> 1.4.7"
s.add_development_dependency "rspec", "~> 3.3"
end
......@@ -3,9 +3,11 @@ module GitLab
module Monitor
autoload :CLI, "gitlab_monitor/cli"
autoload :TimeTracker, "gitlab_monitor/util"
autoload :Utils, "gitlab_monitor/util"
autoload :PrometheusMetrics, "gitlab_monitor/prometheus"
autoload :Git, "gitlab_monitor/git"
autoload :GitProber, "gitlab_monitor/git"
autoload :Database, "gitlab_monitor/database"
autoload :WebExporter, "gitlab_monitor/web_exporter"
end
end
require "yaml"
module GitLab
module Monitor
# Stores runner classes in a single place
......@@ -48,7 +50,7 @@ module GitLab
def run
validate!
::GitLab::Monitor::GitProber.new(self)
::GitLab::Monitor::GitProber.new(labels: labels, source: source)
.probe_pull
.probe_push
.write_to(@target)
......@@ -200,12 +202,51 @@ module GitLab
end
end
# Run a web server that exposes the metrics specified in a config file
class Server
COMMAND_NAME = "web".freeze
def initialize(args)
@options = options(args)
@options.parse!
end
def options(args)
args.options do |opts|
opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
opts.on("-c config.yml", "Monitoring config") do |val|
@config_file = val
end
end
end
def help
@options.help
end
def run
validate!
config = Utils.deep_symbolize_hash_keys(YAML.load_file(@config_file))
WebExporter.setup(config)
WebExporter.run!
end
private
def validate!
fail InvalidCLICommand.new(help) unless @config_file
end
end
def self.commands
[
GIT,
DatabaseDeadTuples,
DatabaseBlockedQueries,
DatabaseSlowQueries
DatabaseSlowQueries,
Server
].each_with_object({}) do |command_class, commands|
commands[command_class::COMMAND_NAME] = command_class
commands
......
......@@ -20,7 +20,7 @@ module GitLab
# Probes the DB specified by opts[:connection_string] for dead tubles stats, then converts them to metrics
class DeadTuplesProber
def initialize(opts, metrics = PrometheusMetrics.new)
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@collector = DeadTuplesCollector.new(connection_string: opts[:connection_string])
end
......
......@@ -62,10 +62,10 @@ module GitLab
# Optionally takes a metrics object which by default is a PrometheusMetrics, useful to change the
# metrics writer to something else.
class GitProber
def initialize(options, metrics: PrometheusMetrics.new)
def initialize(opts, metrics: PrometheusMetrics.new)
@metrics = metrics
@labels = options.labels || {}
@git = Git.new(options.source)
@labels = opts[:labels] || {}
@git = Git.new(opts[:source])
end
def probe_pull
......
......@@ -7,8 +7,9 @@ module GitLab
#
# The add method also can take any arbitrary amount of labels in a `key: value` format.
class PrometheusMetrics
def initialize
def initialize(include_timestamp: true)
@metrics = Hash.new { |h, k| h[k] = [] }
@include_timestamp = include_timestamp
end
def add(name, value, **labels)
......@@ -17,16 +18,18 @@ module GitLab
end
def to_s
buffer = StringIO.new
buffer = ""
@metrics.each do |name, measurements|
measurements.each do |measurement|
buffer.write(name.to_s)
buffer << name.to_s
labels = (measurement[:labels] || {}).map { |label, value| "#{label}=\"#{value}\"" }.join(",")
buffer.write("{#{labels}}") unless labels.empty?
buffer.write(" #{measurement[:value]} #{measurement[:timestamp]}\n")
buffer << "{#{labels}}" unless labels.empty?
buffer << " #{measurement[:value]}"
buffer << " #{measurement[:timestamp]}" if @include_timestamp
buffer << "\n"
end
end
buffer.string
buffer
end
end
end
......
......@@ -18,5 +18,34 @@ module GitLab
TrackedResult.new(result, Time.now.to_f - @start)
end
end
# Stuff that's copied from ActiveSupport
module Utils
def camel_case_string(str)
str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable PerlBackrefs
end
module_function :camel_case_string
def deep_symbolize_hash_keys(hash)
deep_transform_keys_in_object(hash, &:to_sym)
end
module_function :deep_symbolize_hash_keys
def deep_transform_keys_in_object(object, &block)
case object
when Hash
object.keys.each do |key|
value = object.delete(key)
object[yield(key)] = deep_transform_keys_in_object(value, &block)
end
object
when Array
object.map! { |e| deep_transform_keys_in_object(e, &block) }
else
object
end
end
module_function :deep_transform_keys_in_object
end
end
end
require "sinatra/base"
module GitLab
module Monitor
# Metrics web exporter
class WebExporter < Sinatra::Base
class << self
def setup(config)
config[:probes].each do |probe_name, params|
prober_class_name = params.delete(:class_name) || Utils.camel_case_string("#{probe_name}_prober")
prober_class = GitLab::Monitor.const_get(prober_class_name)
get "/#{probe_name}" do
prober = prober_class.new(params[:opts], metrics: PrometheusMetrics.new(include_timestamp: false))
params[:methods].each do |meth|
prober.send(meth)
end
prober.write_to(response)
end
end
end
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment