Add a web server to allow web scraping

Gemfile

 source "https://rubygems.org"
 
+gemspec
+
 group :test do
   gem "rspec", "~>3.5"
   gem "rubocop", "~>0.42"

Gemfile.lock

@@ -3,6 +3,7 @@ PATH
   specs:
     gitlab-monitor (0.0.1)
       pg (~> 0.18.4)
+      sinatra (~> 1.4.7)
 
 GEM
   remote: https://rubygems.org/
@@ -13,6 +14,9 @@ GEM
       ast (~> 2.2)
     pg (0.18.4)
     powerpack (0.1.1)
+    rack (1.6.4)
+    rack-protection (1.5.3)
+      rack
     rainbow (2.1.0)
     rspec (3.5.0)
       rspec-core (~> 3.5.0)
@@ -34,6 +38,11 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
+    sinatra (1.4.7)
+      rack (~> 1.5)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    tilt (2.0.5)
     unicode-display_width (1.1.0)
 
 PLATFORMS

config/gitlab-monitor.yml.example

+probes:
+  dead_tuples_count:
+    class_name: DatabaseDeadTuplesProber
+    methods:
+      - probe_db
+    opts:
+      connection_string: dbname=gitlabhq_development user=postgres
+  git:
+    methods:
+      - probe_pull
+      - probe_push
+    opts:
+      source: /home/git/repo

gitlab-monitor.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.license       = "MIT"
 
   s.add_runtime_dependency "pg", "~> 0.18.4"
+  s.add_runtime_dependency "sinatra", "~> 1.4.7"
 
   s.add_development_dependency "rspec", "~> 3.3"
 end

lib/gitlab_monitor.rb

@@ -3,9 +3,11 @@ module GitLab
   module Monitor
     autoload :CLI,               "gitlab_monitor/cli"
     autoload :TimeTracker,       "gitlab_monitor/util"
+    autoload :Utils,             "gitlab_monitor/util"
     autoload :PrometheusMetrics, "gitlab_monitor/prometheus"
     autoload :Git,               "gitlab_monitor/git"
     autoload :GitProber,         "gitlab_monitor/git"
     autoload :Database,          "gitlab_monitor/database"
+    autoload :WebExporter,       "gitlab_monitor/web_exporter"
   end
 end

lib/gitlab_monitor/cli.rb

+require "yaml"
+
 module GitLab
   module Monitor
     # Stores runner classes in a single place
@@ -200,12 +202,51 @@ module GitLab
         end
       end
 
+      # Run a web server that exposes the metrics specified in a config file
+      class Server
+        COMMAND_NAME = "web".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("-c config.yml", "Monitoring config") do |val|
+              @config_file = val
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          validate!
+
+          config = Utils.deep_symbolize_hash_keys(YAML.load_file(@config_file))
+
+          WebExporter.setup(config)
+          WebExporter.run!
+        end
+
+        private
+
+        def validate!
+          fail InvalidCLICommand.new(help) unless @config_file
+        end
+      end
+
       def self.commands
         [
           GIT,
           DatabaseDeadTuples,
           DatabaseBlockedQueries,
-          DatabaseSlowQueries
+          DatabaseSlowQueries,
+          Server
         ].each_with_object({}) do |command_class, commands|
           commands[command_class::COMMAND_NAME] = command_class
           commands

lib/gitlab_monitor/prometheus.rb

@@ -7,8 +7,9 @@ module GitLab
     #
     # The add method also can take any arbitrary amount of labels in a `key: value` format.
     class PrometheusMetrics
-      def initialize
+      def initialize(include_timestamp: true)
         @metrics = Hash.new { |h, k| h[k] = [] }
+        @include_timestamp = include_timestamp
       end
 
       def add(name, value, **labels)
@@ -23,7 +24,9 @@ module GitLab
             buffer.write(name.to_s)
             labels = (measurement[:labels] || {}).map { |label, value| "#{label}=\"#{value}\"" }.join(",")
             buffer.write("{#{labels}}") unless labels.empty?
-            buffer.write(" #{measurement[:value]} #{measurement[:timestamp]}\n")
+            buffer.write(" #{measurement[:value]}")
+            buffer.write(" #{measurement[:timestamp]}") if @include_timestamp
+            buffer.write("\n")
           end
         end
         buffer.string

lib/gitlab_monitor/util.rb

@@ -18,5 +18,34 @@ module GitLab
         TrackedResult.new(result, Time.now.to_f - @start)
       end
     end
+
+    # Stuff that's copied from ActiveSupport
+    module Utils
+      def camel_case_string(str)
+        str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable PerlBackrefs
+      end
+      module_function :camel_case_string
+
+      def deep_symbolize_hash_keys(hash)
+        deep_transform_keys_in_object(hash, &:to_sym)
+      end
+      module_function :deep_symbolize_hash_keys
+
+      def deep_transform_keys_in_object(object, &block)
+        case object
+        when Hash
+          object.keys.each do |key|
+            value = object.delete(key)
+            object[yield(key)] = deep_transform_keys_in_object(value, &block)
+          end
+          object
+        when Array
+          object.map! { |e| deep_transform_keys_in_object(e, &block) }
+        else
+          object
+        end
+      end
+      module_function :deep_transform_keys_in_object
+    end
   end
 end

lib/gitlab_monitor/web_exporter.rb

+require "sinatra/base"
+
+module GitLab
+  module Monitor
+    # Metrics web exporter
+    class WebExporter < Sinatra::Base
+      class << self
+        def setup(config)
+          config[:probes].each do |probe_name, params|
+            prober_class_name = params.delete(:class_name) || Utils.camel_case_string("#{probe_name}_prober")
+            prober_class = GitLab::Monitor.const_get(prober_class_name)
+
+            get "/#{probe_name}" do
+              prober = prober_class.new(params[:opts], metrics: PrometheusMetrics.new(include_timestamp: false))
+              params[:methods].each do |meth|
+                prober.send(meth)
+              end
+              prober.write_to(response)
+            end
+          end
+        end
+      end
+    end
+  end
+end