Allow returning quantiles for process-related probes

Closes #4

Gemfile.lock

 PATH
   remote: .
   specs:
-    gitlab-monitor (0.0.6)
+    gitlab-monitor (0.0.8)
       pg (~> 0.18.4)
+      quantile (~> 0.2.0)
       sinatra (~> 1.4.7)
 
 GEM
@@ -14,6 +15,7 @@ GEM
       ast (~> 2.2)
     pg (0.18.4)
     powerpack (0.1.1)
+    quantile (0.2.0)
     rack (1.6.4)
     rack-protection (1.5.3)
       rack

config/gitlab-monitor.yml.example

@@ -28,3 +28,4 @@ probes:
         name: unicorn
       - pid_or_pattern: "git-upload-pack --stateless-rpc"
         name: git_upload_pack
+        quantiles: true

gitlab-monitor.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "pg", "~> 0.18.4"
   s.add_runtime_dependency "sinatra", "~> 1.4.7"
+  s.add_runtime_dependency "quantile", "~> 0.2.0"
 
   s.add_development_dependency "rspec", "~> 3.3"
 end

lib/gitlab_monitor/cli.rb

@@ -308,6 +308,9 @@ module GitLab
             opts.on("--name=NAME", "Process name to be used in metrics") do |val|
               @name = val
             end
+            opts.on("--quantiles", "Return quantiles instead of exact metrics") do
+              @quantiles = true
+            end
           end
         end
 
@@ -316,7 +319,7 @@ module GitLab
         end
 
         def run
-          ::GitLab::Monitor::ProcessProber.new(pid_or_pattern: @pid || @pattern, name: @name)
+          ::GitLab::Monitor::ProcessProber.new(pid_or_pattern: @pid || @pattern, name: @name, quantiles: @quantiles)
                                           .probe_memory
                                           .probe_age
                                           .write_to(@target)

lib/gitlab_monitor/process.rb

@@ -75,6 +75,7 @@ module GitLab
                    else
                      Utils.pgrep(options[:pid_or_pattern])
                    end
+        @use_quantiles = options.fetch(:quantiles, false)
       end
 
       def probe_memory
@@ -85,7 +86,7 @@ module GitLab
             value = memory[field]
             next unless value
 
-            @metrics.add("process_memory_bytes", value, name: @name.downcase, pid: pid, field: field)
+            @metrics.add("process_memory_bytes", value, @use_quantiles, name: @name.downcase, field: field)
           end
         end
 
@@ -97,7 +98,7 @@ module GitLab
           stats = ProcessStats.new(pid)
           next unless stats.valid?
 
-          @metrics.add("process_age_seconds", stats.age, name: @name.downcase, pid: pid)
+          @metrics.add("process_age_seconds", stats.age, @use_quantiles, name: @name.downcase)
         end
 
         self

lib/gitlab_monitor/prometheus.rb

+require "quantile"
+
 module GitLab
   module Monitor
     # Prometheus metrics container
@@ -9,15 +11,23 @@ module GitLab
     class PrometheusMetrics
       def initialize(include_timestamp: true)
         @metrics = Hash.new { |h, k| h[k] = [] }
+        @quantiles = Hash.new { |h, k| h[k] = [] }
         @include_timestamp = include_timestamp
       end
 
-      def add(name, value, **labels)
-        @metrics[name] << { value: value, labels: labels, timestamp: (Time.now.to_f * 1000).to_i }
+      def add(name, value, quantile = false, **labels)
+        if quantile
+          @quantiles[{ name: name, labels: labels }] << value
+        else
+          @metrics[name] << { value: value, labels: labels, timestamp: (Time.now.to_f * 1000).to_i }
+        end
+
         self
       end
 
-      def to_s
+      def to_s # rubocop:disable Metrics/AbcSize
+        add_quantiles_to_metrics
+
         buffer = ""
         @metrics.each do |name, measurements|
           measurements.each do |measurement|
@@ -31,6 +41,24 @@ module GitLab
         end
         buffer
       end
+
+      private
+
+      def add_quantiles_to_metrics # rubocop:disable Metrics/AbcSize
+        @quantiles.each do |data, measurements|
+          estimator = Quantile::Estimator.new
+
+          measurements.each do |value|
+            estimator.observe(value)
+          end
+
+          estimator.invariants.each do |invariant|
+            data[:labels][:quantile] = "#{(invariant.quantile * 100).to_i}th"
+
+            add(data[:name], estimator.query(invariant.quantile), **data[:labels])
+          end
+        end
+      end
     end
   end
 end