Add a process monitor

lib/gitlab_monitor.rb

@@ -5,9 +5,11 @@ module GitLab
     autoload :TimeTracker,       "gitlab_monitor/util"
     autoload :Utils,             "gitlab_monitor/util"
     autoload :PrometheusMetrics, "gitlab_monitor/prometheus"
+    autoload :Utils,             "gitlab_monitor/util"
     autoload :Git,               "gitlab_monitor/git"
     autoload :GitProber,         "gitlab_monitor/git"
     autoload :Database,          "gitlab_monitor/database"
+    autoload :ProcessProber,     "gitlab_monitor/process"
     autoload :WebExporter,       "gitlab_monitor/web_exporter"
   end
 end

lib/gitlab_monitor/cli.rb

@@ -240,12 +240,54 @@ module GitLab
         end
       end
 
+      # Process runner
+      #
+      # Takes a pid and name for metrics
+      class Process
+        COMMAND_NAME = "process".freeze
+
+        def initialize(args)
+          @options = options(args)
+          @options.parse!
+
+          @target = args.shift || STDOUT
+          @target = File.open(@target, "a") if @target.is_a?(String)
+        end
+
+        def options(args)
+          args.options do |opts|
+            opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
+            opts.on("--pid=123", "Process ID") do |val|
+              @pid = val
+            end
+            opts.on("--pattern=unicron", "Process command pattern") do |val|
+              @pattern = val
+            end
+            opts.on("--name=NAME", "Process name to be used in metrics") do |val|
+              @name = val
+            end
+          end
+        end
+
+        def help
+          @options.help
+        end
+
+        def run
+          ::GitLab::Monitor::ProcessProber.new(pid_or_pattern: @pid || @pattern, name: @name)
+                                          .probe_memory
+                                          .probe_age
+                                          .write_to(@target)
+        end
+      end
+
       def self.commands
         [
           GIT,
           DatabaseDeadTuples,
           DatabaseBlockedQueries,
           DatabaseSlowQueries,
+          Process,
           Server
         ].each_with_object({}) do |command_class, commands|
           commands[command_class::COMMAND_NAME] = command_class

lib/gitlab_monitor/process.rb

+module GitLab
+  module Monitor
+    # A helper class to extract memory info from /proc/<pid>/status
+    #
+    # It takes a pid
+    class ProcessMemory
+      def initialize(pid)
+        @pid  = pid
+        @info = {}
+
+        populate_info
+      end
+
+      def [](field)
+        @info[field]
+      end
+
+      private
+
+      def populate_info
+        status = File.read("/proc/#{@pid}/status")
+
+        status.each_line do |line|
+          line.strip!
+
+          next unless line.end_with?("kB") # We're only interested in sizes
+
+          match = line.match(/(\w+):\s*(\d+) kB/)
+          next unless match
+
+          @info[match[1]] = match[2].to_i * 1024
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+    end
+
+    # A helper class to stats from /proc/<pid>/stat
+    #
+    # It takes a pid
+    class ProcessStats
+      def initialize(pid)
+        @pid   = pid
+        @stats = nil
+
+        populate_info
+      end
+
+      def valid?
+        !@stats.nil?
+      end
+
+      def age
+        Utils.system_uptime - (@stats[21].to_i / Process.clock_getres(:TIMES_BASED_CLOCK_PROCESS_CPUTIME_ID, :hertz))
+      end
+
+      private
+
+      def populate_info
+        @stats = File.read("/proc/#{@pid}/stat").split(" ")
+      rescue Errno::ENOENT
+        nil
+      end
+    end
+
+    # Probes a process for info then writes metrics to a target
+    class ProcessProber
+      MEMORY_FIELDS = %w(RssFile RssAnon VmRss VmData).freeze
+
+      def initialize(options, metrics: PrometheusMetrics.new)
+        @metrics = metrics
+        @name    = options[:name]
+        @pids    = if options[:pid_or_pattern] =~ /^\d+$/
+                     [options[:pid_or_pattern]]
+                   else
+                     Utils.pgrep(options[:pid_or_pattern])
+                   end
+      end
+
+      def probe_memory
+        @pids.each do |pid|
+          memory = ProcessMemory.new(pid)
+
+          MEMORY_FIELDS.each do |field|
+            value = memory[field]
+            next unless value
+
+            @metrics.add("#{@name.downcase}_memory_bytes", value, pid: pid, field: field)
+          end
+        end
+
+        self
+      end
+
+      def probe_age
+        @pids.each do |pid|
+          stats = ProcessStats.new(pid)
+          next unless stats.valid?
+
+          @metrics.add("#{@name.downcase}_age_seconds", stats.age, pid: pid)
+        end
+
+        self
+      end
+
+      def write_to(target)
+        target.write(@metrics.to_s)
+      end
+    end
+  end
+end

lib/gitlab_monitor/util.rb

@@ -19,7 +19,7 @@ module GitLab
       end
     end
 
-    # Stuff that's copied from ActiveSupport
+    # Helper methods, some stuff was copied from ActiveSupport
     module Utils
       def camel_case_string(str)
         str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable PerlBackrefs
@@ -46,6 +46,16 @@ module GitLab
         end
       end
       module_function :deep_transform_keys_in_object
+
+      def pgrep(pattern)
+        `pgrep -f #{pattern}`.split("\n")
+      end
+      module_function :pgrep
+
+      def system_uptime
+        File.read("/proc/uptime").split(" ")[0].to_f
+      end
+      module_function :system_uptime
     end
   end
 end