Merge branch 'sh-process-smaps' into 'master'

Add support for parsing process smaps

See merge request gitlab-org/gitlab-monitor!80

Gemfile.lock

@@ -14,7 +14,7 @@ GEM
   specs:
     ast (2.4.0)
     connection_pool (2.2.2)
-    diff-lcs (1.2.5)
+    diff-lcs (1.3)
     parser (2.5.1.0)
       ast (~> 2.4.0)
     pg (0.18.4)
@@ -27,19 +27,19 @@ GEM
     redis (3.3.5)
     redis-namespace (1.5.3)
       redis (~> 3.0, >= 3.0.4)
-    rspec (3.5.0)
-      rspec-core (~> 3.5.0)
-      rspec-expectations (~> 3.5.0)
-      rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.1)
-      rspec-support (~> 3.5.0)
-    rspec-expectations (3.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-mocks (3.5.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-support (3.5.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
     rubocop (0.42.0)
       parser (>= 2.3.1.1, < 3.0)
       powerpack (~> 0.1)
@@ -64,7 +64,8 @@ PLATFORMS
 DEPENDENCIES
   gitlab-monitor!
   rspec (~> 3.5)
+  rspec-expectations (~> 3.7.0)
   rubocop (~> 0.42)
 
 BUNDLED WITH
-   1.16.4
+   1.17.1

README.md

@@ -32,7 +32,19 @@ metrics.
     * CPU time -- `process_cpu_seconds_total`
     * Start time -- `process_start_time_seconds`
     * Count -- `process_count`
-    * Memory usage -- `process_resident_memory_bytes`, `process_virtual_memory_bytes`
+    * Memory usage
+      * Data from /proc/<pid>/cmdline:
+        * `process_resident_memory_bytes`
+        * `process_virtual_memory_bytes`
+      * Data from /proc/<pid>/smaps:
+        * `process_smaps_size_bytes`
+        * `process_smaps_rss_bytes`
+        * `process_smaps_shared_clean_bytes`
+        * `process_smaps_shared_dirty_bytes`
+        * `process_smaps_private_clean_bytes`
+        * `process_smaps_private_dirty_bytes`
+        * `process_smaps_swap_bytes`
+        * `process_smaps_pss_bytes`
 1. [Sidekiq](lib/gitlab_monitor/sidekiq.rb) -- `sidekiq_queue_size`, `sidekiq_queue_paused`,
    `sidekiq_queue_latency_seconds`, `sidekiq_enqueued_jobs`, `sidekiq_dead_jobs`,
    `sidekiq_running_jobs`, `sidekiq_to_be_retried_jobs`

config/gitlab-monitor.yml.example

@@ -59,10 +59,11 @@ probes:
     methods:
       - probe_stat
       - probe_count
+      - probe_smaps
     opts:
       - pid_or_pattern: "sidekiq .* \\[.*?\\]"
         name: sidekiq
-      - pid_or_pattern: "unicorn worker\\[.*?\\]"
+      - pid_or_pattern: "unicorn.* worker\\[.*?\\]"
         name: unicorn
       - pid_or_pattern: "git-upload-pack --stateless-rpc"
         name: git_upload_pack

gitlab-monitor.gemspec

@@ -27,5 +27,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "redis-namespace", "~> 1.5.2"
   s.add_runtime_dependency "connection_pool", "~> 2.2.1"
 
-  s.add_development_dependency "rspec", "~> 3.3"
+  s.add_development_dependency "rspec", "~> 3.7.0"
+  s.add_development_dependency "rspec-expectations", "~> 3.7.0"
 end

lib/gitlab_monitor/cli.rb

@@ -238,6 +238,7 @@ module GitLab
           ::GitLab::Monitor::ProcessProber.new(pid_or_pattern: @pid || @pattern, name: @name, quantiles: @quantiles)
                                           .probe_stat
                                           .probe_count
+                                          .probe_smaps
                                           .write_to(@target)
         end
       end

lib/gitlab_monitor/memstats.rb

+require_relative "memstats/mapping"
+
+# Ported from https://github.com/discourse/discourse/blob/master/script/memstats.rb
+#
+# Aggregate Print useful information from /proc/[pid]/smaps
+#
+# pss  - Roughly the amount of memory that is "really" being used by the pid
+# swap - Amount of swap this process is currently using
+#
+# Reference:
+#  http://www.mjmwired.net/kernel/Documentation/filesystems/proc.txt#361
+#
+# Example:
+#   # ./memstats.rb 4386
+#   Process:             4386
+#   Command Line:        /usr/bin/mongod -f /etc/mongo/mongod.conf
+#   Memory Summary:
+#     private_clean             107,132 kB
+#     private_dirty           2,020,676 kB
+#     pss                     2,127,860 kB
+#     rss                     2,128,536 kB
+#     shared_clean                  728 kB
+#     shared_dirty                    0 kB
+#     size                  149,281,668 kB
+#     swap                    1,719,792 kB
+module GitLab
+  module Monitor
+    module MemStats
+      # Aggregates all metrics for a single PID in /proc/<pid>/smaps
+      class Aggregator
+        attr_accessor :pid, :totals
+
+        def initialize(pid)
+          @pid = pid
+          @totals = Hash.new(0)
+          @mappings = []
+          @valid = true
+
+          populate_info
+        end
+
+        def valid?
+          @valid
+        end
+
+        private
+
+        attr_accessor :mappings
+
+        def consume_mapping(map_lines, totals)
+          m = Mapping.new(map_lines)
+
+          Mapping::FIELDS.each do |field|
+            totals[field] += m.send(field)
+          end
+
+          m
+        end
+
+        def create_memstats_not_available(totals)
+          Mapping::FIELDS.each do |field|
+            totals[field] += Float::NAN
+          end
+        end
+
+        def populate_info # rubocop:disable Metrics/MethodLength
+          File.open("/proc/#{@pid}/smaps") do |smaps|
+            map_lines = []
+
+            loop do
+              break if smaps.eof?
+
+              line = smaps.readline.strip
+
+              case line
+              when /\w+:\s+/
+                map_lines << line
+              when /[0-9a-f]+:[0-9a-f]+\s+/
+                mappings << consume_mapping(map_lines, totals) if map_lines.size.positive?
+
+                map_lines.clear
+                map_lines << line
+              else
+                break
+              end
+            end
+          end
+        rescue => e
+          puts "Error: #{e}"
+          @valid = false
+          create_memstats_not_available(totals)
+        end
+      end
+    end
+  end
+end

lib/gitlab_monitor/memstats/mapping.rb

+module GitLab
+  module Monitor
+    module MemStats
+      # Parses one entry in /proc/[pid]/smaps. For example:
+      #
+      # 00400000-00401000 r-xp 00000000 08:01 541055                             /opt/gitlab/embedded/bin/ruby
+      # Size:                  4 kB
+      # Rss:                   4 kB
+      # Pss:                   0 kB
+      # Shared_Clean:          4 kB
+      # Shared_Dirty:          0 kB
+      # Private_Clean:         0 kB
+      # Private_Dirty:         0 kB
+      # Referenced:            4 kB
+      # Anonymous:             0 kB
+      # AnonHugePages:         0 kB
+      # Shared_Hugetlb:        0 kB
+      # Private_Hugetlb:       0 kB
+      # Swap:                  0 kB
+      # SwapPss:               0 kB
+      # KernelPageSize:        4 kB
+      # MMUPageSize:           4 kB
+      # Locked:                0 kB
+      # VmFlags: rd ex mr mw me dw sd
+      class Mapping
+        FIELDS = %w(size rss shared_clean shared_dirty private_clean private_dirty swap pss).freeze
+
+        attr_reader :address_start
+        attr_reader :address_end
+        attr_reader :perms
+        attr_reader :offset
+        attr_reader :device_major
+        attr_reader :device_minor
+        attr_reader :inode
+        attr_reader :region
+
+        attr_accessor :size
+        attr_accessor :rss
+        attr_accessor :shared_clean
+        attr_accessor :shared_dirty
+        attr_accessor :private_dirty
+        attr_accessor :private_clean
+        attr_accessor :swap
+        attr_accessor :pss
+
+        def initialize(lines)
+          FIELDS.each do |field|
+            send("#{field}=", 0)
+          end
+
+          parse_first_line(lines.shift)
+
+          lines.each do |l|
+            parse_field_line(l)
+          end
+        end
+
+        def parse_first_line(line)
+          parts = line.strip.split
+          @address_start, @address_end = parts[0].split("-")
+          @perms = parts[1]
+          @offset = parts[2]
+          @device_major, @device_minor = parts[3].split(":")
+          @inode = parts[4]
+          @region = parts[5] || "anonymous"
+        end
+
+        def parse_field_line(line)
+          parts = line.strip.split
+          field = parts[0].downcase.sub(":", "")
+
+          return unless respond_to? "#{field}="
+
+          value = Float(parts[1]).to_i
+          send("#{field}=", value)
+        end
+      end
+    end
+  end
+end

lib/gitlab_monitor/process.rb

+require_relative "memstats"
+
 module GitLab
   module Monitor
     # A helper class to extract memory info from /proc/<pid>/status
@@ -89,6 +91,27 @@ module GitLab
         self
       end
 
+      def probe_smaps
+        @pids.each do |pid|
+          stats = ::GitLab::Monitor::MemStats::Aggregator.new(pid)
+
+          next unless stats.valid?
+
+          labels = { name: @name.downcase }
+          labels[:pid] = pid unless @use_quantiles
+
+          ::GitLab::Monitor::MemStats::Mapping::FIELDS.each do |field|
+            value = stats.totals[field]
+
+            if value >= 0
+              @metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels)
+            end
+          end
+        end
+
+        self
+      end
+
       def write_to(target)
         target.write(@metrics.to_s)
       end

lib/gitlab_monitor/util.rb

+# frozen_string_literal: true
+
 module GitLab
   module Monitor
     # Simple time wrapper that provides a to_i and wraps the execution result
@@ -48,10 +50,19 @@ module GitLab
       module_function :deep_transform_keys_in_object
 
       def pgrep(pattern)
-        `pgrep -f "#{pattern}"`.split("\n")
+        # pgrep will include the PID of the shell, so strip that out
+        exec_pgrep(pattern).split("\n").each_with_object([]) do |line, arr|
+          pid, name = line.split(" ")
+          arr << pid if name != "sh"
+        end
       end
       module_function :pgrep
 
+      def exec_pgrep(pattern)
+        `pgrep -fl "#{pattern}"`
+      end
+      module_function :exec_pgrep
+
       def system_uptime
         File.read("/proc/uptime").split(" ")[0].to_f
       end

spec/memstats_spec.rb

+require "spec_helper"
+require "gitlab_monitor/memstats"
+
+describe GitLab::Monitor::MemStats do
+  let(:pid) { 100 }
+  let(:smaps_data) { File.open("spec/fixtures/smaps/sample.txt") }
+
+  subject { described_class::Aggregator.new(pid) }
+
+  before do
+    expect(File).to receive(:open).with("/proc/#{pid}/smaps").and_yield(smaps_data)
+  end
+
+  it "parses the data properly" do
+    expect(subject.valid?).to be_truthy
+
+    nonzero_fields = %w(size rss shared_clean shared_dirty private_dirty pss)
+    zero_fields = %w(private_clean swap)
+
+    nonzero_fields.each do |field|
+      expect(subject.totals[field]).to be > 0 # rubocop:disable Style/NumericPredicate
+    end
+
+    zero_fields.each do |field|
+      expect(subject.totals[field]).to eq(0)
+    end
+  end
+end

spec/util_spec.rb

@@ -5,3 +5,11 @@ describe GitLab::Monitor::TimeTracker do
     expect(subject.track { sleep 0.1 }.time).to satisfy { |v| v >= 0.1 }
   end
 end
+
+describe GitLab::Monitor::Utils do
+  it "excludes extraneous PIDs" do
+    allow(described_class).to receive(:exec_pgrep).and_return("12345 my-process\n98765 sh\n")
+
+    expect(described_class.pgrep("some-process")).to eq(["12345"])
+  end
+end