Commit 4e5a1c0f authored by Ahmad Sherif's avatar Ahmad Sherif
Browse files

Merge branch 'sh-process-smaps' into 'master'

Add support for parsing process smaps

See merge request gitlab-org/gitlab-monitor!80
parents 0b574f14 685cf2cb
Pipeline #100013 passed with stage
in 37 seconds
......@@ -14,7 +14,7 @@ GEM
specs:
ast (2.4.0)
connection_pool (2.2.2)
diff-lcs (1.2.5)
diff-lcs (1.3)
parser (2.5.1.0)
ast (~> 2.4.0)
pg (0.18.4)
......@@ -27,19 +27,19 @@ GEM
redis (3.3.5)
redis-namespace (1.5.3)
redis (~> 3.0, >= 3.0.4)
rspec (3.5.0)
rspec-core (~> 3.5.0)
rspec-expectations (~> 3.5.0)
rspec-mocks (~> 3.5.0)
rspec-core (3.5.1)
rspec-support (~> 3.5.0)
rspec-expectations (3.5.0)
rspec (3.7.0)
rspec-core (~> 3.7.0)
rspec-expectations (~> 3.7.0)
rspec-mocks (~> 3.7.0)
rspec-core (3.7.1)
rspec-support (~> 3.7.0)
rspec-expectations (3.7.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.5.0)
rspec-mocks (3.5.0)
rspec-support (~> 3.7.0)
rspec-mocks (3.7.0)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.5.0)
rspec-support (3.5.0)
rspec-support (~> 3.7.0)
rspec-support (3.7.1)
rubocop (0.42.0)
parser (>= 2.3.1.1, < 3.0)
powerpack (~> 0.1)
......@@ -64,7 +64,8 @@ PLATFORMS
DEPENDENCIES
gitlab-monitor!
rspec (~> 3.5)
rspec-expectations (~> 3.7.0)
rubocop (~> 0.42)
BUNDLED WITH
1.16.4
1.17.1
......@@ -32,7 +32,19 @@ metrics.
* CPU time -- `process_cpu_seconds_total`
* Start time -- `process_start_time_seconds`
* Count -- `process_count`
* Memory usage -- `process_resident_memory_bytes`, `process_virtual_memory_bytes`
* Memory usage
* Data from /proc/<pid>/cmdline:
* `process_resident_memory_bytes`
* `process_virtual_memory_bytes`
* Data from /proc/<pid>/smaps:
* `process_smaps_size_bytes`
* `process_smaps_rss_bytes`
* `process_smaps_shared_clean_bytes`
* `process_smaps_shared_dirty_bytes`
* `process_smaps_private_clean_bytes`
* `process_smaps_private_dirty_bytes`
* `process_smaps_swap_bytes`
* `process_smaps_pss_bytes`
1. [Sidekiq](lib/gitlab_monitor/sidekiq.rb) -- `sidekiq_queue_size`, `sidekiq_queue_paused`,
`sidekiq_queue_latency_seconds`, `sidekiq_enqueued_jobs`, `sidekiq_dead_jobs`,
`sidekiq_running_jobs`, `sidekiq_to_be_retried_jobs`
......
......@@ -59,10 +59,11 @@ probes:
methods:
- probe_stat
- probe_count
- probe_smaps
opts:
- pid_or_pattern: "sidekiq .* \\[.*?\\]"
name: sidekiq
- pid_or_pattern: "unicorn worker\\[.*?\\]"
- pid_or_pattern: "unicorn.* worker\\[.*?\\]"
name: unicorn
- pid_or_pattern: "git-upload-pack --stateless-rpc"
name: git_upload_pack
......
......@@ -27,5 +27,6 @@ Gem::Specification.new do |s|
s.add_runtime_dependency "redis-namespace", "~> 1.5.2"
s.add_runtime_dependency "connection_pool", "~> 2.2.1"
s.add_development_dependency "rspec", "~> 3.3"
s.add_development_dependency "rspec", "~> 3.7.0"
s.add_development_dependency "rspec-expectations", "~> 3.7.0"
end
......@@ -238,6 +238,7 @@ module GitLab
::GitLab::Monitor::ProcessProber.new(pid_or_pattern: @pid || @pattern, name: @name, quantiles: @quantiles)
.probe_stat
.probe_count
.probe_smaps
.write_to(@target)
end
end
......
require_relative "memstats/mapping"
# Ported from https://github.com/discourse/discourse/blob/master/script/memstats.rb
#
# Aggregate Print useful information from /proc/[pid]/smaps
#
# pss - Roughly the amount of memory that is "really" being used by the pid
# swap - Amount of swap this process is currently using
#
# Reference:
# http://www.mjmwired.net/kernel/Documentation/filesystems/proc.txt#361
#
# Example:
# # ./memstats.rb 4386
# Process: 4386
# Command Line: /usr/bin/mongod -f /etc/mongo/mongod.conf
# Memory Summary:
# private_clean 107,132 kB
# private_dirty 2,020,676 kB
# pss 2,127,860 kB
# rss 2,128,536 kB
# shared_clean 728 kB
# shared_dirty 0 kB
# size 149,281,668 kB
# swap 1,719,792 kB
module GitLab
module Monitor
module MemStats
# Aggregates all metrics for a single PID in /proc/<pid>/smaps
class Aggregator
attr_accessor :pid, :totals
def initialize(pid)
@pid = pid
@totals = Hash.new(0)
@mappings = []
@valid = true
populate_info
end
def valid?
@valid
end
private
attr_accessor :mappings
def consume_mapping(map_lines, totals)
m = Mapping.new(map_lines)
Mapping::FIELDS.each do |field|
totals[field] += m.send(field)
end
m
end
def create_memstats_not_available(totals)
Mapping::FIELDS.each do |field|
totals[field] += Float::NAN
end
end
def populate_info # rubocop:disable Metrics/MethodLength
File.open("/proc/#{@pid}/smaps") do |smaps|
map_lines = []
loop do
break if smaps.eof?
line = smaps.readline.strip
case line
when /\w+:\s+/
map_lines << line
when /[0-9a-f]+:[0-9a-f]+\s+/
mappings << consume_mapping(map_lines, totals) if map_lines.size.positive?
map_lines.clear
map_lines << line
else
break
end
end
end
rescue => e
puts "Error: #{e}"
@valid = false
create_memstats_not_available(totals)
end
end
end
end
end
module GitLab
module Monitor
module MemStats
# Parses one entry in /proc/[pid]/smaps. For example:
#
# 00400000-00401000 r-xp 00000000 08:01 541055 /opt/gitlab/embedded/bin/ruby
# Size: 4 kB
# Rss: 4 kB
# Pss: 0 kB
# Shared_Clean: 4 kB
# Shared_Dirty: 0 kB
# Private_Clean: 0 kB
# Private_Dirty: 0 kB
# Referenced: 4 kB
# Anonymous: 0 kB
# AnonHugePages: 0 kB
# Shared_Hugetlb: 0 kB
# Private_Hugetlb: 0 kB
# Swap: 0 kB
# SwapPss: 0 kB
# KernelPageSize: 4 kB
# MMUPageSize: 4 kB
# Locked: 0 kB
# VmFlags: rd ex mr mw me dw sd
class Mapping
FIELDS = %w(size rss shared_clean shared_dirty private_clean private_dirty swap pss).freeze
attr_reader :address_start
attr_reader :address_end
attr_reader :perms
attr_reader :offset
attr_reader :device_major
attr_reader :device_minor
attr_reader :inode
attr_reader :region
attr_accessor :size
attr_accessor :rss
attr_accessor :shared_clean
attr_accessor :shared_dirty
attr_accessor :private_dirty
attr_accessor :private_clean
attr_accessor :swap
attr_accessor :pss
def initialize(lines)
FIELDS.each do |field|
send("#{field}=", 0)
end
parse_first_line(lines.shift)
lines.each do |l|
parse_field_line(l)
end
end
def parse_first_line(line)
parts = line.strip.split
@address_start, @address_end = parts[0].split("-")
@perms = parts[1]
@offset = parts[2]
@device_major, @device_minor = parts[3].split(":")
@inode = parts[4]
@region = parts[5] || "anonymous"
end
def parse_field_line(line)
parts = line.strip.split
field = parts[0].downcase.sub(":", "")
return unless respond_to? "#{field}="
value = Float(parts[1]).to_i
send("#{field}=", value)
end
end
end
end
end
require_relative "memstats"
module GitLab
module Monitor
# A helper class to extract memory info from /proc/<pid>/status
......@@ -89,6 +91,27 @@ module GitLab
self
end
def probe_smaps
@pids.each do |pid|
stats = ::GitLab::Monitor::MemStats::Aggregator.new(pid)
next unless stats.valid?
labels = { name: @name.downcase }
labels[:pid] = pid unless @use_quantiles
::GitLab::Monitor::MemStats::Mapping::FIELDS.each do |field|
value = stats.totals[field]
if value >= 0
@metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels)
end
end
end
self
end
def write_to(target)
target.write(@metrics.to_s)
end
......
# frozen_string_literal: true
module GitLab
module Monitor
# Simple time wrapper that provides a to_i and wraps the execution result
......@@ -48,10 +50,19 @@ module GitLab
module_function :deep_transform_keys_in_object
def pgrep(pattern)
`pgrep -f "#{pattern}"`.split("\n")
# pgrep will include the PID of the shell, so strip that out
exec_pgrep(pattern).split("\n").each_with_object([]) do |line, arr|
pid, name = line.split(" ")
arr << pid if name != "sh"
end
end
module_function :pgrep
def exec_pgrep(pattern)
`pgrep -fl "#{pattern}"`
end
module_function :exec_pgrep
def system_uptime
File.read("/proc/uptime").split(" ")[0].to_f
end
......
This diff is collapsed.
require "spec_helper"
require "gitlab_monitor/memstats"
describe GitLab::Monitor::MemStats do
let(:pid) { 100 }
let(:smaps_data) { File.open("spec/fixtures/smaps/sample.txt") }
subject { described_class::Aggregator.new(pid) }
before do
expect(File).to receive(:open).with("/proc/#{pid}/smaps").and_yield(smaps_data)
end
it "parses the data properly" do
expect(subject.valid?).to be_truthy
nonzero_fields = %w(size rss shared_clean shared_dirty private_dirty pss)
zero_fields = %w(private_clean swap)
nonzero_fields.each do |field|
expect(subject.totals[field]).to be > 0 # rubocop:disable Style/NumericPredicate
end
zero_fields.each do |field|
expect(subject.totals[field]).to eq(0)
end
end
end
......@@ -5,3 +5,11 @@ describe GitLab::Monitor::TimeTracker do
expect(subject.track { sleep 0.1 }.time).to satisfy { |v| v >= 0.1 }
end
end
describe GitLab::Monitor::Utils do
it "excludes extraneous PIDs" do
allow(described_class).to receive(:exec_pgrep).and_return("12345 my-process\n98765 sh\n")
expect(described_class.pgrep("some-process")).to eq(["12345"])
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment