Commit b151058a authored by Frédéric PLANCHON's avatar Frédéric PLANCHON Committed by Ahmad Sherif

Add Global stats

parent 128fba92
......@@ -60,6 +60,9 @@ probes:
- soft_deleted_projects
- orphaned_projects
- uploads
- users
- projects
- groups
remote_mirrors:
class_name: Database::RemoteMirrorsProber
<<: *db_common
......
......@@ -79,7 +79,40 @@ module GitLab
joins: "LEFT JOIN namespaces ON projects.namespace_id = namespaces.id",
where: "namespaces.id IS NULL"
},
uploads: { select: :uploads }
uploads: { select: :uploads },
users: {
select: :users,
joins: "LEFT JOIN
(
SELECT
members.user_id,
MAX(access_level) as access_level
FROM members
GROUP BY members.user_id
) AS u
ON users.id = u.user_id",
where: "ghost IS NULL AND bot_type IS NULL",
fields: {
admin: {},
external: {},
state: {},
access_level: { definition: "COALESCE(u.access_level, 0)" }
}
},
projects: {
select: :projects,
fields: {
visibility_level: {},
archived: {}
}
},
groups: {
select: :namespaces,
fields: {
visibility_level: {},
root: { definition: "(parent_id IS NULL)" }
}
}
}.freeze
def initialize(args)
......@@ -105,21 +138,37 @@ module GitLab
def count_from_query_hash(query_hash)
result = execute(construct_query(query_hash))
return 0 unless result
return [{ "count": 0, "labels": {} }] unless result
result[0]["count"]
result.map do |row|
labels = {}
(query_hash[:fields] || []).each do |key, _| labels[key] = row[key.to_s] end
{ "count": row["count"], "labels": labels }
end
end
def successful_check?(query)
result = execute("SELECT EXISTS (#{query})")
return unless result
result[0]["exists"] == "t"
result[0]["exists"]
end
def execute(query)
with_connection_pool do |conn|
conn.exec(query)
tm = PG::BasicTypeMapForResults.new(conn)
# Remove warning message:
# Warning: no type cast defined for type "name" with oid 19.
# Please cast this type explicitly to TEXT to be safe for future changes.
# Warning: no type cast defined for type "regproc" with oid 24.
# Please cast this type explicitly to TEXT to be safe for future changes.
[{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
old_coder = tm.coders.find { |c| c.name == value[:type] }
tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
end
conn.exec(query).map_types!(tm)
end
rescue PG::UndefinedTable, PG::UndefinedColumn
nil
......@@ -127,9 +176,16 @@ module GitLab
# Not private so I can test it without meta programming tricks
def construct_query(query)
query_string = "SELECT COUNT(*) FROM #{query[:select]} "
query_string << "#{query[:joins]} " if query[:joins]
query_string << "WHERE #{query[:where]}" if query[:where]
query_string = "SELECT COUNT(*)"
(query[:fields] || []).each do |key, value|
query_string << ", "
query_string << "(#{value[:definition]}) AS " if value[:definition]
query_string << key.to_s
end
query_string << " FROM #{query[:select]}"
query_string << " #{query[:joins]}" if query[:joins]
query_string << " WHERE #{query[:where]}" if query[:where]
query_string << " GROUP BY " + query[:fields].keys.join(", ") if query[:fields]
query_string << ";"
end
end
......@@ -146,8 +202,11 @@ module GitLab
def probe_db
results = @collector.run
results.each do |key, value|
@metrics.add("gitlab_database_rows", value.to_f, query_name: key.to_s)
results.each do |query_name, result|
labels = { query_name: query_name.to_s }
result.each do |row|
@metrics.add("gitlab_database_rows", row[:count].to_f, **labels, **row[:labels])
end
end
self
......
......@@ -4,7 +4,8 @@ require "gitlab_exporter/database/row_count"
describe GitLab::Exporter::Database::RowCountCollector do
let(:query) {
{ project_1: { select: :projects, where: "id=1" },
project_2: { select: :projects, where: "id=2" } }
project_2: { select: :projects, where: "id=2" },
project_3: { select: :projects, fields: { is_public: { definition: "visibility_level == 20" } } } }
}
let(:collector) { described_class.new(connection_string: "host=localhost") }
......@@ -12,19 +13,31 @@ describe GitLab::Exporter::Database::RowCountCollector do
before do
stub_const("GitLab::Exporter::Database::RowCountCollector::QUERIES", query)
allow(collector).to receive(:count_from_query_hash).with(query[:project_1]).and_return(3)
allow(collector).to receive(:count_from_query_hash).with(query[:project_2]).and_return(6)
allow(collector).to receive(:execute).with("SELECT COUNT(*) FROM projects WHERE id=1;").and_return(false)
allow(collector).to receive(:execute).with("SELECT COUNT(*) FROM projects WHERE id=2;").and_return(
[{ "count" => 6 }]
)
allow(collector).to receive(:execute).with(
"SELECT COUNT(*), (visibility_level == 20) AS is_public FROM projects GROUP BY is_public;"
).and_return(
[{ "count" => 3, "is_public" => true }, { "count" => 6, "is_public" => false }]
)
end
it "executes all the queries" do
expect(collector.run).to eq(project_1: 3, project_2: 6)
expect(collector.run).to eq(
project_1: [{ count: 0, labels: {} }],
project_2: [{ count: 6, labels: {} }],
project_3: [{ count: 3, labels: { is_public: true } },
{ count: 6, labels: { is_public: false } }]
)
end
context "when selected_queries is passed" do
let(:collector) { described_class.new(connection_string: "host=localhost", selected_queries: ["project_2"]) }
it "executes the selected queries" do
expect(collector.run).to eq(project_2: 6)
expect(collector.run).to eq(project_2: [{ count: 6, labels: {} }])
end
end
end
......@@ -33,5 +46,10 @@ describe GitLab::Exporter::Database::RowCountCollector do
it "accepts a table and where clause" do
expect(collector.send(:construct_query, query[:project_1])).to eq "SELECT COUNT(*) FROM projects WHERE id=1;"
end
it "accepts a table and group (field) clause" do
expect(collector.send(:construct_query, query[:project_3])).to eq \
"SELECT COUNT(*), (visibility_level == 20) AS is_public FROM projects GROUP BY is_public;"
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment