Add Global stats

config/gitlab-exporter.yml.example

@@ -60,6 +60,9 @@ probes:
         - soft_deleted_projects
         - orphaned_projects
         - uploads
+        - users
+        - projects
+        - groups
     remote_mirrors:
       class_name: Database::RemoteMirrorsProber
       <<: *db_common

lib/gitlab_exporter/database/row_count.rb

@@ -79,7 +79,40 @@ module GitLab
             joins: "LEFT JOIN namespaces ON projects.namespace_id = namespaces.id",
             where: "namespaces.id IS NULL"
           },
-          uploads: { select: :uploads }
+          uploads: { select: :uploads },
+          users: {
+            select: :users,
+            joins: "LEFT JOIN
+              (
+                SELECT
+                  members.user_id,
+                  MAX(access_level) as access_level
+                FROM members
+                GROUP BY members.user_id
+              ) AS u
+              ON users.id = u.user_id",
+            where: "ghost IS NULL AND bot_type IS NULL",
+            fields: {
+              admin:        {},
+              external:     {},
+              state:        {},
+              access_level: { definition: "COALESCE(u.access_level, 0)" }
+            }
+          },
+          projects: {
+            select: :projects,
+            fields: {
+              visibility_level: {},
+              archived:         {}
+            }
+          },
+          groups: {
+            select: :namespaces,
+            fields: {
+              visibility_level: {},
+              root:             { definition: "(parent_id IS NULL)" }
+            }
+          }
         }.freeze
 
         def initialize(args)
@@ -105,21 +138,37 @@ module GitLab
 
         def count_from_query_hash(query_hash)
           result = execute(construct_query(query_hash))
-          return 0 unless result
+          return [{ "count": 0, "labels": {} }] unless result
 
-          result[0]["count"]
+          result.map do |row|
+            labels = {}
+            (query_hash[:fields] || []).each do |key, _| labels[key] = row[key.to_s] end
+            { "count": row["count"], "labels": labels }
+          end
         end
 
         def successful_check?(query)
           result = execute("SELECT EXISTS (#{query})")
           return unless result
 
-          result[0]["exists"] == "t"
+          result[0]["exists"]
         end
 
         def execute(query)
           with_connection_pool do |conn|
-            conn.exec(query)
+            tm = PG::BasicTypeMapForResults.new(conn)
+
+            # Remove warning message:
+            # Warning: no type cast defined for type "name" with oid 19.
+            # Please cast this type explicitly to TEXT to be safe for future changes.
+            # Warning: no type cast defined for type "regproc" with oid 24.
+            # Please cast this type explicitly to TEXT to be safe for future changes.
+            [{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
+              old_coder = tm.coders.find { |c| c.name == value[:type] }
+              tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
+            end
+
+            conn.exec(query).map_types!(tm)
           end
         rescue PG::UndefinedTable, PG::UndefinedColumn
           nil
@@ -127,9 +176,16 @@ module GitLab
 
         # Not private so I can test it without meta programming tricks
         def construct_query(query)
-          query_string = "SELECT COUNT(*) FROM #{query[:select]} "
-          query_string << "#{query[:joins]} "       if query[:joins]
-          query_string << "WHERE #{query[:where]}"  if query[:where]
+          query_string = "SELECT COUNT(*)"
+          (query[:fields] || []).each do |key, value|
+            query_string << ", "
+            query_string << "(#{value[:definition]}) AS " if value[:definition]
+            query_string << key.to_s
+          end
+          query_string << " FROM #{query[:select]}"
+          query_string << " #{query[:joins]}"       if query[:joins]
+          query_string << " WHERE #{query[:where]}" if query[:where]
+          query_string << " GROUP BY " + query[:fields].keys.join(", ") if query[:fields]
           query_string << ";"
         end
       end
@@ -146,8 +202,11 @@ module GitLab
 
         def probe_db
           results = @collector.run
-          results.each do |key, value|
-            @metrics.add("gitlab_database_rows", value.to_f, query_name: key.to_s)
+          results.each do |query_name, result|
+            labels = { query_name: query_name.to_s }
+            result.each do |row|
+              @metrics.add("gitlab_database_rows", row[:count].to_f, **labels, **row[:labels])
+            end
           end
 
           self

spec/database/row_count_spec.rb

@@ -4,7 +4,8 @@ require "gitlab_exporter/database/row_count"
 describe GitLab::Exporter::Database::RowCountCollector do
   let(:query) {
     { project_1: { select: :projects, where: "id=1" },
-      project_2: { select: :projects, where: "id=2" } }
+      project_2: { select: :projects, where: "id=2" },
+      project_3: { select: :projects, fields: { is_public: { definition: "visibility_level == 20" } } } }
   }
   let(:collector) { described_class.new(connection_string: "host=localhost") }
 
@@ -12,19 +13,31 @@ describe GitLab::Exporter::Database::RowCountCollector do
     before do
       stub_const("GitLab::Exporter::Database::RowCountCollector::QUERIES", query)
 
-      allow(collector).to receive(:count_from_query_hash).with(query[:project_1]).and_return(3)
-      allow(collector).to receive(:count_from_query_hash).with(query[:project_2]).and_return(6)
+      allow(collector).to receive(:execute).with("SELECT COUNT(*) FROM projects WHERE id=1;").and_return(false)
+      allow(collector).to receive(:execute).with("SELECT COUNT(*) FROM projects WHERE id=2;").and_return(
+        [{ "count" => 6 }]
+      )
+      allow(collector).to receive(:execute).with(
+        "SELECT COUNT(*), (visibility_level == 20) AS is_public FROM projects GROUP BY is_public;"
+      ).and_return(
+        [{ "count" => 3, "is_public" => true }, { "count" => 6, "is_public" => false }]
+      )
     end
 
     it "executes all the queries" do
-      expect(collector.run).to eq(project_1: 3, project_2: 6)
+      expect(collector.run).to eq(
+        project_1: [{ count: 0, labels: {} }],
+        project_2: [{ count: 6, labels: {} }],
+        project_3: [{ count: 3, labels: { is_public: true  } },
+                    { count: 6, labels: { is_public: false } }]
+      )
     end
 
     context "when selected_queries is passed" do
       let(:collector) { described_class.new(connection_string: "host=localhost", selected_queries: ["project_2"]) }
 
       it "executes the selected queries" do
-        expect(collector.run).to eq(project_2: 6)
+        expect(collector.run).to eq(project_2: [{ count: 6, labels: {} }])
       end
     end
   end
@@ -33,5 +46,10 @@ describe GitLab::Exporter::Database::RowCountCollector do
     it "accepts a table and where clause" do
       expect(collector.send(:construct_query, query[:project_1])).to eq "SELECT COUNT(*) FROM projects WHERE id=1;"
     end
+
+    it "accepts a table and group (field) clause" do
+      expect(collector.send(:construct_query, query[:project_3])).to eq \
+        "SELECT COUNT(*), (visibility_level == 20) AS is_public FROM projects GROUP BY is_public;"
+    end
   end
 end