Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
G
gitlab-exporter
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
gitlab-org
gitlab-exporter
Commits
b151058a
Commit
b151058a
authored
Feb 13, 2020
by
Frédéric PLANCHON
Committed by
Ahmad Sherif
Feb 13, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add Global stats
parent
128fba92
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
95 additions
and
15 deletions
+95
-15
config/gitlab-exporter.yml.example
config/gitlab-exporter.yml.example
+3
-0
lib/gitlab_exporter/database/row_count.rb
lib/gitlab_exporter/database/row_count.rb
+69
-10
spec/database/row_count_spec.rb
spec/database/row_count_spec.rb
+23
-5
No files found.
config/gitlab-exporter.yml.example
View file @
b151058a
...
...
@@ -60,6 +60,9 @@ probes:
- soft_deleted_projects
- orphaned_projects
- uploads
- users
- projects
- groups
remote_mirrors:
class_name: Database::RemoteMirrorsProber
<<: *db_common
...
...
lib/gitlab_exporter/database/row_count.rb
View file @
b151058a
...
...
@@ -79,7 +79,40 @@ module GitLab
joins:
"LEFT JOIN namespaces ON projects.namespace_id = namespaces.id"
,
where:
"namespaces.id IS NULL"
},
uploads:
{
select: :uploads
}
uploads:
{
select: :uploads
},
users:
{
select: :users
,
joins:
"LEFT JOIN
(
SELECT
members.user_id,
MAX(access_level) as access_level
FROM members
GROUP BY members.user_id
) AS u
ON users.id = u.user_id"
,
where:
"ghost IS NULL AND bot_type IS NULL"
,
fields:
{
admin:
{},
external:
{},
state:
{},
access_level:
{
definition:
"COALESCE(u.access_level, 0)"
}
}
},
projects:
{
select: :projects
,
fields:
{
visibility_level:
{},
archived:
{}
}
},
groups:
{
select: :namespaces
,
fields:
{
visibility_level:
{},
root:
{
definition:
"(parent_id IS NULL)"
}
}
}
}.
freeze
def
initialize
(
args
)
...
...
@@ -105,21 +138,37 @@ module GitLab
def
count_from_query_hash
(
query_hash
)
result
=
execute
(
construct_query
(
query_hash
))
return
0
unless
result
return
[{
"count"
:
0
,
"labels"
:
{}
}]
unless
result
result
[
0
][
"count"
]
result
.
map
do
|
row
|
labels
=
{}
(
query_hash
[
:fields
]
||
[]).
each
do
|
key
,
_
|
labels
[
key
]
=
row
[
key
.
to_s
]
end
{
"count"
:
row
[
"count"
],
"labels"
:
labels
}
end
end
def
successful_check?
(
query
)
result
=
execute
(
"SELECT EXISTS (
#{
query
}
)"
)
return
unless
result
result
[
0
][
"exists"
]
==
"t"
result
[
0
][
"exists"
]
end
def
execute
(
query
)
with_connection_pool
do
|
conn
|
conn
.
exec
(
query
)
tm
=
PG
::
BasicTypeMapForResults
.
new
(
conn
)
# Remove warning message:
# Warning: no type cast defined for type "name" with oid 19.
# Please cast this type explicitly to TEXT to be safe for future changes.
# Warning: no type cast defined for type "regproc" with oid 24.
# Please cast this type explicitly to TEXT to be safe for future changes.
[{
"type"
:
"text"
,
"oid"
:
19
},
{
"type"
:
"int4"
,
"oid"
:
24
}].
each
do
|
value
|
old_coder
=
tm
.
coders
.
find
{
|
c
|
c
.
name
==
value
[
:type
]
}
tm
.
add_coder
(
old_coder
.
dup
.
tap
{
|
c
|
c
.
oid
=
value
[
:oid
]
})
end
conn
.
exec
(
query
).
map_types!
(
tm
)
end
rescue
PG
::
UndefinedTable
,
PG
::
UndefinedColumn
nil
...
...
@@ -127,9 +176,16 @@ module GitLab
# Not private so I can test it without meta programming tricks
def
construct_query
(
query
)
query_string
=
"SELECT COUNT(*) FROM
#{
query
[
:select
]
}
"
query_string
<<
"
#{
query
[
:joins
]
}
"
if
query
[
:joins
]
query_string
<<
"WHERE
#{
query
[
:where
]
}
"
if
query
[
:where
]
query_string
=
"SELECT COUNT(*)"
(
query
[
:fields
]
||
[]).
each
do
|
key
,
value
|
query_string
<<
", "
query_string
<<
"(
#{
value
[
:definition
]
}
) AS "
if
value
[
:definition
]
query_string
<<
key
.
to_s
end
query_string
<<
" FROM
#{
query
[
:select
]
}
"
query_string
<<
"
#{
query
[
:joins
]
}
"
if
query
[
:joins
]
query_string
<<
" WHERE
#{
query
[
:where
]
}
"
if
query
[
:where
]
query_string
<<
" GROUP BY "
+
query
[
:fields
].
keys
.
join
(
", "
)
if
query
[
:fields
]
query_string
<<
";"
end
end
...
...
@@ -146,8 +202,11 @@ module GitLab
def
probe_db
results
=
@collector
.
run
results
.
each
do
|
key
,
value
|
@metrics
.
add
(
"gitlab_database_rows"
,
value
.
to_f
,
query_name:
key
.
to_s
)
results
.
each
do
|
query_name
,
result
|
labels
=
{
query_name:
query_name
.
to_s
}
result
.
each
do
|
row
|
@metrics
.
add
(
"gitlab_database_rows"
,
row
[
:count
].
to_f
,
**
labels
,
**
row
[
:labels
])
end
end
self
...
...
spec/database/row_count_spec.rb
View file @
b151058a
...
...
@@ -4,7 +4,8 @@ require "gitlab_exporter/database/row_count"
describe
GitLab
::
Exporter
::
Database
::
RowCountCollector
do
let
(
:query
)
{
{
project_1:
{
select: :projects
,
where:
"id=1"
},
project_2:
{
select: :projects
,
where:
"id=2"
}
}
project_2:
{
select: :projects
,
where:
"id=2"
},
project_3:
{
select: :projects
,
fields:
{
is_public:
{
definition:
"visibility_level == 20"
}
}
}
}
}
let
(
:collector
)
{
described_class
.
new
(
connection_string:
"host=localhost"
)
}
...
...
@@ -12,19 +13,31 @@ describe GitLab::Exporter::Database::RowCountCollector do
before
do
stub_const
(
"GitLab::Exporter::Database::RowCountCollector::QUERIES"
,
query
)
allow
(
collector
).
to
receive
(
:count_from_query_hash
).
with
(
query
[
:project_1
]).
and_return
(
3
)
allow
(
collector
).
to
receive
(
:count_from_query_hash
).
with
(
query
[
:project_2
]).
and_return
(
6
)
allow
(
collector
).
to
receive
(
:execute
).
with
(
"SELECT COUNT(*) FROM projects WHERE id=1;"
).
and_return
(
false
)
allow
(
collector
).
to
receive
(
:execute
).
with
(
"SELECT COUNT(*) FROM projects WHERE id=2;"
).
and_return
(
[{
"count"
=>
6
}]
)
allow
(
collector
).
to
receive
(
:execute
).
with
(
"SELECT COUNT(*), (visibility_level == 20) AS is_public FROM projects GROUP BY is_public;"
).
and_return
(
[{
"count"
=>
3
,
"is_public"
=>
true
},
{
"count"
=>
6
,
"is_public"
=>
false
}]
)
end
it
"executes all the queries"
do
expect
(
collector
.
run
).
to
eq
(
project_1:
3
,
project_2:
6
)
expect
(
collector
.
run
).
to
eq
(
project_1:
[{
count:
0
,
labels:
{}
}],
project_2:
[{
count:
6
,
labels:
{}
}],
project_3:
[{
count:
3
,
labels:
{
is_public:
true
}
},
{
count:
6
,
labels:
{
is_public:
false
}
}]
)
end
context
"when selected_queries is passed"
do
let
(
:collector
)
{
described_class
.
new
(
connection_string:
"host=localhost"
,
selected_queries:
[
"project_2"
])
}
it
"executes the selected queries"
do
expect
(
collector
.
run
).
to
eq
(
project_2:
6
)
expect
(
collector
.
run
).
to
eq
(
project_2:
[{
count:
6
,
labels:
{}
}]
)
end
end
end
...
...
@@ -33,5 +46,10 @@ describe GitLab::Exporter::Database::RowCountCollector do
it
"accepts a table and where clause"
do
expect
(
collector
.
send
(
:construct_query
,
query
[
:project_1
])).
to
eq
"SELECT COUNT(*) FROM projects WHERE id=1;"
end
it
"accepts a table and group (field) clause"
do
expect
(
collector
.
send
(
:construct_query
,
query
[
:project_3
])).
to
eq
\
"SELECT COUNT(*), (visibility_level == 20) AS is_public FROM projects GROUP BY is_public;"
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment