Commit a1d2d83f authored by John Skarbek's avatar John Skarbek Committed by Craig Barrett

Changes from `sum` to `rate` for sidekiq exceptions

parent a0bda0f4
groups:
- name: sidekiq-exception-count.rules
rules:
- alert: SidekiqExceptionCountTooHigh
expr: sum by (controller,environment) (gitlab_transaction_event_sidekiq_exception_total{controller!="RepositoryUpdateMirrorWorker"}) > 2000
for: 1h
labels:
severity: warn
annotations:
description: There have been over 2000 Sidekiq exceptions for {{ $labels.controller }} in {{ $labels.environment }} for the last hour.
Check https://dashboards.gitlab.net/dashboard/db/sidekiq-stats?panelId=66&fullscreen&orgId=1&from=now-6h&to=now.
title: 'Large number of Sidekiq exceptions for {{ $labels.controller }} in {{ $labels.environment }}: {{$value}}'
- alert: RepositoryMirrorUpdateWorkerExceptionCountTooHigh
expr: sum by (controller,environment) (gitlab_transaction_event_sidekiq_exception_total{controller="RepositoryUpdateMirrorWorker"}) > 10000
for: 1h
labels:
severity: warn
annotations:
description: There have been over 10000 Sidekiq exceptions for {{ $labels.controller }} in {{ $labels.environment }} for the last hour.
Check https://dashboards.gitlab.net/dashboard/db/sidekiq-stats?panelId=66&fullscreen&orgId=1&from=now-6h&to=now.
title: 'Large number of Sidekiq exceptions for {{ $labels.controller }} in {{ $labels.environment }}: {{$value}}'
runbook: troubleshooting/sidekiq_exception_count.md
groups:
- name: sidekiq-exception-rate.rules
rules:
- alert: SidekiqExceptionRateTooHigh
expr: |
sum(rate(gitlab_transaction_event_sidekiq_exception_total[1m])) by (controller,environment) > 1
for: 5m
labels:
severity: warn
annotations:
description: |
The rate of Sidekiq {{ $labels.controller }} Exceptions in {{ $labels.environment }} is high.
Check https://dashboards.gitlab.net/d/9GOIu9Siz/sidekiq-stats?panelId=58&orgId=1&from=1541138657538&to=1541160257538&fullscreen&var-environment=gprd&var-prometheus=prometheus-01-inf-gprd&var-app_prometheus=prometheus-app-01-inf-gprd
title: 'High Rate of Sidekiq {{ $labels.controller }} exceptions in {{ $labels.environment }}: {{$value}}'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment