Commit bbbb646a authored by Andrew Newdigate's avatar Andrew Newdigate

Send Gitaly abuse detection data to a new index for long-term analysis

parent 0bf1f8a7
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
curl -XPUT "${ES_URL}/gitaly-abuse-detection" -H 'Content-Type: application/json' -d'
{
"mappings" : {
"doc" : {
"properties" : {
"detected_at": { "type": "date" }
}
}
}
}'
......@@ -8,5 +8,5 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
for i in "${SCRIPT_DIR}"/watches/*.json; do
base_name=$(basename "$i")
name=${base_name%.json}
curl --retry 3 --silent --fail -X PUT "${ES_URL}/_xpack/watcher/watch/${name}?pretty=true" -H 'Content-Type: application/json' --data-binary "@${i}"
curl --retry 3 --fail -X PUT "${ES_URL}/_xpack/watcher/watch/${name}?pretty=true" -H 'Content-Type: application/json' --data-binary "@${i}"
done
{
"trigger": {
"schedule": {
"interval": "30s"
}
},
"input": {
"search": {
"request": {
"search_type": "query_then_fetch",
"indices": [
"pubsub-gitaly-inf-gprd-*"
],
"types": [],
"body": {
"size": 0,
"query": {
"bool": {
"must": [
{
"match_phrase": {
"json.grpc.code.keyword": {
"query": "OK"
}
}
},
{
"range": {
"@timestamp": {
"gte": "now-1m",
"lte": "now"
}
}
}
]
}
},
"aggs": {
"significant_repos": {
"significant_terms": {
"field": "json.grpc.request.repoPath.keyword",
"size": 3
},
"aggs": {
"fqdn": {
"top_hits": {
"docvalue_fields": [
"json.fqdn.keyword"
],
"_source": "json.fqdn.keyword",
"size": 1
}
},
"wall_time_ms": {
"sum": {
"field": "json.grpc.time_ms"
}
}
}
}
}
}
}
}
},
"actions": {
"index_payload" : {
"transform" : {
"script" : {
"inline" : "return [ '_doc' : ctx.payload.aggregations.significant_repos.buckets.collect(bucket -> ['detected_at': System.currentTimeMillis(),'repoPath': bucket.key,'count': bucket.doc_count,'wall_time_ms_per_second': Math.round(bucket.wall_time_ms.value / params.time_period_seconds),'invocation_rate_per_second': Math.round(bucket.doc_count / params.time_period_seconds), 'fqdn': bucket.fqdn.hits.hits[0].fields['json.fqdn.keyword'][0]])];",
"lang" : "painless",
"params" : {
"time_period_seconds" : 60
}
}
},
"index" : {
"index" : "gitaly-abuse-detection",
"doc_type" : "doc"
}
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment