docs/mixin: split query error high alert by handler and use different threshold for query_range

Signed-off-by: Paweł Krupa (paulfantom) <[email protected]>

Author: paulfantom

docs/mixin/alerts/alerts.yaml

@@ -104,25 +104,42 @@ groups:
   - alert: PromscaleQueryHighErrorRate
     expr: |
       (
-        sum by (job, instance, namespace, type) (
-          rate(promscale_query_requests_total{code=~"5.."}[5m])
+        sum by (job, instance, namespace, type, handler) (
+          rate(promscale_query_requests_total{code=~"5..",handler!="/api/v1/query_range",err!="canceled"}[5m])
         )
       /
-        sum by (job, instance, namespace, type) (
-          rate(promscale_query_requests_total[5m])
+        sum by (job, instance, namespace, type, handler) (
+          rate(promscale_query_requests_total{handler!="/api/v1/query_range",err!="canceled"}[5m])
         )
       ) > 0.05
     labels:
       severity: warning
     annotations:
       summary: High error rate in querying Promscale.
-      description: "Evaluating queries via Promscale has {{ $value | humanizePercentage }} error rate."
+      description: "Evaluating queries via Promscale {{ $labels.handler }} endpoint has {{ $value | humanizePercentage }} error rate."
+      runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscaleQueryHighErrorRate.md
+  - alert: PromscaleQueryHighErrorRate
+    expr: |
+      (
+        sum by (job, instance, namespace, type, handler) (
+          rate(promscale_query_requests_total{code=~"5..",handler="/api/v1/query_range",err!="canceled"}[5m])
+        )
+      /
+        sum by (job, instance, namespace, type, handler) (
+          rate(promscale_query_requests_total{handler="/api/v1/query_range",err!="canceled"}[5m])
+        )
+      ) > 0.1
+    labels:
+      severity: warning
+    annotations:
+      summary: High error rate in querying Promscale.
+      description: "Evaluating queries via Promscale {{ $labels.handler }} endpoint has {{ $value | humanizePercentage }} error rate."
       runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscaleQueryHighErrorRate.md
   - alert: PromscaleQueryHighErrorRate
     expr: |
       (
         sum by (job, instance, namespace, type) (
-          rate(promscale_query_requests_total{code=~"5.."}[5m])
+          rate(promscale_query_requests_total{code=~"5..",err!="canceled"}[5m])
         )
       /
         sum by (job, instance, namespace, type) (
@@ -345,8 +362,8 @@ groups:
   - alert: PromscalePostgreSQLSharedBuffersLow
     expr: |
       (
-        ((promscale_sql_database_open_chunks_total_table_size + promscale_sql_database_open_chunks_total_index_size) 
-        / 
+        ((promscale_sql_database_open_chunks_total_table_size + promscale_sql_database_open_chunks_total_index_size)
+        /
         promscale_sql_database_shared_buffers_size)
         > 1 )
     for: 10m
@@ -356,4 +373,3 @@ groups:
       summary: Promscale database performance will be affected.
       description: "Currently open chunks are {{ $value | humanizePercentage }} of PostgreSQL shared_buffers. This will impact database performance."
       runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscalePostgreSQLSharedBuffersLow.md
-