Avoid using total suffix for gauge metrics and apply rate for counters

Latest mixtool linter which relies on grafana-linter pkg is failing when gauge metrics with name ending with `total`[1].

[1] https://github.com/grafana/dashboard-linter/blob/44d415fb6bdc4d8e6585e514c448174d4de1ff02/lint/rule_target_counter_agg.go#L30

Signed-off-by: Arunprasad Rajkumar <[email protected]>

Author: arajkumar

docs/mixin/.lint

@@ -10,10 +10,3 @@ exclusions:
   template-datasource-rule:
   target-counter-agg-rule:
     reason: "disabled for promscale_sql_database_worker_maintenance_job_locks_total, promscale_sql_database_worker_maintenance_job_long_running_total. This metric should be not have total due to naming conventions as its a gauge, not a counter"
-    entries:
-      - dashboard: Promscale
-        panel: Long running maintenance queries by job type
-      - dashboard: Promscale
-        panel: Long running maintenance queries by wait event
-      - dashboard: Promscale
-        panel: Locks held by maintenance jobs by lock mode

docs/mixin/dashboards/promscale.json

@@ -2370,7 +2370,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_total{namespace=~\"$namespace\"})",
+              "expr": "max(promscale_sql_database_worker_maintenance_job_long_running{namespace=~\"$namespace\"})",
               "interval": "",
               "legendFormat": "total",
               "range": true,
@@ -2918,7 +2918,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(promscale_sql_database_worker_maintenance_job_long_running_total{namespace=~\"$namespace\"})",
+              "expr": "max(promscale_sql_database_worker_maintenance_job_long_running{namespace=~\"$namespace\"})",
               "interval": "",
               "legendFormat": "total",
               "range": true,
@@ -3094,7 +3094,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(promscale_sql_database_worker_maintenance_job_locks_total{namespace=~\"$namespace\"})",
+              "expr": "max(promscale_sql_database_worker_maintenance_job_locks{namespace=~\"$namespace\"})",
               "interval": "",
               "legendFormat": "total",
               "range": true,

pkg/pgmodel/metrics/database/metrics.go

@@ -249,7 +249,7 @@ var metrics = []metricQueryWrap{
 			prometheus.GaugeOpts{
 				Namespace: util.PromNamespace,
 				Subsystem: "sql_database",
-				Name:      "worker_maintenance_job_locks_total",
+				Name:      "worker_maintenance_job_locks",
 				Help:      "Number of locks held by Promscale maintenance workers.",
 			},
 		),
@@ -326,7 +326,7 @@ var metrics = []metricQueryWrap{
 			prometheus.GaugeOpts{
 				Namespace: util.PromNamespace,
 				Subsystem: "sql_database",
-				Name:      "worker_maintenance_job_long_running_total",
+				Name:      "worker_maintenance_job_long_running",
 				Help:      "Number of Promscale maintenance workers executing long running queries.",
 			},
 		),