Add counter of failed attempts to retrieve a connection from the pool (#5057)

* Add counter of failed attempts to retrieve a connection from the pool

* Test counter of failed attempts to retrieve a connection from the pool

Author: codesimplicity

micrometer-core/src/main/java/io/micrometer/core/instrument/binder/mongodb/MongoMetricsConnectionPoolListener.java

@@ -21,6 +21,7 @@
 import io.micrometer.common.lang.NonNullApi;
 import io.micrometer.common.lang.NonNullFields;
 import io.micrometer.core.annotation.Incubating;
+import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Gauge;
 import io.micrometer.core.instrument.Meter;
 import io.micrometer.core.instrument.MeterRegistry;
@@ -51,6 +52,8 @@ public class MongoMetricsConnectionPoolListener implements ConnectionPoolListene
 
     private final Map<ServerId, AtomicInteger> checkedOutCounts = new ConcurrentHashMap<>();
 
+    private final Map<ServerId, Counter> checkOutFailedCounters = new ConcurrentHashMap<>();
+
     private final Map<ServerId, AtomicInteger> waitQueueSizes = new ConcurrentHashMap<>();
 
     private final Map<ServerId, List<Meter>> meters = new ConcurrentHashMap<>();
@@ -85,6 +88,8 @@ public void connectionPoolCreated(ConnectionPoolCreatedEvent event) {
                 "the current size of the connection pool, including idle and and in-use members", poolSizes));
         connectionMeters.add(registerGauge(event, METRIC_PREFIX + "checkedout",
                 "the count of connections that are currently in use", checkedOutCounts));
+        connectionMeters.add(registerCounter(event, METRIC_PREFIX + "checkoutfailed",
+                "the count of failed attempts to retrieve a connection", checkOutFailedCounters));
         connectionMeters.add(registerGauge(event, METRIC_PREFIX + "waitqueuesize",
                 "the current size of the wait queue for a connection from the pool", waitQueueSizes));
         meters.put(event.getServerId(), connectionMeters);
@@ -99,6 +104,7 @@ public void connectionPoolClosed(ConnectionPoolClosedEvent event) {
         meters.remove(serverId);
         poolSizes.remove(serverId);
         checkedOutCounts.remove(serverId);
+        checkOutFailedCounters.remove(serverId);
         waitQueueSizes.remove(serverId);
     }
 
@@ -129,6 +135,11 @@ public void connectionCheckOutFailed(ConnectionCheckOutFailedEvent event) {
         if (waitQueueSize != null) {
             waitQueueSize.decrementAndGet();
         }
+
+        Counter checkOutFailedCounter = checkOutFailedCounters.get(event.getServerId());
+        if (checkOutFailedCounter != null) {
+            checkOutFailedCounter.increment();
+        }
     }
 
     @Override
@@ -165,4 +176,14 @@ private Gauge registerGauge(ConnectionPoolCreatedEvent event, String metricName,
             .register(registry);
     }
 
+    private Counter registerCounter(ConnectionPoolCreatedEvent event, String metricName, String description,
+            Map<ServerId, Counter> metrics) {
+        Counter counter = Counter.builder(metricName)
+            .description(description)
+            .tags(tagsProvider.connectionPoolTags(event))
+            .register(registry);
+        metrics.put(event.getServerId(), counter);
+        return counter;
+    }
+
 }

micrometer-core/src/test/java/io/micrometer/core/instrument/binder/mongodb/MongoMetricsConnectionPoolListenerTest.java

@@ -24,12 +24,14 @@
 import com.mongodb.connection.ConnectionPoolSettings;
 import com.mongodb.connection.ServerId;
 import com.mongodb.event.*;
+import io.micrometer.common.lang.NonNull;
 import io.micrometer.core.Issue;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Tags;
 import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import org.junit.jupiter.api.Test;
 
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static java.util.Collections.singletonList;
@@ -55,7 +57,7 @@ void shouldCreatePoolMetrics() {
             .applyToClusterSettings(builder -> builder.hosts(singletonList(new ServerAddress(host, port)))
                 .addClusterListener(new ClusterListener() {
                     @Override
-                    public void clusterOpening(ClusterOpeningEvent event) {
+                    public void clusterOpening(@NonNull ClusterOpeningEvent event) {
                         clusterId.set(event.getClusterId().getValue());
                     }
                 }))
@@ -91,7 +93,7 @@ void shouldCreatePoolMetricsWithCustomTags() {
             .applyToClusterSettings(builder -> builder.hosts(singletonList(new ServerAddress(host, port)))
                 .addClusterListener(new ClusterListener() {
                     @Override
-                    public void clusterOpening(ClusterOpeningEvent event) {
+                    public void clusterOpening(@NonNull ClusterOpeningEvent event) {
                         clusterId.set(event.getClusterId().getValue());
                     }
                 }))
@@ -107,6 +109,7 @@ public void clusterOpening(ClusterOpeningEvent event) {
 
         assertThat(registry.get("mongodb.driver.pool.size").tags(tags).gauge().value()).isEqualTo(2);
         assertThat(registry.get("mongodb.driver.pool.checkedout").gauge().value()).isZero();
+        assertThat(registry.get("mongodb.driver.pool.checkoutfailed").counter().count()).isZero();
         assertThat(registry.get("mongodb.driver.pool.waitqueuesize").gauge().value()).isZero();
 
         mongo.close();
@@ -117,6 +120,28 @@ public void clusterOpening(ClusterOpeningEvent event) {
             .isNull();
     }
 
+    @Test
+    void shouldIncrementCheckoutFailedCount() {
+        ServerId serverId = new ServerId(new ClusterId(), new ServerAddress(host, port));
+        MongoMetricsConnectionPoolListener listener = new MongoMetricsConnectionPoolListener(registry);
+        listener
+            .connectionPoolCreated(new ConnectionPoolCreatedEvent(serverId, ConnectionPoolSettings.builder().build()));
+
+        // start a connection checkout
+        listener.connectionCheckOutStarted(new ConnectionCheckOutStartedEvent(serverId, -1));
+        assertThat(registry.get("mongodb.driver.pool.waitqueuesize").gauge().value()).isEqualTo(1);
+        assertThat(registry.get("mongodb.driver.pool.checkoutfailed").counter().count()).isZero();
+
+        // let the connection checkout fail, simulating a timeout
+        ConnectionCheckOutFailedEvent.Reason reason = ConnectionCheckOutFailedEvent.Reason.TIMEOUT;
+        long elapsedTimeNanos = TimeUnit.SECONDS.toNanos(120);
+        ConnectionCheckOutFailedEvent checkOutFailedEvent = new ConnectionCheckOutFailedEvent(serverId, -1, reason,
+                elapsedTimeNanos);
+        listener.connectionCheckOutFailed(checkOutFailedEvent);
+        assertThat(registry.get("mongodb.driver.pool.waitqueuesize").gauge().value()).isZero();
+        assertThat(registry.get("mongodb.driver.pool.checkoutfailed").counter().count()).isEqualTo(1);
+    }
+
     @Issue("#2384")
     @Test
     void whenConnectionCheckedInAfterPoolClose_thenNoExceptionThrown() {
@@ -125,9 +150,9 @@ void whenConnectionCheckedInAfterPoolClose_thenNoExceptionThrown() {
         MongoMetricsConnectionPoolListener listener = new MongoMetricsConnectionPoolListener(registry);
         listener
             .connectionPoolCreated(new ConnectionPoolCreatedEvent(serverId, ConnectionPoolSettings.builder().build()));
-        listener.connectionCheckedOut(new ConnectionCheckedOutEvent(connectionId));
+        listener.connectionCheckedOut(new ConnectionCheckedOutEvent(connectionId, -1, 0));
         listener.connectionPoolClosed(new ConnectionPoolClosedEvent(serverId));
-        assertThatCode(() -> listener.connectionCheckedIn(new ConnectionCheckedInEvent(connectionId)))
+        assertThatCode(() -> listener.connectionCheckedIn(new ConnectionCheckedInEvent(connectionId, -1)))
             .doesNotThrowAnyException();
     }