Allow for custom Grafana dashboards

docs/src/main/asciidoc/observability-devservices-lgtm.adoc

@@ -185,6 +185,12 @@ Each dashboard is tuned for the specific application setup. The available dashbo
 Some panels in the dashboards might take a few minutes to show accurate data when their values are calculated over a sliding  time window.
 ====
 
+==== Custom dashboards
+
+Users can add their own Grafana dashboards aka their configuration. All you need to do is to place a `grafana-dashboard-[your name].json` into a `META-INF/grafana` directory in your application, and LGTM DevResource will pick this up automatically, and include this dashboard configuration in the overall Grafana dashboards yaml configuration.
+
+e.g. /META-INF/grafana/grafana-dashboard-my-simple-prometheus-dashboard.json configuration file creates `My Simple Promeheus Dashboard` titled dashboard
+
 === Additional configuration
 
 This extension will configure your `quarkus-opentelemetry` and `quarkus-micrometer-registry-otlp` extensions to send data to the OTel Collector bundled with the Grafana OTel LGTM image.

extensions/observability-devservices/testcontainers/pom.xml

@@ -31,6 +31,10 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-junit4-mock</artifactId>

extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/GrafanaUtils.java

@@ -0,0 +1,107 @@
+package io.quarkus.observability.testcontainers;
+
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.yaml.snakeyaml.Yaml;
+
+import io.quarkus.bootstrap.classloading.ClassPathElement;
+import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
+
+public class GrafanaUtils {
+
+    static final String META_INF_GRAFANA = "META-INF/grafana";
+
+    static String consumeGrafanaResources(String config, Consumer<String> consumer) {
+        return consumeGrafanaResources(config, consumer, findGrafanaDashboards());
+    }
+
+    @SuppressWarnings("unchecked")
+    // Take dashboards as param, so we can test this
+    static String consumeGrafanaResources(String config, Consumer<String> consumer, Set<String> dashboards) {
+        Yaml yaml = new Yaml();
+        Map<String, Object> data = yaml.load(config);
+        List<Map<String, Object>> providers = (List<Map<String, Object>>) data.get("providers");
+
+        dashboards.forEach(s -> {
+            String sub = s.substring("grafana-dashboard-".length(), s.length() - ".json".length());
+            Map<String, Object> provider = new LinkedHashMap<>();
+            String name = toName(sub);
+            provider.put("name", name);
+            provider.put("type", "file");
+            Map<String, Object> options = new LinkedHashMap<>();
+            options.put("path", "/otel-lgtm/" + s);
+            options.put("foldersFromFilesStructure", false);
+            provider.put("options", options);
+            providers.add(provider);
+
+            consumer.accept(s);
+        });
+
+        StringWriter writer = new StringWriter();
+        yaml.dump(data, writer);
+
+        return writer.toString();
+    }
+
+    private static String toName(String path) {
+        if (path.isEmpty()) {
+            throw new IllegalArgumentException("Illegal path: " + path);
+        }
+        StringBuilder name = new StringBuilder();
+        boolean dash = true;
+        for (int i = 0; i < path.length(); i++) {
+            char ch = path.charAt(i);
+            if (dash) {
+                ch = Character.toUpperCase(ch);
+                dash = false;
+            } else {
+                if (ch == '-') {
+                    dash = true;
+                    ch = ' ';
+                }
+            }
+            name.append(ch);
+        }
+        return name.toString();
+    }
+
+    /**
+     * Visits all {@code META-INF/grafana} directories and their content found on the runtime classpath
+     * and returns all Grafana dashboard json configurations
+     */
+    private static Set<String> findGrafanaDashboards() {
+        Set<String> dashboards = new HashSet<>();
+        final List<ClassPathElement> elements = QuarkusClassLoader.getElements(META_INF_GRAFANA, false);
+        if (!elements.isEmpty()) {
+            for (var element : elements) {
+                if (element.isRuntime()) {
+                    element.apply(tree -> {
+                        tree.walkIfContains(META_INF_GRAFANA, visit -> {
+                            Path visitPath = visit.getPath();
+                            if (!Files.isDirectory(visitPath)) {
+                                String rel = visit.getRelativePath();
+                                // Ensure that the relative path starts with the right prefix and suffix before calling substring
+                                if (rel.startsWith(META_INF_GRAFANA + "/grafana-dashboard-") && rel.endsWith(".json")) {
+                                    // Strip the "META-INF/grafana/" prefix
+                                    String subPath = rel.substring(META_INF_GRAFANA.length() + 1);
+                                    dashboards.add(subPath);
+                                }
+                            }
+                        });
+                        return null;
+                    });
+                }
+            }
+        }
+        return dashboards;
+    }
+
+}

extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/LgtmContainer.java

@@ -1,5 +1,6 @@
 package io.quarkus.observability.testcontainers;
 
+import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -106,9 +107,19 @@ public LgtmContainer(LgtmConfig config, boolean scrapingRequired) {
         Optional<Set<LgtmComponent>> logging = config.logging();
         logging.ifPresent(set -> set.forEach(l -> withEnv("ENABLE_LOGS_" + l.name(), "true")));
 
+        String dashboards = GrafanaUtils.consumeGrafanaResources(
+                DASHBOARDS_CONFIG,
+                s -> {
+                    withCopyFileToContainer(
+                            MountableFile.forClasspathResource(GrafanaUtils.META_INF_GRAFANA + "/" + s),
+                            "/otel-lgtm/" + s);
+                    log.infof("Adding custom Grafana dashboard config: %s", s);
+                });
+
         // Replacing bundled dashboards with our own
-        addFileToContainer(DASHBOARDS_CONFIG.getBytes(),
+        addFileToContainer(dashboards.getBytes(StandardCharsets.UTF_8),
                 "/otel-lgtm/grafana/conf/provisioning/dashboards/grafana-dashboards.yaml");
+
         withCopyFileToContainer(
                 MountableFile.forClasspathResource("/grafana-dashboard-quarkus-micrometer-prometheus.json"),
                 "/otel-lgtm/grafana-dashboard-quarkus-micrometer-prometheus.json");

extensions/observability-devservices/testcontainers/src/test/java/io/quarkus/observability/testcontainers/GrafanaUtilsTest.java

@@ -0,0 +1,46 @@
+package io.quarkus.observability.testcontainers;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+import org.yaml.snakeyaml.Yaml;
+
+public class GrafanaUtilsTest {
+
+    protected static final String DASHBOARDS_CONFIG = """
+            apiVersion: 1
+
+            providers:
+              - name: "Quarkus Micrometer Prometheus registry"
+                type: file
+                options:
+                  path: /otel-lgtm/grafana-dashboard-quarkus-micrometer-prometheus.json
+                  foldersFromFilesStructure: false
+            """;
+
+    @Test
+    public void testGrafanaUtils() {
+        String config = GrafanaUtils.consumeGrafanaResources(
+                DASHBOARDS_CONFIG,
+                s -> {
+                    try (InputStream stream = Thread.currentThread().getContextClassLoader()
+                            .getResourceAsStream(GrafanaUtils.META_INF_GRAFANA + "/" + s)) {
+                        System.out.println(new String(stream.readAllBytes()));
+                        System.out.println("------");
+                    } catch (IOException ignored) {
+                    }
+                },
+                Set.of("grafana-dashboard-my-test.json"));
+
+        Yaml yaml = new Yaml();
+        Map<String, Object> data = yaml.load(config);
+        StringWriter writer = new StringWriter();
+        yaml.dump(data, writer);
+        System.out.println(writer);
+    }
+
+}