K3sContainer: expose kubeConfig for docker network access (#5097)

Co-authored-by: Sergei Egorov <[email protected]>

Author: johnathana

modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java

@@ -8,23 +8,29 @@
 import com.github.dockerjava.api.command.InspectContainerResponse;
 import com.github.dockerjava.api.model.DockerObjectAccessor;
 import lombok.SneakyThrows;
+import org.apache.commons.io.IOUtils;
 import org.testcontainers.containers.BindMode;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
 import org.testcontainers.utility.DockerImageName;
 
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 
 public class K3sContainer extends GenericContainer<K3sContainer> {
 
+    public static int KUBE_SECURE_PORT = 6443;
+
+    public static int RANCHER_WEBHOOK_PORT = 8443;
+
     private String kubeConfigYaml;
 
     public K3sContainer(DockerImageName dockerImageName) {
         super(dockerImageName);
         dockerImageName.assertCompatibleWith(DockerImageName.parse("rancher/k3s"));
 
-        addExposedPorts(6443, 8443);
+        addExposedPorts(KUBE_SECURE_PORT, RANCHER_WEBHOOK_PORT);
         setPrivilegedMode(true);
         withCreateContainerCmdModifier(it -> {
             it.getHostConfig().withCgroupnsMode("host");
@@ -44,30 +50,57 @@ public K3sContainer(DockerImageName dockerImageName) {
         setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*Node controller sync successful.*"));
     }
 
-    @SneakyThrows
     @Override
     protected void containerIsStarted(InspectContainerResponse containerInfo) {
+        String rawKubeConfig = copyFileFromContainer("/etc/rancher/k3s/k3s.yaml",
+            is -> IOUtils.toString(is, StandardCharsets.UTF_8)
+        );
+        String serverUrl = "https://" + this.getHost() + ":" + this.getMappedPort(KUBE_SECURE_PORT);
+        kubeConfigYaml = kubeConfigWithServerUrl(rawKubeConfig, serverUrl);
+    }
+
+    /**
+     * Return the kubernetes client configuration to access k3s from the host machine.
+     *
+     * @return the kubeConfig yaml.
+     */
+    public String getKubeConfigYaml() {
+        return kubeConfigYaml;
+    }
+
+    /**
+     * Generate a kubernetes client configuration for use on a docker internal network. The kubeConfig can be used by
+     * another docker container running in the same network as the k3s container. For access from the host, use
+     * the {@link #getKubeConfigYaml()} method instead.
+     *
+     * @param networkAlias a valid network alias of the k3s container.
+     * @return the kubeConfig yaml.
+     */
+    public String generateInternalKubeConfigYaml(String networkAlias) {
+        if (this.getNetworkAliases().contains(networkAlias)) {
+            String serverUrl = "https://" + networkAlias + ":" + KUBE_SECURE_PORT;
+            return kubeConfigWithServerUrl(kubeConfigYaml, serverUrl);
+        } else {
+            throw new IllegalArgumentException(networkAlias + " is not a network alias for k3s container");
+        }
+    }
+
+    @SneakyThrows
+    private String kubeConfigWithServerUrl(String kubeConfigYaml, String serverUrl) {
         ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
 
-        ObjectNode rawKubeConfig = copyFileFromContainer(
-            "/etc/rancher/k3s/k3s.yaml",
-            is -> objectMapper.readValue(is, ObjectNode.class)
-        );
+        ObjectNode kubeConfigObjectNode = objectMapper.readValue(kubeConfigYaml, ObjectNode.class);
 
-        JsonNode clusterNode = rawKubeConfig.at("/clusters/0/cluster");
+        JsonNode clusterNode = kubeConfigObjectNode.at("/clusters/0/cluster");
         if (!clusterNode.isObject()) {
             throw new IllegalStateException("'/clusters/0/cluster' expected to be an object");
         }
         ObjectNode clusterConfig = (ObjectNode) clusterNode;
+        clusterConfig.replace("server", new TextNode(serverUrl));
 
-        clusterConfig.replace("server", new TextNode("https://" + this.getHost() + ":" + this.getMappedPort(6443)));
+        kubeConfigObjectNode.set("current-context", new TextNode("default"));
 
-        rawKubeConfig.set("current-context", new TextNode("default"));
-
-        kubeConfigYaml = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(rawKubeConfig);
-    }
-
-    public String getKubeConfigYaml() {
-        return kubeConfigYaml;
+        return objectMapper.writerWithDefaultPrettyPrinter()
+            .writeValueAsString(kubeConfigObjectNode);
     }
 }

modules/k3s/src/test/java/org/testcontainers/k3s/KubectlContainerTest.java

@@ -0,0 +1,45 @@
+package org.testcontainers.k3s;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy;
+import org.testcontainers.images.builder.Transferable;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class KubectlContainerTest {
+
+    public static Network network = Network.SHARED;
+
+    @ClassRule
+    public static K3sContainer k3s = new K3sContainer(DockerImageName.parse("rancher/k3s:v1.21.3-k3s1"))
+        .withNetwork(network)
+        .withNetworkAliases("k3s");
+
+    @Test
+    public void shouldExposeKubeConfigForNetworkAlias() throws Exception {
+        String kubeConfigYaml = k3s.generateInternalKubeConfigYaml("k3s");
+
+        try (
+            GenericContainer<?> kubectlContainer = new GenericContainer<>("rancher/kubectl:v1.23.3")
+                .withNetwork(network)
+                .withCopyToContainer(Transferable.of(kubeConfigYaml), "/.kube/config")
+                .withCommand("get namespaces")
+                .withStartupCheckStrategy(new OneShotStartupCheckStrategy().withTimeout(Duration.ofSeconds(30)))
+        ) {
+            kubectlContainer.start();
+
+            assertThat(kubectlContainer.getLogs()).contains("kube-system");
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void shouldThrowAnExceptionForUnknownNetworkAlias() {
+        k3s.generateInternalKubeConfigYaml("not-set-network-alias");
+    }
+}