feat: add containerd metrics to pod slo tests

modules/python/clusterloader2/cri/config/config.yaml

@@ -20,6 +20,7 @@ name: resource-consumer
 {{$provider := DefaultParam .CL2_PROVIDER "aks"}}
 {{$osType := DefaultParam .CL2_OS_TYPE "linux"}}
 {{$scrapeKubelets := DefaultParam .CL2_SCRAPE_KUBELETS false}}
+{{$scrapeContainerd := DefaultParam .CL2_SCRAPE_CONTAINERD false}}
 {{$hostNetwork := DefaultParam .CL2_HOST_NETWORK "true"}}
 
 namespace:
@@ -67,6 +68,13 @@ steps:
         action: start
   {{end}}
 
+  {{if $scrapeContainerd}}
+  - module:
+      path: /containerd-measurement.yaml
+      params:
+        action: start
+  {{end}}
+
   {{range $j := Loop $steps}}
   - name: Create deployment {{$j}}
     phases:
@@ -141,6 +149,13 @@ steps:
         action: gather
   {{end}}
 
+  {{if $scrapeContainerd}}
+  - module:
+      path: /containerd-measurement.yaml
+      params:
+        action: gather
+  {{end}}
+
   {{range $j := Loop $steps}}
   - name: Deleting deployments {{$j}}
     phases:

modules/python/clusterloader2/cri/config/containerd-measurement.yaml

@@ -0,0 +1,67 @@
+{{$action := .action}} # start, gather
+
+steps:
+  - name: {{$action}} Containerd Measurements
+    measurements:
+    - Identifier: ContainerdCriSandboxCreateNetwork
+      Method: GenericPrometheusQuery
+      Params:
+        action: {{$action}}
+        metricName: ContainerdCriSandboxCreateNetwork
+        metricVersion: v1
+        unit: s
+        queries:
+        - name: Perc100
+          query: histogram_quantile(1, sum(rate(containerd_cri_sandbox_create_network_seconds_bucket[5m])) by (le))
+        - name: Perc99
+          query: histogram_quantile(0.99, sum(rate(containerd_cri_sandbox_create_network_seconds_bucket[5m])) by (le))
+        - name: Perc90
+          query: histogram_quantile(0.90, sum(rate(containerd_cri_sandbox_create_network_seconds_bucket[5m])) by (le))
+        - name: Perc50
+          query: histogram_quantile(0.50, sum(rate(containerd_cri_sandbox_create_network_seconds_bucket[5m])) by (le))
+        - name: Sum
+          query: sum(containerd_cri_sandbox_create_network_seconds_sum)
+        - name: Count
+          query: sum(containerd_cri_sandbox_create_network_seconds_count)
+    - Identifier: ContainerdCriSandboxDeleteNetwork
+      Method: GenericPrometheusQuery
+      Params:
+        action: {{$action}}
+        metricName: ContainerdCriSandboxDeleteNetwork
+        metricVersion: v1
+        unit: s
+        queries:
+        - name: Perc100
+          query: histogram_quantile(1, sum(rate(containerd_cri_sandbox_delete_network_seconds_bucket[5m])) by (le))
+        - name: Perc99
+          query: histogram_quantile(0.99, sum(rate(containerd_cri_sandbox_delete_network_seconds_bucke}[5m])) by (le))
+        - name: Perc90
+          query: histogram_quantile(0.90, sum(rate(containerd_cri_sandbox_delete_network_seconds_bucket[5m])) by (le))
+        - name: Perc50
+          query: histogram_quantile(0.50, sum(rate(containerd_cri_sandbox_delete_network_seconds_bucket[5m])) by (le))
+        - name: Sum
+          query: sum(containerd_cri_sandbox_delete_network_seconds_sum)
+        - name: Count
+          query: sum(containerd_cri_sandbox_delete_network_seconds_count)
+    - Identifier: ContainerdCriNetworkPluginOperations
+      Method: GenericPrometheusQuery
+      Params:
+        action: {{$action}}
+        metricName: ContainerdCriNetworkPluginOperations
+        metricVersion: v1
+        unit: s
+        dimensions:
+          - operation_type
+        queries:
+        - name: Perc100
+          query: histogram_quantile(1, sum(rate(containerd_cri_network_plugin_operations_duration_seconds_seconds_bucket[5m])) by (operation_type, le))
+        - name: Perc99
+          query: histogram_quantile(0.99, sum(rate(containerd_cri_network_plugin_operations_duration_seconds_seconds_bucket[5m])) by (operation_type, le))
+        - name: Perc90
+          query: histogram_quantile(0.90, sum(rate(containerd_cri_network_plugin_operations_duration_seconds_seconds_bucket[5m])) by (operation_type, le))
+        - name: Perc50
+          query: histogram_quantile(0.50, sum(rate(containerd_cri_network_plugin_operations_duration_seconds_seconds_bucket[5m])) by (operation_type, le))
+        - name: Sum
+          query: sum(containerd_cri_network_plugin_operations_duration_seconds_seconds_sum) by (operation_type)
+        - name: Count
+          query: sum(containerd_cri_network_plugin_operations_duration_seconds_seconds_count) by (operation_type)

modules/python/clusterloader2/cri/cri.py

@@ -16,7 +16,7 @@
 def override_config_clusterloader2(
     node_count, node_per_step, max_pods, repeats, operation_timeout,
     load_type, scale_enabled, pod_startup_latency_threshold, provider,
-    os_type, scrape_kubelets, host_network, override_file):
+    os_type, scrape_kubelets, scrape_containerd, host_network, override_file):
     client = KubernetesClient(os.path.expanduser("~/.kube/config"))
     nodes = client.get_nodes(label_selector="cri-resource-consume=true")
     if len(nodes) == 0:
@@ -65,7 +65,7 @@ def override_config_clusterloader2(
     steps = node_count // node_per_step
     logger.info(
         f"Scaled enabled: {scale_enabled}, node per step: {node_per_step}, steps: {steps}, "
-        f"scrape kubelets: {scrape_kubelets}, host network: {host_network}"
+        f"scrape kubelets: {scrape_kubelets}, scrape containerd: {scrape_containerd}, host network: {host_network}"
     )
 
     with open(override_file, 'w', encoding='utf-8') as file:
@@ -89,13 +89,16 @@ def override_config_clusterloader2(
         file.write(f"CL2_PROVIDER: {provider}\n")
         file.write(f"CL2_OS_TYPE: {os_type}\n")
         file.write(f"CL2_SCRAPE_KUBELETS: {str(scrape_kubelets).lower()}\n")
+        if scrape_containerd:
+            file.write(f"CL2_SCRAPE_CONTAINERD: {str(scrape_containerd).lower()}\n")
+            file.write("CONTAINERD_SCRAPE_INTERVAL: 5m\n")
         file.write(f"CL2_HOST_NETWORK: {str(host_network).lower()}\n")
 
     file.close()
 
-def execute_clusterloader2(cl2_image, cl2_config_dir, cl2_report_dir, kubeconfig, provider, scrape_kubelets):
+def execute_clusterloader2(cl2_image, cl2_config_dir, cl2_report_dir, kubeconfig, provider, scrape_kubelets, scrape_containerd):
     run_cl2_command(kubeconfig, cl2_image, cl2_config_dir, cl2_report_dir, provider, overrides=True, enable_prometheus=True,
-                    tear_down_prometheus=False, scrape_kubelets=scrape_kubelets)
+                    tear_down_prometheus=False, scrape_kubelets=scrape_kubelets, scrape_containerd=scrape_containerd)
 
 def verify_measurement():
     client = KubernetesClient(os.path.expanduser("~/.kube/config"))
@@ -140,11 +143,16 @@ def collect_clusterloader2(
     run_id,
     run_url,
     result_file,
-    scrape_kubelets
+    scrape_kubelets,
+    scrape_containerd
 ):
     if scrape_kubelets:
         verify_measurement()
 
+    if scrape_containerd:
+        # TODO: Add verification for containerd metrics
+        pass
+
     details = parse_xml_to_json(os.path.join(cl2_report_dir, "junit.xml"), indent = 2)
     json_data = json.loads(details)
     testsuites = json_data["testsuites"]
@@ -257,6 +265,13 @@ def main():
         default=False,
         help="Whether to scrape kubelets",
     )
+    parser_override.add_argument(
+        "--scrape_containerd",
+        type=str2bool,
+        choices=[True, False],
+        default=False,
+        help="Whether to scrape containerd",
+    )
     parser_override.add_argument(
         "--host_network",
         type=str2bool,
@@ -290,6 +305,13 @@ def main():
         default=False,
         help="Whether to scrape kubelets",
     )
+    parser_execute.add_argument(
+        "--scrape_containerd",
+        type=str2bool,
+        choices=[True, False],
+        default=False,
+        help="Whether to scrape containerd",
+    )
 
     # Sub-command for collect_clusterloader2
     parser_collect = subparsers.add_parser(
@@ -327,6 +349,13 @@ def main():
         default=False,
         help="Whether to scrape kubelets",
     )
+    parser_collect.add_argument(
+        "--scrape_containerd",
+        type=str2bool,
+        choices=[True, False],
+        default=False,
+        help="Whether to scrape containerd",
+    )
 
     args = parser.parse_args()
 
@@ -343,6 +372,7 @@ def main():
             args.provider,
             args.os_type,
             args.scrape_kubelets,
+            args.scrape_containerd,
             args.host_network,
             args.cl2_override_file,
         )
@@ -354,6 +384,7 @@ def main():
             args.kubeconfig,
             args.provider,
             args.scrape_kubelets,
+            args.scrape_containerd,
         )
     elif args.command == "collect":
         collect_clusterloader2(
@@ -367,6 +398,7 @@ def main():
             args.run_url,
             args.result_file,
             args.scrape_kubelets,
+            args.scrape_containerd,
         )
 
 if __name__ == "__main__":

modules/python/tests/test_cri.py

@@ -62,6 +62,7 @@ def test_override_config_clusterloader2(self, mock_kubernetes_client, mock_open)
             provider="aks",
             os_type="linux",
             scrape_kubelets=True,
+            scrape_containerd=True,
             host_network=True,
             override_file="/mock/override.yaml"
         )
@@ -116,6 +117,7 @@ def test_override_config_clusterloader2_host_network_false(self, mock_kubernetes
             provider="aks",
             os_type="linux",
             scrape_kubelets=False,
+            scrape_containerd=False,
             host_network=False,
             override_file="/mock/override.yaml"
         )
@@ -135,13 +137,14 @@ def test_execute_clusterloader2(self, mock_run_cl2_command):
             cl2_report_dir="/mock/report",
             kubeconfig="/mock/kubeconfig",
             provider="aks",
-            scrape_kubelets=True
+            scrape_kubelets=True,
+            scrape_containerd=False
         )
 
         # Verify the command execution
         mock_run_cl2_command.assert_called_once_with(
             "/mock/kubeconfig", "mock-image", "/mock/config", "/mock/report", "aks",
-            overrides=True, enable_prometheus=True, tear_down_prometheus=False, scrape_kubelets=True
+            overrides=True, enable_prometheus=True, tear_down_prometheus=False, scrape_kubelets=True, scrape_containerd=False
         )
 
     @patch('clusterloader2.cri.cri.KubernetesClient')
@@ -185,7 +188,8 @@ def test_collect_clusterloader2(self):
             run_id="12345",
             run_url="http://example.com",
             result_file=result_file,
-            scrape_kubelets=False
+            scrape_kubelets=False,
+            scrape_containerd=False
         )
 
         self.assertTrue(os.path.exists(result_file))
@@ -210,7 +214,8 @@ def test_collect_clusterloader2_no_testsuites(self, mock_parse_xml_to_json):
                 run_id="12345",
                 run_url="http://example.com",
                 result_file="/mock/result.json",
-                scrape_kubelets=False
+                scrape_kubelets=False,
+                scrape_containerd=False
             )
 
         self.assertIn("No testsuites found in the report", str(context.exception))
@@ -230,13 +235,14 @@ def test_override_command(self, mock_override):
             "--provider", "aws", 
             "--os_type", "linux", 
             "--scrape_kubelets", "False", 
+            "--scrape_containerd", "False",
             "--host_network", "False",
             "--cl2_override_file", "/tmp/override.yaml"
         ]
         with patch.object(sys, 'argv', test_args):
             main()
             mock_override.assert_called_once_with(
-                5, 1, 110, 3, "2m", "cpu", True, "10s", "aws", "linux", False, False, "/tmp/override.yaml"
+                5, 1, 110, 3, "2m", "cpu", True, "10s", "aws", "linux", False, False, False, "/tmp/override.yaml"
             )
 
     @patch("clusterloader2.cri.cri.override_config_clusterloader2")
@@ -255,12 +261,13 @@ def test_override_command_default_host_network(self, mock_override):
             "--provider", "aws", 
             "--os_type", "linux", 
             "--scrape_kubelets", "False", 
+            "--scrape_containerd", "False",
             "--cl2_override_file", "/tmp/override.yaml"
         ]
         with patch.object(sys, 'argv', test_args):
             main()
             mock_override.assert_called_once_with(
-                5, 1, 110, 3, "2m", "cpu", True, "10s", "aws", "linux", False, True, "/tmp/override.yaml"
+                5, 1, 110, 3, "2m", "cpu", True, "10s", "aws", "linux", False, False, True, "/tmp/override.yaml"
             )
 
     @patch("clusterloader2.cri.cri.execute_clusterloader2")
@@ -272,13 +279,14 @@ def test_execute_command(self, mock_execute):
             "--cl2_report_dir", "/reports",
             "--kubeconfig", "/home/user/.kube/config", 
             "--provider", "gcp", 
-            "--scrape_kubelets", "True"
+            "--scrape_kubelets", "True",
+            "--scrape_containerd", "False"
         ]
         with patch.object(sys, 'argv', test_args):
             main()
             mock_execute.assert_called_once_with(
                 "gcr.io/cl2:latest", "/configs", "/reports",
-                "/home/user/.kube/config", "gcp", True
+                "/home/user/.kube/config", "gcp", True, False
             )
 
     @patch("clusterloader2.cri.cri.collect_clusterloader2")
@@ -294,13 +302,14 @@ def test_collect_command(self, mock_collect):
             "--run_id", "run-123", 
             "--run_url", "https://run.url", 
             "--result_file", "/tmp/results.json", 
-            "--scrape_kubelets", "False"
+            "--scrape_kubelets", "False",
+            "--scrape_containerd", "False"
         ]
         with patch.object(sys, 'argv', test_args):
             main()
             mock_collect.assert_called_once_with(
                 3, 100, 5, "memory", "/reports", "gcp-zone", "run-123",
-                "https://run.url", "/tmp/results.json", False
+                "https://run.url", "/tmp/results.json", False, False
             )
 
 if __name__ == '__main__':