tests: enhance automated tests scripts to run on firecracker

This patch modifies the recently added scripts that help automate
testing various application on OSv on firecracker.

Signed-off-by: Waldemar Kozaczuk <[email protected]>

Author: wkozaczuk

modules/httpserver-api/test.sh

@@ -6,7 +6,7 @@ CMD="java.so -Djetty.base=/jetty/demo-base -jar /jetty/start.jar"
 
 case $PROTOCOL in
 	http)
-		PYTHONPATH=$OSV_DIR/scripts $OSV_DIR/modules/httpserver-api/tests/testhttpserver-api.py --cmd "$CMD" ;;
+		PYTHONPATH=$OSV_DIR/scripts $OSV_DIR/modules/httpserver-api/tests/testhttpserver-api.py --cmd "$CMD" --hypervisor $OSV_HYPERVISOR;;
 	https)
-		PYTHONPATH=$OSV_DIR/scripts $OSV_DIR/modules/httpserver-api/tests/testhttpserver-api.py --cmd "$CMD" --cert $OSV_DIR/modules/certs/build/client.pem --key $OSV_DIR/modules/certs/build/client.key --cacert $OSV_DIR/modules/certs/build/cacert.pem ;;
+		PYTHONPATH=$OSV_DIR/scripts $OSV_DIR/modules/httpserver-api/tests/testhttpserver-api.py --cmd "$CMD" --cert $OSV_DIR/modules/certs/build/client.pem --key $OSV_DIR/modules/certs/build/client.key --cacert $OSV_DIR/modules/certs/build/cacert.pem --hypervisor $OSV_HYPERVISOR;;
 esac

modules/httpserver-api/tests/basetest.py

@@ -17,6 +17,10 @@ class Basetest(unittest.TestCase):
     @classmethod
     def set_config(cls, parser):
         cls.config = parser.parse_args()
+        if cls.config.hypervisor == 'firecracker':
+            module_base = os.path.join(os.path.realpath(os.path.dirname(__file__)), '..')
+            cls.config.run_script = os.path.join(module_base, "..", "..", "scripts", "firecracker.py")
+            cls.config.host = '172.16.0.2'
         cls._client = client.Client(cls.config)
 
     @classmethod
@@ -91,15 +95,15 @@ def assertHttpError(self, url, code=404):
             raise Exception('Expected failure but request succeeded')
 
     @classmethod
-    def curl(cls, api, method='GET', data=None):
+    def curl(cls, api, method='GET', data=None, timeout=None):
         url = cls.get_url(api)
 
         r = {
             'GET': requests.get,
             'POST': requests.post,
             'DELETE': requests.delete,
             'PUT': requests.put,
-        }[method](url, data=data, **cls._client.get_request_kwargs())
+        }[method](url, data=data, timeout=timeout, **cls._client.get_request_kwargs())
 
         if r.status_code != 200:
             raise HttpError(r.status_code)
@@ -122,7 +126,9 @@ def get_ca_cert_path(cls):
     @classmethod
     def exec_os(cls):
         args = []
-        if cls.config.use_sudo:
+        if cls.config.hypervisor == 'firecracker':
+            args += [cls.config.run_script, "-l", "-m 2048M", "-n", "-c 4"]
+        elif cls.config.use_sudo:
             args += ["/usr/bin/sudo", cls.config.run_script, "-n"]
         else:
             args += [cls.config.run_script, "--forward", "tcp::" + str(cls._client.get_port()) + "-:" + str(cls._client.get_port())]
@@ -142,7 +148,7 @@ def shutdown(cls):
 
         path = cls.path_by_nick(cls.os_api, "os_poweroff")
         try:
-            cls.curl(path, method='POST')
+            cls.curl(path, method='POST', timeout=0.5)
         except:
             pass
         retry = 10

modules/httpserver-api/tests/testhttpserver-api.py

@@ -18,6 +18,7 @@
 parser.add_argument('--use_sudo', help='Use sudo with -n option instead of port forwarding', action='store_true')
 parser.add_argument('--jsondir', help='location of the json files', default=os.path.join(module_base, 'api-doc/listings/'))
 parser.add_argument('--test_image', help='the path to the test image')
+parser.add_argument('--hypervisor', action="store", default="qemu", help="choose hypervisor to run: qemu, firecracker")
 client.Client.add_arguments(parser)
 
 class test_httpserver(basetest.Basetest):

modules/tests/test.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 THIS_DIR=$(readlink -f $(dirname $0))
-$THIS_DIR/../../scripts/test.py
+$THIS_DIR/../../scripts/test.py -p $OSV_HYPERVISOR

scripts/tests/compose_and_test_selected_apps.sh

@@ -21,6 +21,7 @@ usage() {
 	  -r              Run test app only (must have been composed earlier)
 	  -R              Compose test app image with RoFS (ZFS is the default)
 	  -l              Use latest OSv kernel from build/last to build test image
+	  -f              Run OSv on firecracker
 	EOF
 	exit ${1:-0}
 }
@@ -29,18 +30,22 @@ FS=zfs
 COMPOSE_ONLY=false
 RUN_ONLY=false
 LOADER="osv-loader"
+OSV_HYPERVISOR="qemu"
 
-while getopts crRlh: OPT ; do
+while getopts crRlfh: OPT ; do
 	case ${OPT} in
 	c) COMPOSE_ONLY=true;;
 	r) RUN_ONLY=true;;
 	R) FS=rofs;;
         l) LOADER="osv-latest-loader";;
+        f) OSV_HYPERVISOR="firecracker";;
 	h) usage;;
 	?) usage 1;;
 	esac
 done
 
+export OSV_HYPERVISOR
+
 shift $((OPTIND - 1))
 [[ -z $1 ]] && usage 1
 
@@ -171,7 +176,7 @@ test_apps_with_tester()
 {
   compose_and_run_test_app "iperf3"
   compose_and_run_test_app "graalvm-netty-plot"
-  compose_test_app "ffmpeg" "libz" && run_test_app "ffmpeg" "video_subclip" && run_test_app "ffmpeg" "video_transcode"
+  compose_test_app "ffmpeg" && run_test_app "ffmpeg" "video_subclip" && run_test_app "ffmpeg" "video_transcode"
   compose_and_run_test_app "redis-memonly"
   compose_and_run_test_app "cli"
   compose_and_run_test_app "mysql"
@@ -182,6 +187,9 @@ test_apps_with_tester()
 
 run_unit_tests()
 {
+  # Unit tests are special as the unit tests runner depends on usr.manifest which
+  # needs to be placed in the tests module. So let us gegnerate it on the fly from the unit tests mpm
+  capstan package describe osv.unit-tests -c | grep "/tests/tst-" | grep -o "/tests/tst-.*" | sed 's/$/: dummy/' > $OSV_DIR/modules/tests/usr.manifest
   compose_test_app "unit-tests" && run_test_app "tests"
   compose_test_app "httpserver-api-tests" && run_test_app "httpserver-api" "http"
   #compose_test_app "httpserver-api-https-tests" "httpserver-api-tests" && run_test_app "httpserver-api" "https"

scripts/tests/test_app.py

@@ -35,7 +35,7 @@ def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, h
     parser = argparse.ArgumentParser(prog='test_app')
     parser.add_argument("-i", "--image", action="store", default=None, metavar="IMAGE",
                         help="path to disk image file. defaults to build/$mode/usr.img")
-    parser.add_argument("-p", "--hypervisor", action="store", default="qemu",
+    parser.add_argument("-p", "--hypervisor", action="store", default=None,
                         help="choose hypervisor to run: qemu, firecracker")
     parser.add_argument("--line", action="store", default=None,
                         help="expect line in guest output")
@@ -47,6 +47,15 @@ def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, h
     parser.add_argument("--kill", action="store_true", help="kill the app instead of waiting until terminates itself")
 
     cmdargs = parser.parse_args()
+
+    hypervisor_name = 'qemu'
+    if cmdargs.hypervisor != None:
+        hypervisor_name = cmdargs.hypervisor
+    else:
+        hypervisor_from_env = os.getenv('OSV_HYPERVISOR')
+        if hypervisor_from_env != None:
+            hypervisor_name = hypervisor_from_env
+
     set_verbose_output(True)
-    run(cmdargs.execute, cmdargs.hypervisor, cmdargs.image, cmdargs.line,
+    run(cmdargs.execute, hypervisor_name, cmdargs.image, cmdargs.line,
         cmdargs.guest_port, cmdargs.host_port, cmdargs.input_line, cmdargs.kill)

scripts/tests/test_app_with_test_script.py

@@ -32,7 +32,7 @@ def run(command, hypervisor_name, host_port, guest_port, script_path, image_path
     parser = argparse.ArgumentParser(prog='test_app')
     parser.add_argument("-i", "--image", action="store", default=None, metavar="IMAGE",
                         help="path to disk image file. defaults to build/$mode/usr.img")
-    parser.add_argument("-p", "--hypervisor", action="store", default="qemu",
+    parser.add_argument("-p", "--hypervisor", action="store", default=None,
                         help="choose hypervisor to run: qemu, firecracker")
     parser.add_argument("--start_line", action="store", default=None,
                         help="expect line in guest output before executing test script")
@@ -46,5 +46,19 @@ def run(command, hypervisor_name, host_port, guest_port, script_path, image_path
                         help="edit command line before execution")
 
     cmdargs = parser.parse_args()
+
+    hypervisor_name = 'qemu'
+    if cmdargs.hypervisor != None:
+        hypervisor_name = cmdargs.hypervisor
+    else:
+        hypervisor_from_env = os.getenv('OSV_HYPERVISOR')
+        if hypervisor_from_env != None:
+            hypervisor_name = hypervisor_from_env
+
+    if hypervisor_name == 'firecracker':
+        os.environ['OSV_HOSTNAME'] = '172.16.0.2'
+    else:
+        os.environ['OSV_HOSTNAME'] = 'localhost'
+
     set_verbose_output(True)
-    run(cmdargs.execute, cmdargs.hypervisor, cmdargs.host_port, cmdargs.guest_port, cmdargs.script_path, cmdargs.image, cmdargs.start_line, cmdargs.end_line)
+    run(cmdargs.execute, hypervisor_name, cmdargs.host_port, cmdargs.guest_port, cmdargs.script_path, cmdargs.image, cmdargs.start_line, cmdargs.end_line)

scripts/tests/test_http_app_with_curl_and_ab.py

@@ -30,7 +30,11 @@ def run(command, hypervisor_name, host_port, guest_port, http_path, expected_htt
        print(pre_script)
        subprocess.check_output([pre_script])
 
-    app_url = "http://localhost:%s%s" % (host_port, http_path)
+    if hypervisor_name == 'firecracker':
+       app_url = "http://172.16.0.2:%s%s" % (guest_port, http_path)
+    else:
+       app_url = "http://localhost:%s%s" % (host_port, http_path)
+
     if expected_http_line != None:
         check_with_curl(app_url, expected_http_line)
 
@@ -85,7 +89,7 @@ def run(command, hypervisor_name, host_port, guest_port, http_path, expected_htt
     parser = argparse.ArgumentParser(prog='test_app')
     parser.add_argument("-i", "--image", action="store", default=None, metavar="IMAGE",
                         help="path to disk image file. defaults to build/$mode/usr.img")
-    parser.add_argument("-p", "--hypervisor", action="store", default="qemu", 
+    parser.add_argument("-p", "--hypervisor", action="store", default=None,
                         help="choose hypervisor to run: qemu, firecracker")
     parser.add_argument("--line", action="store", default=None,
                         help="expect line in guest output")
@@ -105,7 +109,21 @@ def run(command, hypervisor_name, host_port, guest_port, http_path, expected_htt
                         help="error line to ignore on kill")
 
     cmdargs = parser.parse_args()
+
+    hypervisor_name = 'qemu'
+    if cmdargs.hypervisor != None:
+        hypervisor_name = cmdargs.hypervisor
+    else:
+        hypervisor_from_env = os.getenv('OSV_HYPERVISOR')
+        if hypervisor_from_env != None:
+            hypervisor_name = hypervisor_from_env
+
+    if hypervisor_name == 'firecracker':
+        os.environ['OSV_HOSTNAME'] = '172.16.0.2'
+    else:
+        os.environ['OSV_HOSTNAME'] = 'localhost'
+        
     set_verbose_output(True)
-    run(cmdargs.execute, cmdargs.hypervisor, cmdargs.host_port, cmdargs.guest_port,
+    run(cmdargs.execute, hypervisor_name, cmdargs.host_port, cmdargs.guest_port,
        cmdargs.http_path ,cmdargs.http_line, cmdargs.image, cmdargs.line,
        cmdargs.concurrency, cmdargs.count, cmdargs.pre_script, cmdargs.no_keep_alive, cmdargs.error_line_to_ignore_on_kill)

scripts/tests/testing.py

@@ -1,6 +1,7 @@
 import re
 import os
 import sys
+import signal
 import subprocess
 import threading
 import socket
@@ -180,7 +181,7 @@ def join(self):
         if self.pipe_stdin:
             self.process.stdin.close()
         if self.process.returncode:
-            raise Exception('Guest failed (returncode=%d)' % self.proces.returncode)
+            raise Exception('Guest failed (returncode=%d)' % self.process.returncode)
         if self.failed:
             raise Exception('Guest failed')
 
@@ -208,17 +209,24 @@ def line_with_error(self):
 
 def run_command_in_guest(command, **kwargs):
     common_parameters = ["-e", "--power-off-on-abort " + command]
-    if 'hypervisor' in kwargs.keys() and kwargs['hypervisor'] == 'firecracker':
-        return Guest(["-m 1024M", "-n", "-c 4"] + common_parameters, **kwargs)
+
+    if kwargs.get('hypervisor') == 'firecracker':
+        parameters = ["-l", "-m 2048M", "-n", "-c 4"] + common_parameters
     else:
-        return Guest(["-s"] + common_parameters, **kwargs)
+        parameters = ["-s"] + common_parameters
+
+    return Guest(parameters, **kwargs)
 
 class Guest(SupervisedProcess):
     def __init__(self, args, forward=[], hold_with_poweroff=False, show_output_on_error=True,
                  scan_for_failed_to_load_object_error=True, run_py_args=[], hypervisor='qemu', pipe_stdin=False):
 
         if hypervisor == 'firecracker':
             run_script = os.path.join(osv_base, "scripts/firecracker.py")
+            self.monitor_socket = None
+            physical_nic = os.getenv('OSV_FC_NIC')
+            if physical_nic:
+               args.extend(['-p', physical_nic])
         else:
             run_script = os.path.join(osv_base, "scripts/run.py")
 
@@ -233,18 +241,25 @@ def __init__(self, args, forward=[], hold_with_poweroff=False, show_output_on_er
 
             args.extend(['--unsafe-cache'])
 
+        if _verbose_output:
+            print('Running OSv on %s with parameters: [%s]' % (hypervisor, " ".join(args)))
+
         SupervisedProcess.__init__(self, [run_script] + run_py_args + args,
             show_output=_verbose_output,
             show_output_on_error=show_output_on_error,
             scan_for_failed_to_load_object_error=scan_for_failed_to_load_object_error,
             pipe_stdin=pipe_stdin)
 
     def kill(self):
-        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        s.connect(self.monitor_socket)
-        s.send('quit\n'.encode())
-        self.join()
-        s.close()
+        if self.monitor_socket != None:
+            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            s.connect(self.monitor_socket)
+            s.send('quit\n'.encode())
+            self.join()
+            s.close()
+        else:
+            os.kill(self.process.pid, signal.SIGINT)
+            self.join()
 
 def wait_for_line(guest, text):
     return _wait_for_line(guest, lambda line: line == text, text)