[Backport 1.30-strict] Cleans up containers and shims on microk8s stop and snap removal (#4883)

* Cleans up containers and shims on microk8s stop and snap removal (#4755)

The containerd shims and the containers are now removed in both classic
and strict modes when stopping microk8s (microk8s stop) or when removing
the snap.

Note that ctr container delete has no ``--force`` flag.

Adds additional assertion in the upgrade test, ensuring that the Pods /
containers have been removed on snap removal.

Adds additional test, ensuring that the Pods / containers have been
removed when running microk8s stop.

(cherry picked from commit 941b781)

* Add psutil to pip install directives (#4782)

(cherry picked from commit e2b2a2f)

---------

Co-authored-by: Claudiu Belu <[email protected]>

Author: berkayoz

.github/workflows/build-snap.yml

@@ -46,7 +46,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap
@@ -71,7 +71,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap
@@ -139,7 +139,7 @@ jobs:
           set -x
           sudo apt-get install python3-setuptools
           sudo pip3 install --upgrade pip
-          sudo pip3 install -U pytest sh
+          sudo pip3 install -U pytest sh psutil
           sudo apt-get -y install open-iscsi
           sudo systemctl enable iscsid
       - name: Fetch snap

microk8s-resources/actions/common/utils.sh

@@ -988,7 +988,7 @@ remove_all_containers() {
 
     for container in $("${SNAP}/microk8s-ctr.wrapper" containers ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}')
     do
-        "${SNAP}/microk8s-ctr.wrapper" container delete --force $container &>/dev/null || true
+        "${SNAP}/microk8s-ctr.wrapper" container delete $container &>/dev/null || true
     done
 }
 

microk8s-resources/default-hooks/remove.d/90-containers

@@ -2,9 +2,5 @@
 
 . "${SNAP}/actions/common/utils.sh"
 
-if is_strict
-then
-  remove_all_containers
-else
-  kill_all_container_shims
-fi
+remove_all_containers
+kill_all_container_shims

microk8s-resources/wrappers/apiservice-kicker

@@ -65,15 +65,9 @@ do
         echo "cert change detected. Reconfiguring the kube-apiserver"
         rm -rf .srl
         snapctl stop microk8s.daemon-kubelite
-        if is_strict
-        then
-          remove_all_containers
-          snapctl restart microk8s.daemon-containerd
-        else
-          snapctl stop microk8s.daemon-containerd
-          kill_all_container_shims
-          snapctl start microk8s.daemon-containerd
-        fi
+        remove_all_containers
+        kill_all_container_shims
+        snapctl restart microk8s.daemon-containerd
         snapctl start microk8s.daemon-kubelite
         start_all_containers
         restart_attempt=$[$restart_attempt+1]

microk8s-resources/wrappers/microk8s-stop.wrapper

@@ -40,26 +40,22 @@ while true; do
     esac
 done
 
-stopcmd="run_with_sudo snap stop ${SNAP_NAME} --disable"
+prefix_cmd="run_with_sudo snap"
 if is_strict
 then
-  stopcmd="snapctl stop microk8s.daemon-kubelite --disable"
+  prefix_cmd="snapctl"
 fi
 
-$stopcmd
+$prefix_cmd stop microk8s.daemon-kubelite --disable
 stop_status=$?
 
 if ! [ $stop_status -eq 0 ]
 then
     echo 'Failed to stop microk8s services. Check snapd logs with "journalctl -u snapd.service"'
     exit 1
 else
-    if is_strict
-    then
-      remove_all_containers
-      snapctl stop microk8s --disable
-    else
-      kill_all_container_shims
-    fi
+    remove_all_containers
+    kill_all_container_shims
+    $prefix_cmd stop microk8s --disable
     run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock
 fi

snap/hooks/remove

@@ -7,6 +7,15 @@ use_snap_env
 
 snapctl stop ${SNAP_NAME}.daemon-kubelite 2>&1 || true
 
+# Temporarily start containerd so we can stop and kill all the microk8s containers.
+snapctl start ${SNAP_NAME}.daemon-containerd 2>&1 || true
+# wait for containerd to start.
+sleep 5
+
+# Remove any lingering containers and shims.
+remove_all_containers
+kill_all_container_shims
+
 # Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box.
 if test -L /var/lib/kubelet; then
   unlink /var/lib/kubelet || true

tests/libs/utils.sh

@@ -41,7 +41,7 @@ function setup_tests() {
 
   export DEBIAN_FRONTEND=noninteractive
   apt-get install python3-pip -y
-  pip3 install -U pytest requests pyyaml sh
+  pip3 install -U pytest requests pyyaml sh psutil
   apt-get install jq -y
   snap install kubectl --classic
   export ARCH=$(uname -m)

tests/lxc/install-deps/images_almalinux-8

@@ -9,7 +9,7 @@ systemctl enable --now snapd.socket
 ln -s /var/lib/snapd/snap /snap
 yum install python3-pip  -y
 yum install docker -y
-pip3 install pytest requests pyyaml sh
+pip3 install pytest requests pyyaml sh psutil
 
 # wait for the snapd seeding to take place!
 n=0

tests/lxc/install-deps/images_centos-7

@@ -7,7 +7,7 @@ systemctl enable --now snapd.socket
 ln -s /var/lib/snapd/snap /snap
 yum install python3-pip  -y
 yum install docker -y
-pip3 install pytest requests pyyaml sh
+pip3 install pytest requests pyyaml sh psutil
 
 # wait for the snapd seeding to take place!
 n=0

tests/lxc/install-deps/images_centos-8-Stream

@@ -9,7 +9,7 @@ systemctl enable --now snapd.socket
 ln -s /var/lib/snapd/snap /snap
 yum install python3-pip  -y
 yum install docker -y
-pip3 install pytest requests pyyaml sh
+pip3 install pytest requests pyyaml sh psutil
 
 # wait for the snapd seeding to take place!
 n=0