tests: Bump DevStack to Epoxy (2025.1) (#2742)

* tests: Prefer ginkgo timeout

So that we actually get test results.

Signed-off-by: Stephen Finucane <[email protected]>

* tests: Align opts for Cinder, Manila tests

Add a timeout to the Manila job and otherwise move some lines around.

Signed-off-by: Stephen Finucane <[email protected]>

* tests: Suffix image version

While boskos will reap most resources for us, it doesn't reap images
[1]. This has resulted in us using the same image for who knows how long
at this point.

Encode the Ubuntu version to prevent us picking up other version by
mistake.

[1] https://github.com/kubernetes-sigs/boskos/blob/5993cef5a1c719c33c0936d416b7d935058e1204/cmd/janitor/gcp_janitor.py#L46-L88

Signed-off-by: Stephen Finucane <[email protected]>

* tests: Bump DevStack to Epoxy (2025.1)

Signed-off-by: Stephen Finucane <[email protected]>

* tests: Bump amphora image

Use the Ubuntu 24.04 version, rather than the 22.04 version.
This aligns with what we're using for DevStack itself.

Signed-off-by: Stephen Finucane <[email protected]>

* devstack: Remove USE_PYTHON3

It's all Python 3 now, baby.

Signed-off-by: Stephen Finucane <[email protected]>

* tests: Install Ansible from Debian Testing

Signed-off-by: Stephen Finucane <[email protected]>

* tests: Correct broken conditions

Per the Ansible 2.19 porting guide [1].

[1] https://ansible.readthedocs.io/projects/ansible-core/devel/porting_guides/porting_guide_core_2.19.html

Signed-off-by: Stephen Finucane <[email protected]>

* tests: Decrease device detach timeout

Wait less time before retrying.

Signed-off-by: Stephen Finucane <[email protected]>

---------

Signed-off-by: Stephen Finucane <[email protected]>

Author: stephenfin

tests/ci-csi-cinder-e2e.sh

@@ -41,6 +41,24 @@ cleanup() {
 }
 trap cleanup EXIT
 
+# Install Ansible from Trixie (Debian Testing) to allow us to talk to a host
+# running Python 3.12 like Ubuntu 24.04
+#
+# https://woju.eu/blog/2025/03/ansible-bookworm/
+cat << EOF > /etc/apt/sources.list.d/trixie.list
+deb https://deb.debian.org/debian trixie main
+EOF
+
+cat << EOF > /etc/apt/preferences.d/ansible.pref
+Package: *
+Pin: release n=trixie
+Pin-Priority: -10
+
+Package: ansible* python3-netaddr
+Pin: release n=trixie
+Pin-Priority: 990
+EOF
+
 apt-get update
 apt-get install -y python3-requests ansible
 

tests/ci-csi-manila-e2e.sh

@@ -41,6 +41,24 @@ cleanup() {
 }
 trap cleanup EXIT
 
+# Install Ansible from Trixie (Debian Testing) to allow us to talk to a host
+# running Python 3.12 like Ubuntu 24.04
+#
+# https://woju.eu/blog/2025/03/ansible-bookworm/
+cat << EOF > /etc/apt/sources.list.d/trixie.list
+deb https://deb.debian.org/debian trixie main
+EOF
+
+cat << EOF > /etc/apt/preferences.d/ansible.pref
+Package: *
+Pin: release n=trixie
+Pin-Priority: -10
+
+Package: ansible* python3-netaddr
+Pin: release n=trixie
+Pin-Priority: 990
+EOF
+
 apt-get update
 apt-get install -y python3-requests ansible
 

tests/ci-occm-e2e.sh

@@ -42,6 +42,24 @@ cleanup() {
 }
 trap cleanup EXIT
 
+# Install Ansible from Trixie (Debian Testing) to allow us to talk to a host
+# running Python 3.12 like Ubuntu 24.04
+#
+# https://woju.eu/blog/2025/03/ansible-bookworm/
+cat << EOF > /etc/apt/sources.list.d/trixie.list
+deb https://deb.debian.org/debian trixie main
+EOF
+
+cat << EOF > /etc/apt/preferences.d/ansible.pref
+Package: *
+Pin: release n=trixie
+Pin-Priority: -10
+
+Package: ansible* python3-netaddr
+Pin: release n=trixie
+Pin-Priority: 990
+EOF
+
 apt-get update
 apt-get install -y python3-requests ansible
 
@@ -133,4 +151,4 @@ scp -i ~/.ssh/google_compute_engine \
 # If Boskos is being used then release the resource back to Boskos.
 [ -z "${BOSKOS_HOST:-}" ] || python3 tests/scripts/boskos.py --release >> "$ARTIFACTS/logs/boskos.log" 2>&1
 
-exit ${exit_code}
+exit ${exit_code}

tests/playbooks/roles/install-cpo-occm/tasks/main.yaml

@@ -128,7 +128,7 @@
         msg: *failmsg
 
 - name: Run functional tests for openstack-cloud-controller-manager
-  when: run_e2e
+  when: run_e2e | bool
   register: run_tests
   shell:
     executable: /bin/bash
@@ -145,7 +145,7 @@
 
 - name: Print logs for debugging
   when:
-    - run_e2e
+    - run_e2e | bool
     - run_tests.failed
   block:
   - name: Show openstack-cloud-controller-manager pod logs

tests/playbooks/roles/install-csi-cinder/tasks/main.yaml

@@ -187,15 +187,15 @@
         -storage.testdriver=tests/e2e/csi/cinder/test-driver.yaml \
         --ginkgo.focus='External.Storage' \
         --ginkgo.skip='\[Disruptive\]|\[Testpattern:\s+Dynamic\s+PV\s+\(default\s+fs\)\]\s+provisioning\s+should\s+mount\s+multiple\s+PV\s+pointing\s+to\s+the\s+same\s+storage\s+on\s+the\s+same\s+node|\[Testpattern:\s+Dynamic\s+PV\s+\(default\s+fs\)\]\s+provisioning\s+should\s+provision\s+storage\s+with\s+any\s+volume\s+data\s+source\s+\[Serial\]|should\s+support\s+expansion\s+of\s+pvcs\s+created\s+for\s+ephemeral\s+pvcs' \
+        --ginkgo.v \
         --ginkgo.noColor \
         --ginkgo.progress \
-        --ginkgo.v \
-        --ginkgo.timeout=24h \
+        --ginkgo.timeout=1h30m \
         -test.timeout=0 \
         -report-dir="/var/log/csi-pod" | tee "/var/log/csi-pod/cinder-csi-e2e.log"
   register: functional_test_result
   ignore_errors: true
-  async: 5400 # wait 1h30m then fail and fetch the logs
+  async: 5700 # wait 1h35m (i.e. 5 mins longer than the ginkgo timeout) then fail and fetch the logs
   poll: 15
 
 - name: Collect pod logs for debug purpose

tests/playbooks/roles/install-csi-manila/tasks/main.yaml

@@ -227,21 +227,23 @@
       set +x; source {{ devstack_workdir }}/openrc demo demo > /dev/null; set -x
 
       cd {{ ansible_user_dir }}/src/k8s.io/cloud-provider-openstack
-
-      # GATEWAY_IP is the default value in devstack
       mkdir -p /var/log/csi-pod
+      # GATEWAY_IP is the default value in devstack
       GATEWAY_IP=172.24.5.1 \
       OS_RC={{ devstack_workdir }}/openrc \
       go test -v ./cmd/tests/manila-csi-e2e-suite/manila_csi_e2e_suite_test.go \
+        --ginkgo.focus="\[manila-csi-e2e\]" \
+        --ginkgo.skip="\[Disruptive\]|\[sig-storage\]\s+\[manila-csi-e2e\]\s+CSI\s+Volumes\s+\[Driver:\s+nfs.manila.csi.openstack.org\]\s+\[Testpattern:\s+Dynamic\s+PV\s+\(default\s+fs\)\]\s+provisioning\s+should\s+provision\s+storage\s+with\s+any\s+volume\s+data\s+source\s+\[Serial\]|should\s+provision\s+storage\s+with\s+snapshot\s+data\s+source|restoring\s+snapshot\s+to\s+larger\s+size" \
         --ginkgo.v \
         --ginkgo.noColor \
         --ginkgo.progress \
-        --ginkgo.skip="\[Disruptive\]|\[sig-storage\]\s+\[manila-csi-e2e\]\s+CSI\s+Volumes\s+\[Driver:\s+nfs.manila.csi.openstack.org\]\s+\[Testpattern:\s+Dynamic\s+PV\s+\(default\s+fs\)\]\s+provisioning\s+should\s+provision\s+storage\s+with\s+any\s+volume\s+data\s+source\s+\[Serial\]|should\s+provision\s+storage\s+with\s+snapshot\s+data\s+source|restoring\s+snapshot\s+to\s+larger\s+size" \
-        --ginkgo.focus="\[manila-csi-e2e\]" \
-        -report-dir /var/log/csi-pod \
-        -timeout=0 | tee "/var/log/csi-pod/manila-csi-e2e.log"
+        --ginkgo.timeout=1h30m \
+        -timeout=0 \
+        -report-dir /var/log/csi-pod | tee "/var/log/csi-pod/manila-csi-e2e.log"
   register: functional_test_result
   ignore_errors: true
+  async: 5700 # wait 1h35m (i.e. 5 mins longer than the ginkgo timeout) then fail and fetch the logs
+  poll: 15
 
 - name: Collect pod logs for debug purpose
   shell:

tests/playbooks/roles/install-devstack/defaults/main.yaml

@@ -1,7 +1,7 @@
 ---
 user: "stack"
 workdir: "/home/{{ user }}/devstack"
-branch: "2023.2-eol"
+branch: "stable/2025.1"
 enable_services:
   - nova
   - glance
@@ -10,6 +10,6 @@ enable_services:
   - octavia
   - ovn-octavia
   - barbican
-octavia_amphora_url: "https://tarballs.opendev.org/openstack/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-jammy.qcow2"
+octavia_amphora_url: "https://tarballs.opendev.org/openstack/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-noble.qcow2"
 octavia_amphora_dir: /opt/octavia-amphora
 octavia_amphora_filename: amphora-x64-haproxy.qcow2

tests/playbooks/roles/install-devstack/templates/local.conf.j2

@@ -3,7 +3,6 @@ RECLONE=False
 HOST_IP={{ local_ip_address }}
 DEST=/opt/stack
 DATA_DIR=${DEST}/data
-USE_PYTHON3=True
 LOGFILE=$DEST/logs/stack.sh.log
 VERBOSE=True
 LOG_COLOR=False
@@ -49,6 +48,7 @@ enable_service n-cond
 enable_service n-sch
 enable_service n-api-meta
 
+# Placement
 enable_service placement-api
 enable_service placement-client
 {% endif %}
@@ -148,6 +148,16 @@ if [ -f /opt/stack/data/venv/bin/activate ]; then
     deactivate
 fi
 
+{% if "nova" in enable_services %}
+[[post-config|$NOVA_CONF]]
+[libvirt]
+# Workaround for nova bug #2119114. Until that bug is resolved, Nova will
+# always timeout when detaching volumes (which affects deletion of PVCs)
+#
+# https://bugs.launchpad.net/nova/+bug/2119114
+device_detach_timeout = 1
+{% endif %}
+
 {% if "glance" in enable_services %}
 [[post-config|$GLANCE_API_CONF]]
 [glance_store]

tests/playbooks/test-occm-e2e.yaml

@@ -25,5 +25,5 @@
     - role: install-k3s
       worker_node_count: 0
     - role: install-cpo-occm
-      run_e2e: "{{ run_e2e }}"
+      run_e2e: "{{ run_e2e | bool }}"
       octavia_provider: "{{ octavia_provider }}"

tests/scripts/create-gce-vm.sh

@@ -91,17 +91,18 @@ main() {
 
   case "${FLAVOR}" in
   "default")
-    if ! gcloud compute disks describe devstack-${FLAVOR} --zone "${GCP_ZONE}" > /dev/null 2>&1;
+    local disk_name="devstack-${FLAVOR}-ubuntu2404"
+    if ! gcloud compute disks describe "${disk_name}" --zone "${GCP_ZONE}" > /dev/null 2>&1;
     then
-      gcloud compute disks create devstack-${FLAVOR} \
+      gcloud compute disks create "${disk_name}" \
         --image-project ubuntu-os-cloud --image-family ubuntu-2404-lts-amd64 \
         --zone "${GCP_ZONE}"
     fi
 
-    if ! gcloud compute images describe devstack-${FLAVOR} > /dev/null 2>&1;
+    if ! gcloud compute images describe "${disk_name}" > /dev/null 2>&1;
     then
-      gcloud compute images create devstack-${FLAVOR} \
-        --source-disk devstack-${FLAVOR} --source-disk-zone "${GCP_ZONE}" \
+      gcloud compute images create "${disk_name}" \
+        --source-disk "${disk_name}" --source-disk-zone "${GCP_ZONE}" \
         --licenses "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
     fi
     ;;
@@ -115,7 +116,7 @@ main() {
   then
     gcloud compute instances create devstack \
       --zone "${GCP_ZONE}" \
-      --image devstack-${FLAVOR} \
+      --image "${disk_name}" \
       --boot-disk-size 30G \
       --boot-disk-type pd-ssd \
       --can-ip-forward \