[CI] include pum check in CI to test datamodel upgrades

.build/customizations/sigip/delta/post-all.py

@@ -1,4 +1,5 @@
 import os
+import subprocess
 from pum.core.deltapy import DeltaPy
 
 class RecreateViews(DeltaPy):
@@ -9,6 +10,6 @@ def run(self):
         rewrite_views = os.path.join(self.delta_dir, '..', 'rewrite_views.sh')
         cmd = 'PGSERVICE={} SRID=21781 {}'.format(self.pg_service, rewrite_views)
         self.write_message(cmd)
-        os.system(cmd)
+        subprocess.check_output(cmd, shell=True)
 
         self.write_message('Recreating sigip views: done')

.build/customizations/sigip/delta/pre-all.py

@@ -1,4 +1,5 @@
 import os
+import subprocess
 from pum.core.deltapy import DeltaPy
 
 
@@ -10,7 +11,7 @@ def run(self):
         drop_sql = os.path.join(self.delta_dir, '..', 'drop_views.sql')
         cmd = 'psql service={} -f {}'.format(self.pg_service, drop_sql)
         self.write_message(cmd)
-        os.system(cmd)
+        subprocess.check_output(cmd, shell=True)
 
         self.write_message('Dropping sigip views: done')
 

.github/workflows/run_tests.yml

@@ -2,6 +2,7 @@ name: Run tests
 on: push
 
 jobs:
+
   run-tests:
     runs-on: ubuntu-latest
 
@@ -41,3 +42,6 @@ jobs:
 
       - name: Run scalability tests multithreaded
         run: tests/tests_scalability_multithread.sh -i 20
+
+      - name: Run upgrade tests
+        run: tests/tests_upgrade.sh

init_qwat.sh

@@ -146,6 +146,7 @@ psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_object_ref
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_bedding.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_cistern.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_cover_type.sql
+psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_document_type.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_hydrant_model.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_hydrant_material.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -f ${DIR}/value_lists/vl_hydrant_output.sql
@@ -203,6 +204,7 @@ psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_da
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_data/areas/od_folder.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_data/areas/od_printmap.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_data/areas/od_protectionzone.sql
+psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_data/document/document.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_data/functions/fn_get_district.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_data/functions/fn_get_printmap.sql
 psql service=${PGSERVICE} -v ON_ERROR_STOP=1 -v SRID=$SRID -f ${DIR}/ordinary_data/functions/fn_pressurezone.sql

requirements.txt

@@ -1,3 +1,3 @@
-pum
+pum>=0.10.0
 psycopg2-binary
 pyyaml

tests/pg_service.conf

@@ -3,3 +3,9 @@ host=localhost
 dbname=qwat_test
 user=postgres
 password=postgres
+
+[qwat_test_release]
+host=localhost
+dbname=qwat_test_release
+user=postgres
+password=postgres

tests/tests_upgrade.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+# This tests that the database upgraded by PUM is consistent to the database
+# initialized by qwat_init.sh
+#It retrieves the
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+INIT_DB=1
+
+while [[ $# > 0 ]]; do
+key="$1"
+case $key in
+    -h|--help)
+        echo "Arguments:"
+        echo -e "\t-h|--help\tShow this help screen"
+        echo -e "\t-n|--no-init\tDo not init the test database"
+        exit 0
+        ;;
+    -n|--no-init)
+        INIT_DB=0
+        ;;
+esac
+
+shift
+done
+
+PGSERVICE1=qwat_test
+PGSERVICE2=qwat_test_release
+VERSION=1.3.6
+RELEASE_URL=https://github.com/qwat/qwat-data-model/releases/download/${VERSION}/qwat_v${VERSION}_data_and_structure_sample.backup
+RELEASE_LOCATION=/tmp/qwat_v${VERSION}.backup
+
+# Download the release if needed
+if [ -f "${RELEASE_LOCATION}" ]; then
+    echo "Release file '$RELEASE_LOCATION' already exists, skipping download."
+else
+    wget ${RELEASE_URL} -O ${RELEASE_LOCATION} --no-verbose
+fi
+
+# Initialize the databases (qwat_test for the init data, qwat_test_release for restoring the release dump)
+if [ "$INIT_DB" = "1" ]; then
+    cd ${DIR}/..
+    ./init_qwat.sh -p ${PGSERVICE1} -d
+    cd -
+fi
+
+# Restore the release to qwat_test_release
+psql service=${PGSERVICE1} -c 'DROP DATABASE IF EXISTS qwat_test_release'
+psql service=${PGSERVICE1} -c 'CREATE DATABASE qwat_test_release'
+pum restore -p ${PGSERVICE2} -x ${RELEASE_LOCATION}
+
+# Drop qwat_sigip schema that is included in the release but that seems to
+# make migrations fail.
+# TODO: qwat_sigip should be probably be removed from the dumps, or if really obsolete,
+# removed from the source and dumped in a migration
+psql service=${PGSERVICE2} -c 'DROP SCHEMA qwat_sigip CASCADE'
+# despite being in qwat_od, this seems to be a qwat_sigip customization (created in data-model/.build/customizations/sigip/damage/damage.sql)
+psql service=${PGSERVICE2} -c 'DROP TABLE qwat_od.damage CASCADE'
+
+# Upgrade using pum
+pum upgrade -p ${PGSERVICE2} -t qwat_sys.info -d ${DIR}/../update/delta
+
+# Audit triggers are enabled in the init scripts, but not in the released 1.3.6 dump.
+# so we need to enable it, otherwise pum check will detect differences.
+psql service=${PGSERVICE2} -c 'SELECT qwat_sys.activate_audit_views()'
+
+# Compare results
+pum check -p1 ${PGSERVICE1} -p2 ${PGSERVICE2} --exclude-schema public --exclude-schema tiger --exclude-schema topology
+
+echo "Success 😁 !!"

update/delta/delta_1.2.8_schema_visible_perf.pre.py

@@ -1,5 +1,6 @@
 from pum.core.deltapy import DeltaPy
 import os
+import subprocess
 
 class DropViews128(DeltaPy):
 
@@ -16,5 +17,5 @@ def run(self):
         self.write_message(cmd)
 
         # Execute command
-        os.system(cmd)
+        subprocess.check_output(cmd, shell=True)
         self.write_message("Dropping views: done")

update/delta/delta_1.4.0_002_fix_pipe_reverse_nodes.sql

@@ -5,7 +5,7 @@ CREATE TRIGGER tr_pipe_geom_update
     ON qwat_od.pipe
     FOR EACH ROW
     WHEN (ST_OrderingEquals(st_force2d(new.geometry), st_force2d(old.geometry)) IS FALSE)
-    EXECUTE FUNCTION qwat_od.ft_pipe_geom();
+    EXECUTE PROCEDURE qwat_od.ft_pipe_geom();
 
 COMMENT ON TRIGGER tr_pipe_geom_update ON qwat_od.pipe
     IS 'Trigger: updates auto fields of the pipe after geom update.';

update/delta/delta_1.4.0_005_normalize_index_name.sql

@@ -0,0 +1,2 @@
+-- This index was named differently in the delta than it is in the init script
+ALTER INDEX qwat_od.fki_leak_type RENAME TO fki_leak_fk_type;

update/delta/post-all.py

@@ -21,7 +21,7 @@ def run(self):
                                     'rewrite_functions.sh')
         functions_cmd = 'PGSERVICE={} SRID={} {}'.format(
             self.pg_service, srid, functions_sh)
-        os.system(views_cmd)
-        os.system(functions_cmd)
+        subprocess.check_output(views_cmd, shell=True)
+        subprocess.check_output(functions_cmd, shell=True)
 
         self.write_message("Reloading views and functions: done")

update/delta/pre-all.py

@@ -1,6 +1,7 @@
 from pum.core.deltapy import DeltaPy
 import pkg_resources
 import os
+import subprocess
 
 class DropViews(DeltaPy):
 
@@ -22,6 +23,6 @@ def drop_views(self):
                                 'drop_views.sql')
         cmd = 'psql service={} -f {}'.format(self.pg_service, drop_sql)
         self.write_message(cmd)
-        os.system(cmd)
+        subprocess.check_output(cmd, shell=True)
 
         self.write_message("Dropping views: done")