Merge pull request #358 from apel/release-3.4.1

tofu-rocketry · web-flow · commit 6faa077a94b7 · 2024-09-02T12:51:32.000+01:00
Release 3.4.1 to master
diff --git a/.github/workflows/build-pkgs.yml b/.github/workflows/build-pkgs.yml
@@ -51,7 +51,7 @@ jobs:
       run: rpmlint ${{ steps.rpm.outputs.rpm_dir_path }}
 
     - name: Upload artifact
-      uses: actions/upload-artifact@v4.3.1
+      uses: actions/upload-artifact@v4.3.6
       with:
         name: Binary and Source RPMs
         path: |
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -48,7 +48,7 @@ jobs:
         # Build and push Docker image
         # https://github.com/docker/build-push-action
         name: Build and push Docker image
-        uses: docker/build-push-action@v5.1.0
+        uses: docker/build-push-action@v6.7.0
         with:
           # Only push containers to the registry on GitHub pushes,
           # not pull requests. GitHub won't let a rogue PR create a container
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
@@ -4,31 +4,40 @@ on: [push, pull_request]
 
 jobs:
   unit-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04  # 20.04 to allow for Py 3.6
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.x']
+        # Python versions on Rocky 8, Ubuntu 20.04, Rocky 9
+        python-version: ['3.6', '3.8', '3.9']
     name: Python ${{ matrix.python-version }} test
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python
+
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+
     - name: Set up dependencies for python-ldap
       run: sudo apt-get install libsasl2-dev libldap2-dev libssl-dev
+
     - name: Base requirements for SSM
       run: pip install -r requirements.txt
+
     - name: Additional requirements for the unit and coverage tests
       run: pip install -r requirements-test.txt
+
     - name: Pre-test set up
       run: |
         export TMPDIR=$PWD/tmp
         mkdir $TMPDIR
         export PYTHONPATH=$PYTHONPATH:`pwd -P`
         cd test
+
     - name: Run unit tests
       run: coverage run --branch --source=ssm,bin -m unittest discover --buffer
+
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3.1.4
+      uses: codecov/codecov-action@v4
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 # See https://pre-commit.com for more information
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.5.0
+  rev: v4.1.0  # Python 3.6 compatible
   hooks:
   # Python related checks
   - id: check-ast
@@ -13,9 +13,13 @@ repos:
     files: 'test/.*'
   # Other checks
   - id: check-added-large-files
+  - id: check-case-conflict
   - id: check-merge-conflict
   - id: check-yaml
   - id: debug-statements
+  - id: detect-private-key
+    # This file has a test cert and key
+    exclude: 'test_ssm.py'
   - id: end-of-file-fixer
   - id: mixed-line-ending
     name: Force line endings to LF
diff --git a/.travis.yml b/.travis.yml
@@ -2,7 +2,6 @@ os: linux
 language: python
 python:
   - "2.7"
-  - "3.8"
 
 # Cache the dependencies installed by pip
 cache: pip
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,10 @@
+* Fri Aug 30 2024 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 3.4.1-1
+ - Improved error logging to store full traceback on unexpected exceptions.
+ - Changed more code to use pyOpenSSL to improve compatibility with newer OpenSSL versions.
+ - Added a check to prevent a host certificate being to used for target server encryption.
+ - Changed which version of exit function is used to avoid edge case.
+ - Various changes and improvements to build scripts and processes.
+
 * Wed Feb 21 2024 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 3.4.0-1
  - Fixed compatability with newer versions of OpenSSL that only provide comma separated DNs.
  - Fixed Python 3 compatability (indirectly fixing EL8+ compatability) by performing explicit
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,9 @@
-FROM centos:7
-MAINTAINER APEL Administrators <apel-admins@stfc.ac.uk>
+FROM rockylinux:9
+LABEL org.opencontainers.image.authors="apel-admins@stfc.ac.uk"
+LABEL org.opencontainers.image.title="APEL SSM"
+LABEL org.opencontainers.image.description="Secure STOMP Messenger (SSM) is designed to simply send messages using the STOMP protocol or via the ARGO Messaging Service (AMS)."
+LABEL org.opencontainers.image.source="https://github.com/apel/ssm"
+LABEL org.opencontainers.image.licenses="Apache License, Version 2.0"
 
 # Copy the SSM Git repository to /tmp/ssm
 COPY . /tmp/ssm
@@ -9,10 +13,10 @@ WORKDIR /tmp/ssm
 # Add the EPEL repo so we can get pip
 RUN yum -y install epel-release && yum clean all
 # Then get pip
-RUN yum -y install python-pip && yum clean all
+RUN yum -y install python3-pip && yum clean all
 
 # Install the system requirements of python-ldap
-RUN yum -y install gcc python-devel openldap-devel && yum clean all
+RUN yum -y install gcc python3-devel openldap-devel && yum clean all
 
 # Install libffi, a requirement of openssl
 RUN yum -y install libffi-devel && yum clean all
@@ -21,9 +25,9 @@ RUN yum -y install libffi-devel && yum clean all
 RUN yum -y install openssl && yum clean all
 
 # Install the python requirements of SSM
-RUN pip install -r requirements.txt
+RUN pip install -r requirements-docker.txt
 # Then install the SSM
-RUN python setup.py install
+RUN python3 setup.py install
 
 # Set the working directory back to /
 WORKDIR /
diff --git a/apel-ssm.spec b/apel-ssm.spec
@@ -4,7 +4,7 @@
 %endif
 
 Name:           apel-ssm
-Version:        3.4.0
+Version:        3.4.1
 %define releasenumber 1
 Release:        %{releasenumber}%{?dist}
 Summary:        Secure stomp messenger
@@ -100,6 +100,13 @@ rm -rf $RPM_BUILD_ROOT
 %doc %_defaultdocdir/%{name}
 
 %changelog
+* Fri Aug 30 2024 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 3.4.1-1
+ - Improved error logging to store full traceback on unexpected exceptions.
+ - Changed more code to use pyOpenSSL to improve compatibility with newer OpenSSL versions.
+ - Added a check to prevent a host certificate being to used for target server encryption.
+ - Changed which version of exit function is used to avoid edge case.
+ - Various changes and improvements to build scripts and processes.
+
 * Wed Feb 21 2024 Adrian Coveney <adrian.coveney@stfc.ac.uk> - 3.4.0-1
  - Fixed compatability with newer versions of OpenSSL that only provide comma separated DNs.
  - Fixed Python 3 compatability (indirectly fixing EL8+ compatability) by performing explicit
diff --git a/bin/receiver.py b/bin/receiver.py
@@ -66,7 +66,7 @@ def main():
         cp.read(options.config)
     else:
         print("Config file not found at", options.config)
-        exit(1)
+        sys.exit(1)
 
     # Check for pidfile
     pidfile = cp.get('daemon', 'pidfile')
diff --git a/bin/sender.py b/bin/sender.py
@@ -24,6 +24,7 @@
 import logging
 from optparse import OptionParser
 import os
+import sys
 
 try:
     import ConfigParser
@@ -57,7 +58,7 @@ def main():
         cp.read(options.config)
     else:
         print("Config file not found at", options.config)
-        exit(1)
+        sys.exit(1)
 
     ssm.agents.logging_helper(cp)
 
diff --git a/requirements-docker.txt b/requirements-docker.txt
@@ -0,0 +1,12 @@
+# Base requirements for ssm
+
+argo-ams-library
+pyopenssl
+cryptography
+stomp.py
+python-daemon
+python-ldap
+setuptools  # Required for pkg_resources (also happens to be a dependency of python-ldap)
+
+# Dependencies for optional dirq based sending
+dirq
diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh
@@ -16,7 +16,7 @@
 
 set -eu
 
-TAG=3.4.0-1
+TAG=3.4.1-1
 
 SOURCE_DIR=~/debbuild/source
 BUILD_DIR=~/debbuild/build
diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh
@@ -10,7 +10,7 @@
 rpmdev-setuptree
 
 RPMDIR=/home/rpmb/rpmbuild
-VERSION=3.4.0-1
+VERSION=3.4.1-1
 SSMDIR=apel-ssm-$VERSION
 
 # Remove old sources and RPMS
diff --git a/scripts/ssm-build.sh b/scripts/ssm-build.sh
@@ -5,10 +5,13 @@
 # @Author: Nicholas Whyatt (RedProkofiev@github.com)
 
 # Script runs well with FPM 1.14.2 on ruby 2.7.1, setuptools 51.3.3 on RHEL and Deb platforms
-# Download ruby (if you're locked to 2.5, use RVM) and then run:
+
+# Download ruby (if you're locked to 2.5, use RVM, https://www.tecmint.com/install-ruby-on-centos-rhel-8/#installrubyrvm) and then run:
 # sudo gem install fpm -v 1.14.2
+# (may need to be run without the 'sudo')
+
 # for RPM builds, you will also need:
-# sudo yum install rpm-build | sudo apt-get install rpm
+# sudo yum install rpm-build rpmlint | sudo apt-get install rpm lintian
 # ./ssm-build.sh (deb | rpm) <version> <iteration> <python_root_dir>
 # e.g.
 # ./ssm-build.sh deb 3.4.0 1 /usr/lib/python3.6
@@ -107,6 +110,7 @@ rm -f "$TAR_FILE"
 # Get supplied Python version
 PY_VERSION="$(basename "$PYTHON_ROOT_DIR")"
 PY_NUM=${PY_VERSION#python}
+OS_EXTENSION="$(uname -r | grep -o 'el[7-9]' || echo '_all')"
 
 # Universal FPM Call
 FPM_CORE="fpm -s python \
@@ -127,25 +131,27 @@ if [[ ${PY_NUM:0:1} == "3" ]]; then
 
     if [[ "$PACK_TYPE" = "deb" ]]; then
         FPM_PYTHON="--depends python3 \
-        --depends python-pip3 \
-        --depends 'python-stomp' \
-        --depends python-ldap \
+        --depends python3-pip \
+        --depends python3-cryptography \
+        --depends python3-openssl \
+        --depends python3-daemon \
+        --depends 'python3-stomp' \
+        --depends python3-ldap \
         --depends libssl-dev \
         --depends libsasl2-dev \
         --depends openssl "
 
-        OS_EXTENSION="_all"
-
     # Currently builds for el8
     elif [[ "$PACK_TYPE" = "rpm" ]]; then
         FPM_PYTHON="--depends python3 \
         --depends python3-stomppy \
         --depends python3-pip \
+        --depends python3-cryptography \
+        --depends python3-pyOpenSSL \
+        --depends python3-daemon \
         --depends python3-ldap \
         --depends openssl \
         --depends openssl-devel "
-
-        OS_EXTENSION="el8"
     fi
 
 elif [[ ${PY_NUM:0:1} == "2" ]]; then
@@ -156,29 +162,32 @@ elif [[ ${PY_NUM:0:1} == "2" ]]; then
         --depends python-pip \
         --depends 'python-stomp < 5.0.0' \
         --depends python-ldap \
+        --depends python-cryptography \
+        --depends python-openssl \
+        --depends python-daemon \
         --depends libssl-dev \
         --depends libsasl2-dev \
         --depends openssl "
 
-        OS_EXTENSION="_all"
-
     # el7 and below, due to yum package versions
     elif [[ "$PACK_TYPE" = "rpm" ]]; then
         FPM_PYTHON="--depends python2 \
         --depends python2-pip \
+        --depends python2-cryptography \
+        --depends python2-pyOpenSSL \
+        --depends python2-daemon \
         --depends stomppy \
         --depends python-ldap \
         --depends openssl \
         --depends openssl-devel "
-
-        OS_EXTENSION="el7"
     fi
 fi
 
 # python-bin must always be specified in modern linux
 PACKAGE_VERSION="--$PACK_TYPE-changelog $SOURCE_DIR/ssm-$VERSION-$ITERATION/CHANGELOG \
     --$PACK_TYPE-dist $OS_EXTENSION \
     --python-bin /usr/bin/$PY_VERSION \
+    --python-install-bin /usr/bin \
     --python-install-lib $PYTHON_ROOT_DIR$LIB_EXTENSION \
     --exclude *.pyc \
     --package $BUILD_DIR \
@@ -203,3 +212,18 @@ fpm -s pleaserun -t "$PACK_TYPE" \
 --depends apel-ssm \
 --package "$BUILD_DIR" \
 /usr/bin/ssmreceive
+
+echo "Possible Issues to Fix:"
+if [ "$OS_EXTENSION" == "_all" ]
+then
+    # Check the resultant debs for 'lint'
+    TAG="$VERSION-$ITERATION"
+    DEBDIR="$(dirname "$BUILD_DIR")"
+
+    lintian "$DEBDIR"/apel-ssm_"${TAG}"_all.deb
+    lintian "$DEBDIR"/apel-ssm-service_"${TAG}"_all.deb
+else
+    # Check for errors in SPEC and built packages
+    # For instance; Given $(dirname /root/rpmb/rpmbuild/source) will output "/root/rpmb/rpmbuild".
+    rpmlint "$(dirname "$SOURCE_DIR")"
+fi
diff --git a/setup.py b/setup.py
diff --git a/ssm/__init__.py b/ssm/__init__.py
diff --git a/ssm/agents.py b/ssm/agents.py
diff --git a/ssm/crypto.py b/ssm/crypto.py

-Original file line number
+Diff line change
       run: rpmlint ${{ steps.rpm.outputs.rpm_dir_path }}
     - name: Upload artifact
 -      uses: actions/[email protected].1
 +      uses: actions/[email protected].6
       with:
         name: Binary and Source RPMs
         path: |