Merge pull request #163 from apel/release-3.2.1

Release 3.2.1 to master

Author: tofu-rocketry

CHANGELOG

@@ -1,5 +1,9 @@
 Changelog for ssm
 =================
+* Wed Apr 28 2021 Adrian Coveney <[email protected]> - 3.2.1-1
+ - Changed certificate and key comparison to allow both RSA and EC keys.
+ - Corrected dependencies to include OpenSSL.
+
 * Thu Mar 18 2021 Adrian Coveney <[email protected]> - 3.2.0-1
  - Added logging of what certificate DNs/subjects are being used to facilitate
    troubleshooting.

README.md

@@ -32,6 +32,7 @@ SSM is provided by [STFC](https://stfc.ukri.org/), a part of [UK Research and In
 The EPEL repository must be enabled.  This can be done by installing
 the RPM for your version of SL, which is available on this page:
 http://fedoraproject.org/wiki/EPEL
+You will also need to have the OpenSSL library installed. Other prerequisites are listed below.
 
 The Python STOMP library (N.B. versions between 3.1.1 (inclusive) and 5.0.0
 (exclusive) are currently supported)

apel-ssm.spec

@@ -4,7 +4,7 @@
 %endif
 
 Name:           apel-ssm
-Version:        3.2.0
+Version:        3.2.1
 %define releasenumber 1
 Release:        %{releasenumber}%{?dist}
 Summary:        Secure stomp messenger
@@ -21,7 +21,7 @@ BuildArch:      noarch
 BuildRequires:  python-devel
 %endif
 
-Requires:       stomppy < 5.0.0, python-daemon, python-ldap
+Requires:       stomppy < 5.0.0, python-daemon, python-ldap, openssl
 Requires(pre):  shadow-utils
 
 %define ssmconf %_sysconfdir/apel
@@ -100,6 +100,10 @@ rm -rf $RPM_BUILD_ROOT
 %doc %_defaultdocdir/%{name}
 
 %changelog
+* Wed Apr 28 2021 Adrian Coveney <[email protected]> - 3.2.1-1
+ - Changed certificate and key comparison to allow both RSA and EC keys.
+ - Corrected dependencies to include OpenSSL.
+
 * Thu Mar 18 2021 Adrian Coveney <[email protected]> - 3.2.0-1
  - Added logging of what certificate DNs/subjects are being used to facilitate
    troubleshooting.

migrating_to_ams.md

@@ -1,10 +1,25 @@
 # Migrating from using EGI ActiveMQ Message Brokers to using EGI ARGO Messaging Service
 
+## Prerequisites for using AMS
+
+- A valid host certificate from an IGTF Accredited CA.
+- A GOCDB 'Site' entry flagged as 'Production'.
+- A GOCDB 'Service' entry of the correct service type flagged as 'Production'. The following service types are used:
+  - For Grid accounting use 'gLite-APEL'.
+  - For Cloud accounting use 'eu.egi.cloud.accounting'.
+  - For Storage accounting use 'eu.egi.storage.accounting'.
+- The 'Host DN' listed in the GOCDB 'Service' entry must exactly match the certificate DN of the host used for accounting. Make sure there are no leading or trailing spaces in the 'Host DN' field.
+- Messages sent via AMS must be below 1 Megabyte in size, and the messaging service is optimised around this limit. If your messages start hitting this limit when using SSM, see the advice at the bottom of this document.
+
+## Software requirements
+
 Migration requires upgrading APEL SSM to at least version 2.4.0, installing the ARGO AMS Library, and adding new values to your configuration.
 
 The ARGO AMS Library is available in UMD as `python-argo-ams-library`. Versions above 0.5.0 are recommended.
 
-## Sender
+## Configuration changes
+
+### Sender
 
 The sender configuration is usually found under `/etc/apel/sender.cfg`. Follow the steps below to migrate.
 
@@ -42,11 +57,30 @@ The next time `ssmsend` runs it should be using the AMS. You can check this by l
 2018-09-19 14:18:07,862 - ssmsend - INFO - ========================================
 ```
 
-## Receiver
+### Receiver
+
+This is only used for the central Accounting Repository, Accounting Portal, and regional accounting servers.
 
 1. Follow the steps 1 to 4 as per the [Sender documentation](#Sender) but editing your receiver configuration instead, usually found under `/etc/apel/receiver.cfg`, naming the section `[receiver]` rather than `[sender]`.
 1. Change `destination` to be the subscription you are using to pull messages down.
 1. Add your token to the `[messaging]` section of your configuration:
    ```
    token: your_token_here
    ```
+
+## Issues
+
+### Messages too large
+
+- Cloud sites using cASO should ensure they are using at least version 1.4.0 of cASO as this version limits the number of records in a message.
+- Grid sites using the APEL accounting client need to be using APEL 1.9.0 and SSM 3.2.0. They can then modify their APEL client script, usually located at `/usr/bin/apelclient`. At the moment, this requires a manual change, but will become a configuration option in the next version of APEL. For example, to halve the number of records per message from the default of 1000, add the line `unloader.records_per_message = 500` after the call to `DbUnloader`:
+  ```
+  @@ -233,6 +233,7 @@ def run_client(ccp):
+
+           unloader = DbUnloader(db, unload_dir, include_vos, exclude_vos,
+                                 local_jobs, withhold_dns)
+  +        unloader.records_per_message = 500
+           try:
+               if interval == 'latest':
+                   msgs, recs = unloader.unload_latest(table_name, send_ur)
+  ```

scripts/ssm-build-deb.sh

@@ -16,7 +16,7 @@
 
 set -eu
 
-TAG=3.2.0-1
+TAG=3.2.1-1
 
 SOURCE_DIR=~/debbuild/source
 BUILD_DIR=~/debbuild/build
@@ -58,6 +58,7 @@ fpm -s python -t deb \
 --depends python-ldap \
 --depends libssl-dev \
 --depends libsasl2-dev \
+--depends openssl \
 --deb-changelog $SOURCE_DIR/ssm-$TAG/CHANGELOG \
 --python-install-bin /usr/bin \
 --python-install-lib $PYTHON_INSTALL_LIB \

scripts/ssm-build-rpm.sh

@@ -10,7 +10,7 @@
 rpmdev-setuptree
 
 RPMDIR=/home/rpmb/rpmbuild
-VERSION=3.2.0-1
+VERSION=3.2.1-1
 SSMDIR=apel-ssm-$VERSION
 
 # Remove old sources and RPMS

ssm/__init__.py

@@ -19,7 +19,7 @@
 import logging
 import sys
 
-__version__ = (3, 2, 0)
+__version__ = (3, 2, 1)
 
 LOG_BREAK = '========================================'
 

ssm/crypto.py

@@ -63,24 +63,23 @@ def check_cert_key(certpath, keypath):
     if cert == key:
         return False
 
-    p1 = Popen(['openssl', 'x509', '-noout', '-modulus'],
+    p1 = Popen(['openssl', 'x509', '-pubkey', '-noout'],
                stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True)
-    modulus1, error = p1.communicate(cert)
+    pubkey1, error = p1.communicate(cert)
 
     if error != '':
         log.error(error)
         return False
 
-    p2 = Popen(['openssl', 'rsa', '-noout', '-modulus'],
+    p2 = Popen(['openssl', 'pkey', '-pubout'],
                stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True)
-    modulus2, error = p2.communicate(key)
+    pubkey2, error = p2.communicate(key)
 
     if error != '':
         log.error(error)
         return False
 
-    return modulus1.strip() == modulus2.strip()
-
+    return pubkey1.strip() == pubkey2.strip()
 
 def sign(text, certpath, keypath):
     """Sign the message using the certificate and key in the files specified.

test/test_crypto.py

@@ -4,6 +4,7 @@
 import logging
 import os
 from subprocess import call, Popen, PIPE
+import tempfile
 import quopri
 
 from ssm.crypto import check_cert_key, \
@@ -59,24 +60,35 @@ def tearDown(self):
         os.remove(self.ca_certpath)
 
     def test_check_cert_key(self):
-        '''
-        This will print an error log message for the tests that are
-        supposed to fail; you can ignore it.
-        '''
-
-        # One version of the method would have passed this, because of the
-        # way it checked for validity.
-        try:
-            if check_cert_key('hello', 'hello'):
-                self.fail('Accepted non-existent cert and key.')
-        except CryptoException:
-            pass
-
-        if check_cert_key(TEST_CERT_FILE, TEST_CERT_FILE):
-            self.fail('Accepted certificate as key.')
-
-        if not check_cert_key(TEST_CERT_FILE, TEST_KEY_FILE):
-            self.fail('Cert and key match but function failed.')
+        """Check that valid cert and key works."""
+        self.assertTrue(check_cert_key(TEST_CERT_FILE, TEST_KEY_FILE),
+                        'Cert and key match but function failed.')
+
+    def test_check_cert_key_invalid_paths(self):
+        """Check invalid file paths don't return True."""
+        self.assertFalse(check_cert_key('hello', 'hello'),
+                         'Accepted invalid file paths.')
+        self.assertFalse(check_cert_key(TEST_CERT_FILE, 'k'),
+                         'Accepted invalid key path.')
+        self.assertFalse(check_cert_key('c', TEST_KEY_FILE),
+                         'Accepted invalid cert path.')
+
+    def test_check_cert_key_arg_order(self):
+        """Check incorrect order of cert and key path args doesn't succeed."""
+        self.assertFalse(check_cert_key(TEST_CERT_FILE, TEST_CERT_FILE),
+                         'Accepted certificate as key.')
+        self.assertFalse(check_cert_key(TEST_KEY_FILE, TEST_KEY_FILE),
+                         'Accepted key as cert.')
+        self.assertFalse(check_cert_key(TEST_KEY_FILE, TEST_CERT_FILE),
+                         'Accepted key and cert wrong way round.')
+
+    def test_check_cert_key_invalid_files(self):
+        """Check behaviour with an invalid cert or key file."""
+        with tempfile.NamedTemporaryFile() as tmp:
+            self.assertFalse(check_cert_key(tmp.name, TEST_KEY_FILE),
+                             'Accepted invalid cert file.')
+            self.assertFalse(check_cert_key(TEST_CERT_FILE, tmp.name),
+                             'Accepted invalid key file.')
 
     def test_sign(self):
         '''