Bump python requirement to 3.9 and remove 3.8 tests and github workflow

Add support for MIL-1750A float parsing
Add test cases pulled from MIL-1750A standard document

Author: medley56

.github/workflows/tests.yml

@@ -4,7 +4,7 @@ jobs:
   python-version-matrix:
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     uses: ./.github/workflows/test-python-version.yml
     with:
       python-version: ${{ matrix.python-version }}

docker-compose.yml

@@ -9,13 +9,6 @@ services:
     build:
       target: style
 
-  3.8-tests:
-    image: space-packets-3.8-test:latest
-    build:
-      target: test
-      args:
-        - BASE_IMAGE_PYTHON_VERSION=3.8
-
   3.9-tests:
     image: space-packets-3.9-test:latest
     build:

docs/source/changelog.md

@@ -16,6 +16,7 @@ Release notes for the `space_packet_parser` library
   ``f"{int.from_bytes(data, byteorder='big'):0{len(data)*8}b}"``
 - Fix EnumeratedParameterType to handle duplicate labels
 - Add error reporting for unsupported and invalid parameter types
+- Add support for MIL-1750A floats (32-bit only)
 
 ### v4.2.0 (released)
 - Parse short and long descriptions of parameters

pyproject.toml

@@ -35,7 +35,7 @@ keywords = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8"
+python = ">=3.9"
 lxml = ">=4.8.0"
 
 [tool.poetry.group.dev.dependencies]
@@ -50,6 +50,7 @@ myst-parser = "*"
 sphinx-autoapi = "*"
 sphinx-rtd-theme = "*"
 coverage = "*"
+numpy = "*"
 
 [tool.poetry.group.examples]
 optional = true

space_packet_parser/xtcedef.py

@@ -1,6 +1,6 @@
 """Module for parsing XTCE xml files to specify packet format"""
 # Standard
-from abc import ABCMeta
+from abc import ABCMeta, abstractmethod
 from collections import namedtuple
 from dataclasses import dataclass, field
 import inspect
@@ -111,6 +111,7 @@ class MatchCriteria(AttrComparable, metaclass=ABCMeta):
     }
 
     @classmethod
+    @abstractmethod
     def from_match_criteria_xml_element(cls, element: ElementTree.Element, ns: dict):
         """Abstract classmethod to create a match criteria object from an XML element.
 
@@ -127,6 +128,7 @@ def from_match_criteria_xml_element(cls, element: ElementTree.Element, ns: dict)
         """
         raise NotImplementedError()
 
+    @abstractmethod
     def evaluate(self, parsed_data: dict, current_parsed_value: Optional[Union[int, float]] = None) -> bool:
         """Evaluate match criteria down to a boolean.
 
@@ -609,6 +611,7 @@ class Calibrator(AttrComparable, metaclass=ABCMeta):
     """Abstract base class for XTCE calibrators"""
 
     @classmethod
+    @abstractmethod
     def from_calibrator_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'Calibrator':
         """Abstract classmethod to create a default_calibrator object from an XML element.
 
@@ -625,6 +628,7 @@ def from_calibrator_xml_element(cls, element: ElementTree.Element, ns: dict) ->
         """
         return NotImplemented
 
+    @abstractmethod
     def calibrate(self, uncalibrated_value: Union[int, float]) -> Union[int, float]:
         """Takes an integer-encoded or float-encoded value and returns a calibrated version.
 
@@ -1035,6 +1039,7 @@ class DataEncoding(AttrComparable, metaclass=ABCMeta):
     """Abstract base class for XTCE data encodings"""
 
     @classmethod
+    @abstractmethod
     def from_data_encoding_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'DataEncoding':
         """Abstract classmethod to create a data encoding object from an XML element.
 
@@ -1456,6 +1461,7 @@ def __init__(self, size_in_bits: int,
     def _calculate_size(self, packet: Packet) -> int:
         return self.size_in_bits
 
+    @abstractmethod
     def _get_raw_value(self, packet: Packet) -> Union[int, float]:
         """Read the raw value from the packet data
 
@@ -1472,6 +1478,16 @@ def _get_raw_value(self, packet: Packet) -> Union[int, float]:
         """
         raise NotImplementedError()
 
+    @staticmethod
+    def twos_complement(val: int, bit_width: int) -> int:
+        """Take the twos complement of val
+
+        Used when parsing ints and some floats
+        """
+        if (val & (1 << (bit_width - 1))) != 0:  # if sign bit is set e.g., 8bit: 128-255
+            return val - (1 << bit_width)  # compute negative value
+        return val
+
     def parse_value(self,
                     packet: Packet,
                     **kwargs) -> Tuple[Union[int, float], Union[int, float]]:
@@ -1522,10 +1538,8 @@ def _get_raw_value(self, packet: Packet) -> int:
             )
         if self.encoding == 'unsigned':
             return val
-        # It is a signed integer and we need to take into account the first bit
-        if (val & (1 << (self.size_in_bits - 1))) != 0:  # if sign bit is set e.g., 8bit: 128-255
-            return val - (1 << self.size_in_bits)  # compute negative value
-        return val  # return positive value as is
+        # It is a signed integer, and we need to take into account the first bit
+        return self.twos_complement(val, self.size_in_bits)
 
     @classmethod
     def from_data_encoding_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'IntegerDataEncoding':
@@ -1561,8 +1575,6 @@ def __init__(self, size_in_bits: int, encoding: str = 'IEEE-754',
                  context_calibrators: Optional[List[ContextCalibrator]] = None):
         """Constructor
 
-        # TODO: Implement MIL-1650A encoding option
-
         Parameters
         ----------
         size_in_bits : int
@@ -1581,33 +1593,69 @@ def __init__(self, size_in_bits: int, encoding: str = 'IEEE-754',
         if encoding not in self._supported_encodings:
             raise ValueError(f"Invalid encoding type {encoding} for float data. "
                              f"Must be one of {self._supported_encodings}.")
-        if encoding == 'MIL-1750A':
-            raise NotImplementedError("MIL-1750A encoded floats are not supported by this library yet.")
+        if encoding == 'MIL-1750A' and size_in_bits != 32:
+            raise ValueError("MIL-1750A encoded floats must be 32 bits, per the MIL-1750A spec. See "
+                             "https://www.xgc-tek.com/manuals/mil-std-1750a/c191.html#AEN324")
         if encoding == 'IEEE-754' and size_in_bits not in (16, 32, 64):
             raise ValueError(f"Invalid size_in_bits value for IEEE-754 FloatDataEncoding, {size_in_bits}. "
                              "Must be 16, 32, or 64.")
         super().__init__(size_in_bits, encoding=encoding, byte_order=byte_order,
                          default_calibrator=default_calibrator, context_calibrators=context_calibrators)
 
-        if self.byte_order == "leastSignificantByteFirst":
-            self._struct_format = "<"
+        if self.encoding == "MIL-1750A":
+            def _mil_parse_func(mil_bytes: bytes):
+                """Parsing function for MIL-1750A floats"""
+                # MIL 1750A floats are always 32 bit
+                # See: https://www.xgc-tek.com/manuals/mil-std-1750a/c191.html#AEN324
+                #
+                #  MSB                                         LSB MSB          LSB
+                # ------------------------------------------------------------------
+                # | S|                   Mantissa                 |    Exponent    |
+                # ------------------------------------------------------------------
+                #   0  1                                        23 24            31
+                bytes_as_int = int.from_bytes(mil_bytes, byteorder='big')
+                exponent = bytes_as_int & 0xFF  # last 8 bits
+                mantissa = (bytes_as_int >> 8) & 0xFFFFFFFF  # bits 0 through 23 (24 bits)
+                # We include the sign bit with the mantissa because we can just take the twos complement
+                # of it directly and use it in the final calculation for the value
+
+                # Both mantissa and exponent are stored as twos complement with no bias
+                exponent = self.twos_complement(exponent, 8)
+                mantissa = self.twos_complement(mantissa, 24)
+
+                # Calculate float value using native Python floats, which are more precise
+                return mantissa * (2.0 ** (exponent - (24 - 1)))
+
+            # Set up the parsing function just once, so we can use it repeatedly with _get_raw_value
+            self.parse_func = _mil_parse_func
         else:
-            # Big-endian is the default
-            self._struct_format = ">"
+            if self.byte_order == "leastSignificantByteFirst":
+                self._struct_format = "<"
+            else:
+                # Big-endian is the default
+                self._struct_format = ">"
+
+            if self.size_in_bits == 16:
+                self._struct_format += "e"
+            elif self.size_in_bits == 32:
+                self._struct_format += "f"
+            elif self.size_in_bits == 64:
+                self._struct_format += "d"
+
+            def ieee_parse_func(data: bytes):
+                """Parsing function for IEEE floats"""
+                # The packet data we got back is always extracted in big-endian order
+                # but the struct format code contains the endianness of the float data
+                return struct.unpack(self._struct_format, data)[0]
 
-        if self.size_in_bits == 16:
-            self._struct_format += "e"
-        elif self.size_in_bits == 32:
-            self._struct_format += "f"
-        elif self.size_in_bits == 64:
-            self._struct_format += "d"
+            # Set up the parsing function just once, so we can use it repeatedly with _get_raw_value
+            self.parse_func: callable = ieee_parse_func
 
     def _get_raw_value(self, packet):
         """Read the data in as bytes and return a float representation."""
         data = packet.read_as_bytes(self.size_in_bits)
-        # The packet data we got back is always extracted in big-endian order
-        # but the struct format code contains the endianness of the float data
-        return struct.unpack(self._struct_format, data)[0]
+        # The parsing function is fully set during initialization to save time during parsing
+        return self.parse_func(data)
 
     @classmethod
     def from_data_encoding_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'FloatDataEncoding':

tests/unit/test_xtcedef.py

@@ -4,6 +4,7 @@
 # Installed
 import pytest
 import lxml.etree as ElementTree
+import numpy as np
 # Local
 from space_packet_parser import xtcedef, parser
 
@@ -1506,7 +1507,7 @@ def test_float_parameter_type(xml_string: str, expectation):
         # Test big endian 64-bit float
         (xtcedef.FloatParameterType('TEST_FLOAT', xtcedef.FloatDataEncoding(64)),
          xtcedef.Packet(b'\x3F\xF9\xE3\x77\x9B\x97\xF4\xA8'),  # 64-bit IEEE 754 value of Phi
-         1.61803),
+         1.6180339),
         # Test float parameter type encoded as big endian 16-bit integer with contextual polynomial calibrator
         (xtcedef.FloatParameterType(
             'TEST_FLOAT',
@@ -1523,16 +1524,73 @@ def test_float_parameter_type(xml_string: str, expectation):
          xtcedef.Packet(0b1111111111010110.to_bytes(length=2, byteorder='big'),
                             parsed_data={'PKT_APID': parser.ParsedDataItem('PKT_APID', 1101)}),
          -82.600000),
+        # Test MIL 1750A encoded floats.
+        # Test values taken from: https://www.xgc-tek.com/manuals/mil-std-1750a/c191.html#AEN324
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x7f\xff\xff\x7f'),
+         0.9999998 * (2 ** 127)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x40\x00\x00\x7f'),
+         0.5 * (2 ** 127)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x50\x00\x00\x04'),
+         0.625 * (2 ** 4)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x40\x00\x00\x01'),
+         0.5 * (2 ** 1)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x40\x00\x00\x00'),
+         0.5 * (2 ** 0)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x40\x00\x00\xff'),
+         0.5 * (2 ** -1)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x40\x00\x00\x80'),
+         0.5 * (2 ** -128)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x00\x00\x00\x00'),
+         0.0 * (2 ** 0)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x80\x00\x00\x00'),
+         -1.0 * (2 ** 0)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\xBF\xFF\xFF\x80'),
+         -0.5000001 * (2 ** -128)),
+        (xtcedef.FloatParameterType(
+            'MIL_1750A_FLOAT',
+            xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")),
+         xtcedef.Packet(b'\x9F\xFF\xFF\x04'),
+         -0.7500001 * (2 ** 4)),
     ]
 )
 def test_float_parameter_parsing(parameter_type, packet, expected):
     """Test parsing float parameters"""
     raw, derived = parameter_type.parse_value(packet)
+    # NOTE: These results are compared with a relative tolerance due to the imprecise storage of floats
     if derived:
-        # NOTE: These results are rounded due to the imprecise storage of floats
-        assert round(derived, 5) == expected
+        assert np.isclose(derived, expected, rtol=0.000001)
     else:
-        assert round(raw, 5) == expected
+        assert np.isclose(raw, expected, rtol=0.000001)
 
 
 @pytest.mark.parametrize(