Revamp release CI pipeline with TestPyPI support and strict checking of tags

Author: medley56

.github/workflows/release.yml

@@ -8,28 +8,28 @@ concurrency:
 on:
   push:
     tags:
-      - '[0-9]*.[0-9]*.[0-9]*'  # Push events to every tag that looks like a semver
-      - '[0-9]*.[0-9]*.[0-9]*rc[0-9]*'  # Push events to every tag that looks like a release candidate
+      - '[0-9]+.[0-9]+.[0-9]+*'  # Push events for official release tags
+      - 'test-release/[0-9]+.[0-9]+.[0-9]+*'  # Push events for test release tags
 
 jobs:
-  build:
+  build-dist-artifacts:
     # This job uses vanilla Python tools rather than Poetry, so we don't have to use third party GitHub actions
     # e.g. pip, build, twine
     # If we even want to, we could switch to using something like actions/setup-poetry (but do a search for current
     # best implementations)
-    name: Build distribution 📦
+    name: Build distribution artifacts 📦
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout repo
+      - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Install Python 🐍
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
-      - name: Install dependencies
+      - name: Install project dependencies
         run: python -m pip install build twine
 
       - name: Build wheel and source distribution
@@ -40,21 +40,24 @@ jobs:
         run: twine check dist/*
 
       # Save ("upload") the distribution artifacts for use by downstream Actions jobs
-      - name: Upload sdist artifacts 📦
+      - name: Upload distribution artifacts 📦
         uses: actions/upload-artifact@v4  # This allows us to persist the dist directory after the job has completed
         with:
           name: python-package-distributions
           path: dist/
           if-no-files-found: error
 
-  publish-to-pypi:
-    name: Upload release to PyPI
-    if: startsWith(github.ref, 'refs/tags/')  # Belt and suspenders, only ever publish based on a tag
-    needs: build
+  # Job that pushes dist artifacts to public PyPI for official release tags
+  official-pypi-publish:
+    name: Upload official release to PyPI
+    # Prevent running on any PEP 440 suffixed tags or on test-release tags
+    if: startsWith(github.ref, 'refs/tags/test-release') == false
+    needs:
+      - build-dist-artifacts
     runs-on: ubuntu-latest
     environment:
-      name: pypi-publish
-      url: https://pypi.org/p/space_packet_parser
+      name: official-pypi-publish-environment
+      url: https://pypi.org/p/space_packet_parser  # Public PyPI
     permissions:
       id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
 
@@ -66,24 +69,52 @@ jobs:
           name: python-package-distributions
           path: dist/
 
-      - name: Publish distribution 📦 to PyPI
+      - name: Publish distribution artifacts 📦 to PyPI
         uses: pypa/[email protected]
 
+  # Job that pushes dist artifacts to TestPyPI for test release tags
+  # This will fail if the version (according to package metadata) has already been pushed
+  test-pypi-publish:
+    name: Upload testing release to TestPyPI
+    # Only run on test-release tags
+    if: startsWith(github.ref, 'refs/tags/test-release')
+    needs:
+      - build-dist-artifacts
+    runs-on: ubuntu-latest
+    environment:
+      name: test-pypi-publish-environment
+      url: https://test.pypi.org/p/space_packet_parser  # TestPyPI
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+
+    steps:
+      # This downloads the build artifacts from the build job
+      - name: Download all the dists 📦
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish distribution artifacts 📦 to TestPyPI
+        uses: pypa/[email protected]
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  # Job that publishes an official Release to GitHub after pushing to PyPI
+  # This only runs if we have pushed to public PyPI (not TestPyPI)
   create-github-release:
-    name: >-
-      Sign the Python 🐍 distribution 📦 with Sigstore
-      and upload them to GitHub Release
+    name: Upload dist artifacts to GitHub Release
     needs:
-      - publish-to-pypi
+      - official-pypi-publish
     runs-on: ubuntu-latest
     environment:
-      name: pypi-publish
+      name: create-github-release-environment
     permissions:
-      contents: write  # IMPORTANT: mandatory for making GitHub Releases
       id-token: write  # IMPORTANT: mandatory for sigstore
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
 
     steps:
-      - name: Download all the dists 📦
+      - name: Download the artifacts 📦
         uses: actions/download-artifact@v4
         with:
           name: python-package-distributions
@@ -96,14 +127,15 @@ jobs:
             ./dist/*.tar.gz
             ./dist/*.whl
 
-      - name: Determine if it's a pre-release
-        # Dynamically sets the --prerelease option passed to the release create CLI based on matching the *rc*
-        # substring in the git tag. If rc not present, does not pass --prerelease to the CLI.
+      - name: Determine if the release is a prerelease
+        # Checks the regex form of the tag.
+        # Marks final releases only for tags matching the regex (no version suffixes)
+        # All other releases are marked as prereleases
         run: |
-          if [[ "${{ github.ref_name }}" == *rc* ]]; then
-          echo "PRE_RELEASE_OPTION=--prerelease" >> $GITHUB_ENV
+          if [[ "${{ github.ref_name }}" =~ '^[0-9]*\.[0-9]*\.[0-9]*$' ]]; then
+          echo "PRE_RELEASE_OPTION=''" >> $GITHUB_ENV  # Not a prerelease
           else
-          echo "PRE_RELEASE_OPTION=''" >> $GITHUB_ENV
+          echo "PRE_RELEASE_OPTION='--prerelease'" >> $GITHUB_ENV  # Is a prerelease
           fi
 
       - name: Get latest non-prerelease release
@@ -125,9 +157,11 @@ jobs:
         # Uses the GitHub CLI to generate the Release and auto-generate the release notes. Also generates
         # the Release title based on the annotation on the git tag.
         run: >-
+          RELEASE_NAME=$(basename "${{ github.ref_name }}")
           gh release create
           '${{ github.ref_name }}'
           --repo '${{ github.repository }}'
+          --title "$RELEASE_NAME"
           ${{ env.PRE_RELEASE_OPTION }}
           --generate-notes
           --notes-start-tag '${{ env.LATEST_RELEASE_TAG }}'

CITATION.cff

@@ -1,7 +1,7 @@
 cff-version: 1.2.0
 title: 'space_packet_parser'
 type: software
-version: '5.0.0rc6'
+version: '5.0.0rc9'
 description: A CCSDS telemetry packet decoding library based on the XTCE packet format description standard.
 license: BSD-3-Clause
 abstract: The Space Packet Parser Python library is a generalized, configurable packet decoding library for CCSDS telemetry

docs/source/developers.md

@@ -35,6 +35,14 @@ Feel free to fork this repo and submit a PR!
 ## Release Process
 Releases are automatically created using a GitHub Actions workflow that responds to pushes of annotated git tags.
 
+### Versioning
+Version numbers must be PEP440 strings: https://peps.python.org/pep-0440/
+
+That is,
+```
+[N!]N(.N)*[{a|b|rc}N][.postN][.devN]
+```
+
 ### Preparing for Release
 1. Create a release candidate branch named according to the version to be released. This branch is used to polish
    the release but is fundamentally not different from any other feature branch in trunk-based development. 
@@ -59,26 +67,43 @@ Releases are automatically created using a GitHub Actions workflow that responds
 
 ### Automatic Release Process
 GitHub Actions has an automatic release process that responds to pushes of annotated git tags. When a tag matching 
-a semantic version (`[0-9]*.[0-9]*.[0-9]*` or `[0-9]*.[0-9]*.[0-9]*rc[0-9]*`) is pushed, a workflow runs that builds
-the package, pushes the artifacts to PyPI, and creates a GitHub Release from the distributed artifacts. Release notes 
-are automatically generated from commit history and the Release name is taken from the annotation on the tag.
+a semantic version (`[0-9]+.[0-9]+.[0-9]+*` or `test-release/[0-9]+.[0-9]+.[0-9]+*`) is pushed, 
+a workflow runs that builds the package, pushes the artifacts to PyPI or TestPyPI, 
+and creates a GitHub Release from the distributed artifacts. Release notes 
+are automatically generated from commit history and the Release name is taken from the tag.
+
+#### Official Releases
+Official releases are published to the public PyPI (even if they are release candidates like `1.2.3rc1`). This differs
+from test releases, which are only published to TestPyPI and are not published to GitHub at all. 
+If the semantic version has any suffixes (e.g. `rc1`), the release will be marked as 
+a prerelease in GitHub and PyPI.
 
-To trigger a release, push a tag reference to the commit you want to release, like so:
+To trigger an official release, push a tag referencing the commit you want to release. The commit _MUST_ be on 
+the `main` branch. Never publish an official release from a commit that hasn't been merged to `main`!
 
 ```bash
+git checkout main
+git pull
 git tag -a X.Y.Z -m "Version X.Y.Z"
 git push origin X.Y.Z
 ```
 
-To tag and publish a Release Candidate, your tag should look like the following:
+#### Test Releases
+Test releases are published to TestPyPI only. Official releases are also published to TestPyPI.
+
+To publish a test release, prefix the tag with `test-release`. This will prevent any publishing to the public PyPI 
+and will prevent the artifacts being published on GitHub.
 
 ```bash
-git tag -a X.Y.Zrc1 -m "Release Candidate X.Y.Zrc1"
-git push origin X.Y.Zrc1
+git checkout <ref-to-test-release-from>
+git pull
+git tag -a test-release/X.Y.Zrc1 -m "Test Release Candidate X.Y.Zrc1"
+git push origin test-release/X.Y.Zrc1
 ```
 
-Release candidate tags are always marked as Prereleases in GitHub and release notes are generated from the latest
-non-prerelease Release.
+#### Prereleases
+Unless the pushed tag matches the regex `^[0-9]*\.[0-9]*\.[0-9]*`, the release will be marked as a
+prerelease in GitHub. This allows "official" prereleases of suffixed tags.
 
-**For production releases, tags should always reference commits in the `main` branch. Release candidates are less 
-important and tags can reference any commit.**
+#### Release Notes Generation
+Release notes are generated based on commit messages since the latest non-prerelease Release.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "space_packet_parser"
-version = "5.0.0rc6"
+version = "5.0.0rc9"
 description = "A CCSDS telemetry packet decoding library based on the XTCE packet format description standard."
 license = "BSD-3-Clause"
 readme = "README.md"