Merge branch 'master' into ao-send-parent-head-data

* master: (65 commits)
  collator protocol changes for elastic scaling (validator side) (#3302)
  Contracts use polkavm workspace deps (#3715)
  Add elastic scaling support in ParaInherent BenchBuilder  (#3690)
  Removes `as [disambiguation_path]` from `derive_impl` usage (#3652)
  fix(paseo-spec): New Paseo Bootnodes (#3674)
  Improve Penpal runtime + emulated tests (#3543)
  Staking ledger bonding fixes (#3639)
  DescribeAllTerminal for HashedDescription (#3349)
  Increase timeout for assertions (#3680)
  Add subsystems regression tests to CI (#3527)
  Always print connectivity report (#3677)
  Revert "FRAME: Create `TransactionExtension` as a replacement for `SignedExtension` (#2280)" (#3665)
  authority-discovery: Add log for debugging DHT authority records (#3668)
  Construct Runtime v2  (#1378)
  Support for `keyring` in runtimes (#2044)
  Add api-name in `cannot query the runtime API version` warning (#3653)
  Add a PolkaVM-based executor (#3458)
  Adds default config for assets pallet (#3637)
  Bump handlebars from 4.3.7 to 5.1.0 (#3248)
  [Collator Selection] Fix weight refund for `set_candidacy_bond` (#3643)
  ...

Author: ordian

.github/scripts/check-workspace.py

@@ -18,15 +18,15 @@ def parse_args():
 
 	parser.add_argument('workspace_dir', help='The directory to check', metavar='workspace_dir', type=str, nargs=1)
 	parser.add_argument('--exclude', help='Exclude crate paths from the check', metavar='exclude', type=str, nargs='*', default=[])
-	
+
 	args = parser.parse_args()
 	return (args.workspace_dir[0], args.exclude)
 
 def main(root, exclude):
 	workspace_crates = get_members(root, exclude)
 	all_crates = get_crates(root, exclude)
 	print(f'📦 Found {len(all_crates)} crates in total')
-	
+
 	check_duplicates(workspace_crates)
 	check_missing(workspace_crates, all_crates)
 	check_links(all_crates)
@@ -48,14 +48,14 @@ def get_members(workspace_dir, exclude):
 
 	if not 'members' in root_manifest['workspace']:
 		return []
-	
+
 	members = []
 	for member in root_manifest['workspace']['members']:
 		if member in exclude:
 			print(f'❌ Excluded member should not appear in the workspace {member}')
 			sys.exit(1)
 		members.append(member)
-	
+
 	return members
 
 # List all members of the workspace.
@@ -74,20 +74,20 @@ def get_crates(workspace_dir, exclude_crates) -> dict:
 			with open(path, "r") as f:
 				content = f.read()
 				manifest = toml.loads(content)
-			
+
 			if 'workspace' in manifest:
 				if root != workspace_dir:
 					print("⏩ Excluded recursive workspace at %s" % path)
 				continue
-			
+
 			# Cut off the root path and the trailing /Cargo.toml.
 			path = path[len(workspace_dir)+1:-11]
 			name = manifest['package']['name']
 			if path in exclude_crates:
 				print("⏩ Excluded crate %s at %s" % (name, path))
 				continue
 			crates[name] = (path, manifest)
-	
+
 	return crates
 
 # Check that there are no duplicate entries in the workspace.
@@ -138,23 +138,23 @@ def check_deps(deps):
 					if not 'path' in deps[dep]:
 						broken.append((name, dep_name, "crate must be linked via `path`"))
 						return
-		
+
 		def check_crate(deps):
 			to_checks = ['dependencies', 'dev-dependencies', 'build-dependencies']
 
 			for to_check in to_checks:
 				if to_check in deps:
 					check_deps(deps[to_check])
-		
+
 		# There could possibly target dependant deps:
 		if 'target' in manifest:
 			# Target dependant deps can only have one level of nesting:
 			for _, target in manifest['target'].items():
 				check_crate(target)
-		
+
 		check_crate(manifest)
 
-		
+
 
 	links.sort()
 	broken.sort()

.github/scripts/common/lib.sh

@@ -237,6 +237,61 @@ fetch_release_artifacts() {
   popd > /dev/null
 }
 
+# Fetch the release artifacts like binary and sigantures from S3. Assumes the ENV are set:
+# - RELEASE_ID
+# - GITHUB_TOKEN
+# - REPO in the form paritytech/polkadot
+fetch_release_artifacts_from_s3() {
+  echo "Version    : $VERSION"
+  echo "Repo       : $REPO"
+  echo "Binary     : $BINARY"
+  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
+  echo "OUTPUT_DIR : $OUTPUT_DIR"
+
+  URL_BASE=$(get_s3_url_base $BINARY)
+  echo "URL_BASE=$URL_BASE"
+
+  URL_BINARY=$URL_BASE/$VERSION/$BINARY
+  URL_SHA=$URL_BASE/$VERSION/$BINARY.sha256
+  URL_ASC=$URL_BASE/$VERSION/$BINARY.asc
+
+  # Fetch artifacts
+  mkdir -p "$OUTPUT_DIR"
+  pushd "$OUTPUT_DIR" > /dev/null
+
+  echo "Fetching artifacts..."
+  for URL in $URL_BINARY $URL_SHA $URL_ASC; do
+    echo "Fetching %s" "$URL"
+    curl --progress-bar -LO "$URL" || echo "Missing $URL"
+  done
+
+  pwd
+  ls -al --color
+  popd > /dev/null
+
+}
+
+# Pass the name of the binary as input, it will
+# return the s3 base url
+function get_s3_url_base() {
+    name=$1
+    case $name in
+    polkadot | polkadot-execute-worker | polkadot-prepare-worker | staking-miner)
+        printf "https://releases.parity.io/polkadot"
+        ;;
+
+    polkadot-parachain)
+        printf "https://releases.parity.io/cumulus"
+        ;;
+
+    *)
+        printf "UNSUPPORTED BINARY $name"
+        exit 1
+        ;;
+    esac
+}
+
+
 # Check the checksum for a given binary
 function check_sha256() {
     echo "Checking SHA256 for $1"
@@ -248,13 +303,11 @@ function check_sha256() {
 function import_gpg_keys() {
   GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"}
   SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798"
-  WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2"
   EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3"
-  MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF"
   MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF"
 
   echo "Importing GPG keys from $GPG_KEYSERVER in parallel"
-  for key in $SEC $WILL $EGOR $MARA $MORGAN; do
+  for key in $SEC $EGOR $MORGAN; do
     (
       echo "Importing GPG key $key"
       gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key
@@ -344,3 +397,40 @@ function find_runtimes() {
     done
     echo $JSON
 }
+
+# Filter the version matches the particular pattern and return it.
+# input: version (v1.8.0 or v1.8.0-rc1)
+# output: none
+filter_version_from_input() {
+  version=$1
+  regex="(^v[0-9]+\.[0-9]+\.[0-9]+)$|(^v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+)$"
+
+  if [[ $version =~ $regex ]]; then
+      if [ -n "${BASH_REMATCH[1]}" ]; then
+          echo "${BASH_REMATCH[1]}"
+      elif [ -n "${BASH_REMATCH[2]}" ]; then
+          echo "${BASH_REMATCH[2]}"
+      fi
+  else
+      echo "Invalid version: $version"
+      exit 1
+  fi
+
+}
+
+# Check if the release_id is valid number
+# input: release_id
+# output: release_id or exit 1
+check_release_id() {
+  input=$1
+
+  release_id=$(echo "$input" | sed 's/[^0-9]//g')
+
+  if [[ $release_id =~ ^[0-9]+$ ]]; then
+      echo "$release_id"
+  else
+      echo "Invalid release_id from input: $input"
+      exit 1
+  fi
+
+}

.github/workflows/check-licenses.yml

@@ -42,5 +42,4 @@ jobs:
           shopt -s globstar
           npx @paritytech/license-scanner scan \
             --ensure-licenses ${{ env.LICENSES }} \
-            --exclude ./substrate/bin/node-template \
             -- ./substrate/**/*.rs

.github/workflows/release-50_publish-docker.yml

@@ -36,7 +36,7 @@ on:
             -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \
             jq '.[] | { name: .name, id: .id }'
         required: true
-        type: string
+        type: number
 
       registry:
         description: Container registry
@@ -61,7 +61,6 @@ permissions:
   contents: write
 
 env:
-  RELEASE_ID: ${{ inputs.release_id }}
   ENGINE: docker
   REGISTRY: ${{ inputs.registry }}
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -71,6 +70,7 @@ env:
   # EVENT_ACTION: ${{ github.event.action }}
   EVENT_NAME: ${{ github.event_name }}
   IMAGE_TYPE: ${{ inputs.image_type }}
+  VERSION: ${{ inputs.version }}
 
 jobs:
   fetch-artifacts: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build
@@ -95,13 +95,16 @@ jobs:
         #   chmod a+x $BINARY
         #   ls -al
 
-      - name: Fetch rc artifacts or release artifacts based on release id
+      - name: Fetch rc artifacts or release artifacts from s3 based on version
         #this step runs only if the workflow is triggered manually
         if: ${{ env.EVENT_NAME  == 'workflow_dispatch' }}
         run: |
           . ./.github/scripts/common/lib.sh
 
-          fetch_release_artifacts
+          VERSION=$(filter_version_from_input "${{ inputs.version }}")
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+
+          fetch_release_artifacts_from_s3
 
       - name: Cache the artifacts
         uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
@@ -147,7 +150,10 @@ jobs:
         if: ${{ env.IMAGE_TYPE == 'rc' }}
         id: fetch_rc_refs
         run: |
-          release=release-${{ inputs.release_id }} && \
+          . ./.github/scripts/common/lib.sh
+
+          RELEASE_ID=$(check_release_id "${{ inputs.release_id }}")
+          release=release-$RELEASE_ID && \
           echo "release=${release}" >> $GITHUB_OUTPUT
 
           commit=$(git rev-parse --short HEAD) && \

.github/workflows/release-99_notif-published.yml

@@ -16,12 +16,6 @@ jobs:
           - name: "RelEng: Polkadot Release Coordination"
             room: '!cqAmzdIcbOFwrdrubV:parity.io'
             pre-release: true
-          - name: 'General: Rust, Polkadot, Substrate'
-            room: '!aJymqQYtCjjqImFLSb:parity.io'
-            pre-release: false
-          - name: 'Team: DevOps'
-            room: '!lUslSijLMgNcEKcAiE:parity.io'
-            pre-release: true
 
          # External
           - name: 'Ledger <> Polkadot Coordination'
@@ -48,7 +42,9 @@ jobs:
           access_token: ${{ secrets.RELEASENOTES_MATRIX_V2_ACCESS_TOKEN }}
           server: m.parity.io
           message: |
-            A (pre)release has been ${{github.event.action}} in **${{github.event.repository.full_name}}:**<br/>
+            @room
+
+            A new node release has been ${{github.event.action}} in **${{github.event.repository.full_name}}:**<br/>
             Release version: [${{github.event.release.tag_name}}](${{github.event.release.html_url}})
 
             -----

.gitlab/pipeline/build.yml

@@ -337,7 +337,7 @@ build-runtimes-polkavm:
     - .common-refs
     - .run-immediately
   script:
-    - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p minimal-runtime
+    - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p minimal-template-runtime
     - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p westend-runtime
     - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p rococo-runtime
     - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p polkadot-test-runtime

.gitlab/pipeline/check.yml

@@ -259,3 +259,19 @@ find-fail-ci-phrase:
       echo "No $ASSERT_REGEX was found, exiting with 0";
       exit 0;
       fi
+
+check-core-crypto-features:
+  stage: check
+  extends:
+    - .docker-env
+    - .common-refs
+  script:
+    - pushd substrate/primitives/core
+    - ./check-features-variants.sh
+    - popd
+    - pushd substrate/primitives/application-crypto
+    - ./check-features-variants.sh
+    - popd
+    - pushd substrate/primitives/keyring
+    - ./check-features-variants.sh
+    - popd

.gitlab/pipeline/test.yml

@@ -494,3 +494,15 @@ test-syscalls:
       printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
       fi
   allow_failure: false # this rarely triggers in practice
+
+subsystem-regression-tests:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+  script:
+    - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks
+  tags:
+    - benchmark
+  allow_failure: true