GH-41102: [Packaging][Release] Create unique git tags for release candidates (e.g. apache-arrow-{MAJOR}.{MINOR}.{PATCH}-rc{RC_NUM})

.github/workflows/release.yml

@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Release
+
+on:
+  push:
+    tags:
+      # Trigger workflow when a tag whose name matches the pattern
+      # pattern "apache-arrow-{MAJOR}.{MINOR}.{PATCH}" is pushed.
+      - "apache-arrow-[0-9]+.[0-9]+.[0-9]+"
+
+permissions:
+  contents: write
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  publish:
+    name: Publish 
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Get Tag Name of Latest Release Candidate
+        run: |
+          rc_tag=$(gh release list --repo apache/arrow | \
+                    cut -f3 | \
+                    grep -F "${GITHUB_REF_NAME}-rc" | \
+                    head -n1)
+          echo "Latest Release Candidate Tag: ${rc_tag}"
+          echo "RELEASE_CANDIDATE_TAG_NAME=${rc_tag}" >> ${GITHUB_ENV}
+      - name: Store Version and Release Candidate Number
+        run: |
+          version_with_rc=${RELEASE_CANDIDATE_TAG_NAME#apache-arrow-}
+          version=${version_with_rc%-rc*}
+          rc_num=${version_with_rc#${version}-rc}
+          echo "VERSION_WITH_RC=${version_with_rc}" >> ${GITHUB_ENV}
+          echo "VERSION=${version}" >> ${GITHUB_ENV}
+          echo "RC_NUM=${rc_num}" >> ${GITHUB_ENV}
+      - name: Download Release Candidate Artifacts 
+        run: |
+          mkdir release_candidate_artifacts
+          gh release download ${RELEASE_CANDIDATE_TAG_NAME} --repo apache/arrow --dir release_candidate_artifacts
+      - name: Create Release Title
+        run: |
+          title="Apache Arrow ${VERSION}"
+          echo "RELEASE_TITLE=${title}" >> ${GITHUB_ENV}
+      # Set the release notes to "TODO" temporarily. After the release notes page
+      # (https://arrow.apache.org/release/{VERSION}.html) is published, use 
+      # gh release edit to update the release notes to refer to the newly 
+      # pushed web page. See dev/post/post-05-update-gh-release-notes.sh
+      - name: Create GitHub Release
+        run: |
+          gh release create ${GITHUB_REF_NAME} \
+            --repo apache/arrow \
+            --verify-tag \
+            --title "${RELEASE_TITLE}" \
+            --notes "TODO" \
+            release_candidate_artifacts/*

.github/workflows/release_candidate.yml

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Release
+
+on:
+  push:
+    tags:
+      # Trigger workflow when a tag whose name matches the pattern
+      # "apache-arrow-{MAJOR}.{MINOR}.{PATCH}-rc{RC_NUM}" is pushed.
+      - "apache-arrow-[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
+
+permissions:
+  contents: write
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  publish:
+    name: Publish 
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps: 
+      - name: Checkout Arrow
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Store Version and Release Candidate Number
+        run: |
+          version_with_rc=${GITHUB_REF_NAME#apache-arrow-}
+          version=${version_with_rc%-rc*}
+          rc_num=${version_with_rc#${version}-rc}
+          echo "VERSION_WITH_RC=${version_with_rc}" >> ${GITHUB_ENV}
+          echo "VERSION=${version}" >> ${GITHUB_ENV}
+          echo "RC_NUM=${rc_num}" >> ${GITHUB_ENV}
+      - name: Create Release Candidate Title
+        run: |
+          title="Apache Arrow ${VERSION} RC${RC_NUM}"
+          echo "RELEASE_CANDIDATE_TITLE=${title}" >> ${GITHUB_ENV}
+      - name: Create Release Candidate Notes
+        run: |
+          release_notes="Release Candidate: ${VERSION} RC${RC_NUM}"
+          echo "RELEASE_CANDIDATE_NOTES=${release_notes}" >> ${GITHUB_ENV}
+      - name: Create Release tarball
+        run: |
+          cd dev/release/ && ./utils-create-release-tarball.sh ${VERSION} ${RC_NUM}
+          echo "RELEASE_TARBALL=apache-arrow-${VERSION}.tar.gz" >> ${GITHUB_ENV}
+      - name: Create GitHub Release
+        run: |
+          gh release create ${GITHUB_REF_NAME} \
+            --verify-tag \
+            --prerelease \
+            --title "${RELEASE_CANDIDATE_TITLE}" \
+            --notes "Release Notes: ${RELEASE_CANDIDATE_NOTES}" \
+            dev/release/${RELEASE_TARBALL}

dev/release/01-prepare.sh

@@ -33,7 +33,7 @@ next_version=$2
 next_version_snapshot="${next_version}-SNAPSHOT"
 rc_number=$3
 
-release_tag="apache-arrow-${version}"
+release_candidate_tag="apache-arrow-${version}-rc${rc_number}"
 release_branch="release-${version}"
 release_candidate_branch="release-${version}-rc${rc_number}"
 
@@ -46,9 +46,9 @@ release_candidate_branch="release-${version}-rc${rc_number}"
 : ${PREPARE_TAG:=${PREPARE_DEFAULT}}
 
 if [ ${PREPARE_TAG} -gt 0 ]; then
-  if [ $(git tag -l "${release_tag}") ]; then
-    echo "Delete existing git tag $release_tag"
-    git tag -d "${release_tag}"
+  if [ $(git tag -l "${release_candidate_tag}") ]; then
+    echo "Delete existing git tag $release_candidate_tag"
+    git tag -d "${release_candidate_tag}"
   fi
 fi
 
@@ -88,7 +88,7 @@ if [ ${PREPARE_LINUX_PACKAGES} -gt 0 ]; then
 fi
 
 if [ ${PREPARE_VERSION_PRE_TAG} -gt 0 ]; then
-  echo "Prepare release ${version} on tag ${release_tag} then reset to version ${next_version_snapshot}"
+  echo "Prepare release ${version} on tag ${release_candidate_tag} then reset to version ${next_version_snapshot}"
 
   update_versions "${version}" "${next_version}" "release"
   git commit -m "MINOR: [Release] Update versions for ${version}"
@@ -97,5 +97,5 @@ fi
 ############################## Tag the Release ##############################
 
 if [ ${PREPARE_TAG} -gt 0 ]; then
-  git tag -a "${release_tag}" -m "[Release] Apache Arrow Release ${version}"
+  git tag -a "${release_candidate_tag}" -m "[Release] Apache Arrow Release ${version} RC${rc_number}"
 fi

dev/release/02-source-test.rb

@@ -22,8 +22,10 @@ class SourceTest < Test::Unit::TestCase
   def setup
     @current_commit = git_current_commit
     detect_versions
-    @tag_name = "apache-arrow-#{@release_version}"
+    @tag_name = "apache-arrow-#{@release_version}-rc0"
+    @archive_name = "apache-arrow-#{@release_version}.tar.gz"
     @script = File.expand_path("dev/release/02-source.sh")
+    @tarball_script = File.expand_path("dev/release/utils-create-release-tarball.sh")
 
     Dir.mktmpdir do |dir|
       Dir.chdir(dir) do
@@ -40,8 +42,9 @@ def source(*targets)
     targets.each do |target|
       env["SOURCE_#{target}"] = "1"
     end
+    sh(env, @tarball_script, @release_version, "0") 
     output = sh(env, @script, @release_version, "0")
-    sh("tar", "xf", "#{@tag_name}.tar.gz")
+    sh("tar", "xf", @archive_name)
     output
   end
 

dev/release/02-source.sh

@@ -21,6 +21,7 @@
 set -eu
 
 : ${SOURCE_DEFAULT:=1}
+: ${SOURCE_DOWNLOAD:=${SOURCE_DEFAULT}}
 : ${SOURCE_RAT:=${SOURCE_DEFAULT}}
 : ${SOURCE_UPLOAD:=${SOURCE_DEFAULT}}
 : ${SOURCE_PR:=${SOURCE_DEFAULT}}
@@ -37,11 +38,10 @@ fi
 version=$1
 rc=$2
 
-tag=apache-arrow-${version}
+tag=apache-arrow-${version}-rc${rc}
 maint_branch=maint-${version}
 rc_branch="release-${version}-rc${rc}"
-tagrc=${tag}-rc${rc}
-rc_url="https://dist.apache.org/repos/dist/dev/arrow/${tagrc}"
+rc_url="https://dist.apache.org/repos/dist/dev/arrow/${tag}"
 
 echo "Preparing source for tag ${tag}"
 
@@ -56,35 +56,19 @@ fi
 
 echo "Using commit $release_hash"
 
-tarball=${tag}.tar.gz
-
-rm -rf ${tag}
-# be conservative and use the release hash, even though git produces the same
-# archive (identical hashes) using the scm tag
-(cd "${SOURCE_TOP_DIR}" && \
-  git archive ${release_hash} --prefix ${tag}/) | \
-  tar xf -
-
-# Resolve all hard and symbolic links.
-# If we change this, we must change ArrowSources.archive in
-# dev/archery/archery/utils/source.py too.
-rm -rf ${tag}.tmp
-mv ${tag} ${tag}.tmp
-cp -R -L ${tag}.tmp ${tag}
-rm -rf ${tag}.tmp
-
-# Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#.
-dummy_git=${tag}/csharp/dummy.git
-mkdir ${dummy_git}
-pushd ${dummy_git}
-echo ${release_hash} > HEAD
-echo '[remote "origin"] url = https://github.com/apache/arrow.git' >> config
-mkdir objects refs
-popd
-
-# Create new tarball from modified source directory
-tar czf ${tarball} ${tag}
-rm -rf ${tag}
+tarball=apache-arrow-${version}.tar.gz
+
+if [ ${SOURCE_DOWNLOAD} -gt 0 ]; then
+  # Wait for the release candidate workflow to finish before attempting 
+  # to download the tarball from the GitHub Release.
+  . $SOURCE_DIR/utils-watch-gh-workflow.sh ${tag} "release_candidate.yml"
+  rm -f ${tarball}
+  gh release download \
+    ${tag} \
+    --repo apache/arrow \
+    --dir . \
+    --pattern "${tarball}"
+fi
 
 if [ ${SOURCE_RAT} -gt 0 ]; then
   "${SOURCE_DIR}/run-rat.sh" ${tarball}
@@ -105,18 +89,21 @@ if [ ${SOURCE_UPLOAD} -gt 0 ]; then
   ${sha256_generate} $tarball > ${tarball}.sha256
   ${sha512_generate} $tarball > ${tarball}.sha512
 
+  # Upload signed tarballs to GitHub Release
+  gh release upload ${tag} ${tarball}.sha256 ${tarball}.sha512
+
   # check out the arrow RC folder
   svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp
 
   # add the release candidate for the tag
-  mkdir -p tmp/${tagrc}
+  mkdir -p tmp/${tag}
 
   # copy the rc tarball into the tmp dir
-  cp ${tarball}* tmp/${tagrc}
+  cp ${tarball}* tmp/${tag}
 
   # commit to svn
-  svn add tmp/${tagrc}
-  svn ci -m "Apache Arrow ${version} RC${rc}" tmp/${tagrc}
+  svn add tmp/${tag}
+  svn ci -m "Apache Arrow ${version} RC${rc}" tmp/${tag}
 
   # clean up
   rm -rf tmp

dev/release/03-binary-submit.sh

@@ -28,7 +28,7 @@ version=$1
 rc=$2
 version_with_rc="${version}-rc${rc}"
 crossbow_job_prefix="release-${version_with_rc}"
-release_tag="apache-arrow-${version}"
+release_tag="apache-arrow-${version}-rc${rc}"
 rc_branch="release-${version_with_rc}"
 
 : ${ARROW_REPOSITORY:="apache/arrow"}