F Release Promote redex-staging to redex-stable 2nd land

Summary:
We are moving forward with the F release based on the AB test [results](https://fb.workplace.com/groups/RedexTeam/posts/1644837602730337/).

Re-land of D61694226

allow-large-files

Reviewed By: ssj933

Differential Revision: D61984453

fbshipit-source-id: 530dd5bf7f5b6d06c91da09edf23b6a35011dbb1

Author: thezhangwei

.circleci/config.yml

@@ -138,17 +138,6 @@ commands:
 
   # For testing, need additional dependencies not provided by the image.
 
-  # Version where dx is available as a package. May remove this eventually.
-  test-build-setup-no-sdk:
-    steps:
-      - run:
-          name: Setup for tests
-          command: |
-            sudo apt-get install -y --no-install-recommends dalvik-exchange default-jdk-headless
-            sudo ln -s /usr/bin/dalvik-exchange /usr/local/bin/dx
-
-  # Special version of the above to work around Debian Testing package issues.
-  #
   # Note: CircleCI does not allow to drop existing caches (in case they are
   #       outdated, for example). The expectation is that the "cache key"
   #       changes, for example via the checksum of a dependency file.
@@ -163,72 +152,51 @@ commands:
   #       See:
   #       * [https://support.circleci.com/hc/en-us/articles/115015426888-Clear-project-dependency-cache]
   #       * [https://devops.stackexchange.com/questions/9147/how-to-get-other-than-no-value-when-interpolating-environment-some-var]
-  test-build-setup-sdk:
+  test-build-setup:
     steps:
       - run:
-          name: Setup for tests (Deb Testing)
-          command: |
-            sudo apt-get install -y --no-install-recommends default-jdk-headless
-
-      - run:
-          name: Create cache key file
+          name: Setup for tests
           command: |
-            echo "Cache key is: ${CACHE_VERSION}"
-            echo "${CACHE_VERSION}" > cache_version.rev
+            sudo apt-get install -y --no-install-recommends default-jdk-headless zip unzip
 
       - restore_cache:
           name: Restoring SDK Cache
           keys:
-            - android-build-tools-{{ checksum "cache_version.rev" }}
+            - v3-android-build-tools
 
       # We have to emulate cache behavior. Skip downloads if files exist.
       - run:
           name: Check/Install (SDK)
           command: |
-            if [ -e ~/sdk/build-tools/25.0.3/dx ] ; then
-              echo "Found SDK."
-              exit 0
+            if [ -e sdk/build-tools/29.0.2/dx ] ; then echo "Found SDK." ; exit 0 ; fi
+
+            # Check whether cache file exists, then check dx again.
+            if [ -f "sdk.tar.zstd" ] ; then
+              zstd --decompress --stdout sdk.tar.zstd | tar xf -
+              if [ -e sdk/build-tools/29.0.2/dx ] ; then echo "Unpacked SDK." ; exit 0 ; fi
             fi
+
+            # Download.
             echo "SDK missing, downloading..."
-            rm -rf ~/sdk 2>/dev/null
+            rm -rf sdk 2>/dev/null && mkdir -p sdk/cmdline-tools
+            echo "Downloading SDK to $(pwd)/sdk"
             wget https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip
-            mkdir -p ~/sdk/cmdline-tools
-            unzip commandlinetools-linux-6609375_latest.zip -d ~/sdk/cmdline-tools/
-            pushd ~/sdk/cmdline-tools/tools/bin >/dev/null
-            echo 'y' | ./sdkmanager --install 'build-tools;25.0.3'
+            unzip commandlinetools-linux-6609375_latest.zip -d sdk/cmdline-tools/
+            pushd sdk/cmdline-tools/tools/bin >/dev/null
+            echo 'y' | ./sdkmanager --install 'build-tools;29.0.2'
+            ./sdkmanager --install 'platforms;android-29'
             popd >/dev/null
 
-      - run:
-          name: Check/Install (Symlink)
-          command: |
-            if [ -L /usr/local/bin/dx ] ; then
-              echo "Found symlink."
-              exit 0
-            fi
-            echo "Adding symlink..."
-            sudo ln -s ~/sdk/build-tools/25.0.3/dx /usr/local/bin/dx
+            # Compress for cache. For reason see description of build cache.
+            # We do not need the emulator and it is large. Delete.
+            rm -rf sdk/emulator 2>/dev/null
+            tar cf - sdk | zstd -T0 -3 > sdk.tar.zstd
 
       - save_cache:
           name: Saving SDK Cache
           paths:
-            - ~/sdk
-          key: android-build-tools-{{ checksum "cache_version.rev" }}
-
-  test-build-setup:
-    parameters:
-      use_sdk:
-        type: boolean
-        default: true
-    steps:
-      - when:
-          condition: << parameters.use_sdk >>
-          steps:
-            - test-build-setup-sdk
-      - when:
-          condition:
-            not: << parameters.use_sdk >>
-          steps:
-            - test-build-setup-no-sdk
+            - sdk.tar.zstd
+          key: v3-android-build-tools
 
   # Level of indirection because there seems to be no way to set parameters
   # in the steps.
@@ -247,18 +215,14 @@ commands:
       run_tests:
         type: boolean
         default: true
-      use_sdk:
-        type: boolean
-        default: true
       job_name:
         type: string
     steps:
       - base-build-setup:
           save_boost_cache: << parameters.save_boost_cache >>
           setup_toolchain_extra: << parameters.setup_toolchain_extra >>
 
-      - test-build-setup:
-          use_sdk: << parameters.use_sdk >>
+      - test-build-setup
 
       - run:
           name: Create Build Cache File
@@ -270,7 +234,8 @@ commands:
           keys:
           # Always distinguish caches for the job, arch and branch. Look for the current revision first,
           # then "newest" cache.
-            - &build-cache v1-build-cache-{{ checksum "/tmp/build.txt" }}-{{ arch }}-{{ .Branch }}-{{ .Revision }}
+            - &build-cache v2-build-cache-{{ checksum "/tmp/build.txt" }}-{{ arch }}-{{ .Branch }}-{{ .Revision }}
+            - v2-build-cache-{{ checksum "/tmp/build.txt" }}-{{ arch }}-{{ .Branch }}-
             - v1-build-cache-{{ checksum "/tmp/build.txt" }}-{{ arch }}-{{ .Branch }}-
 
       - run:
@@ -289,7 +254,7 @@ commands:
           name: Configure
           working_directory: build
           command: |
-            ../repo/configure --enable-protobuf << parameters.configure_extra >>
+            ../repo/configure --enable-protobuf --with-android-sdk="$(pwd)/../sdk" << parameters.configure_extra >>
 
       - run:
           name: Build
@@ -347,10 +312,6 @@ commands:
       mode_32:
         type: boolean
         default: false
-      # Download DX from the Android SDK.
-      use_sdk:
-        type: boolean
-        default: true
       job_name:
         type: string
     steps:
@@ -359,7 +320,6 @@ commands:
           steps:
             - setup-build-and-test-w-make-impl:
                 save_boost_cache: << parameters.save_boost_cache >>
-                use_sdk: << parameters.use_sdk >>
                 setup_toolchain_extra: "32"
                 # Would be nice to properly use environment, but that's only at
                 # the jobs or run level.
@@ -374,7 +334,6 @@ commands:
           steps:
             - setup-build-and-test-w-make-impl:
                 save_boost_cache: << parameters.save_boost_cache >>
-                use_sdk: << parameters.use_sdk >>
                 job_name: << parameters.job_name >>
 
   build_debian:
@@ -392,9 +351,6 @@ commands:
       mode_32:
         type: boolean
         default: false
-      use_sdk:
-        type: boolean
-        default: true
       job_name:
         type: string
     steps:
@@ -411,7 +367,6 @@ commands:
             git restore-mtime
       - setup-build-and-test-w-make:
           save_boost_cache: << parameters.save_boost_cache >>
-          use_sdk: << parameters.use_sdk >>
           mode_32: << parameters.mode_32 >>
           job_name: << parameters.job_name >>-<< parameters.install_clang_llvm_org >>
 
@@ -482,7 +437,6 @@ jobs:
     resource_class: large
     steps:
       - build_debian:
-          use_sdk: false
           job_name: debian_10
 
   build-deb_stable:

.github/actions/base-build-setup/action.yml

@@ -0,0 +1,29 @@
+name: base-build-setup
+inputs:
+  save_boost_cache:
+    required: false
+    default: false
+  setup_toolchain_extra:
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+  - name: Cache boost
+    if: inputs.save_boost_cache == 'true'
+    uses: actions/[email protected]
+    with:
+      key: boost-1-71-0-v5
+      path: /tmp/toolchain/dl_cache/boost_cache
+  - name: Cache protobuf
+    if: inputs.save_boost_cache == 'true'
+    uses: actions/[email protected]
+    with:
+      key: protobuf3-v3
+      path: /tmp/toolchain/dl_cache/protobuf
+  - name: Setup
+    run: sudo -E ./setup_oss_toolchain.sh ${{ inputs.setup_toolchain_extra }}
+    working-directory: repo
+    env:
+      TOOLCHAIN_TMP: /tmp/toolchain
+    shell: bash

.github/actions/build_debian/action.yml

@@ -0,0 +1,43 @@
+name: build_debian
+inputs:
+  save_boost_cache:
+    required: false
+    default: false
+  install_clang:
+    required: false
+    default: false
+  install_clang_llvm_org:
+    required: false
+    default: -1
+  mode_32:
+    required: false
+    default: false
+  job_name:
+    required: false
+runs:
+  using: composite
+  steps:
+  # Reinitialize. It is easier to do this (again) to not have to deal with
+  # fixing `uses:` paths.
+  - uses: actions/[email protected]
+    with:
+      path: repo
+      # We want to run restore-mtime, which needs to find actual commit data
+      # and not just the shallow default result. The easiest way for this seems
+      # to be the filter support of Github (https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/).
+      fetch-depth: 1000000
+      filter: "tree:0"
+  - uses: ./.github/actions/debian-based-image-build-setup
+    with:
+      mode_32: "${{ inputs.mode_32 }}"
+      install_clang: "${{ inputs.install_clang }}"
+      install_clang_llvm_org: "${{ inputs.install_clang_llvm_org }}"
+  - name: Run git-restore-mtime
+    run: git restore-mtime
+    working-directory: repo
+    shell: bash
+  - uses: ./.github/actions/setup-build-and-test-w-make
+    with:
+      save_boost_cache: "${{ inputs.save_boost_cache }}"
+      mode_32: "${{ inputs.mode_32 }}"
+      job_name: "${{ inputs.job_name }}-${{ inputs.install_clang_llvm_org }}"

.github/actions/debian-based-image-build-setup/action.yml

@@ -0,0 +1,43 @@
+name: debian-based-image-build-setup
+inputs:
+  mode_32:
+    required: false
+    default: false
+  install_clang:
+    required: false
+    default: false
+  install_clang_llvm_org:
+    required: false
+    default: -1
+runs:
+  using: composite
+  steps:
+  - name: Update Apt Data
+    run: sudo apt-get -qq update || ( apt-get -qq update && apt-get -qq install --no-install-recommends -y sudo ; )
+    shell: bash
+  - name: Install tools
+    run: sudo apt-get install -qq --no-install-recommends -y git-restore-mtime zstd tar
+    shell: bash
+  - name: Add 32-bit Arch
+    run: sudo dpkg --add-architecture i386 && sudo apt-get update
+    if: inputs.mode_32 == 'true'
+    shell: bash
+  - name: Install Clang
+    run: sudo apt-get install -qq -y --no-install-recommends clang
+    if: inputs.install_clang == 'true'
+    shell: bash
+  - name: Install Clang (apt.llvm.org)
+    run: |-
+      sudo apt-get install -qq -y --no-install-recommends lsb-release software-properties-common
+      sudo /bin/bash -c "$(wget -O - https://apt.llvm.org/llvm.sh | sed -e 's/^add-apt-repository.*$/& -y\n& -y/')"
+      ls /usr/bin/clang*
+    if: 0 == inputs.install_clang_llvm_org
+    shell: bash
+
+  - name: Install Clang (apt.llvm.org)
+    run: |-
+      sudo apt-get install -qq -y --no-install-recommends lsb-release software-properties-common
+      sudo /bin/bash -c "$(wget -O - https://apt.llvm.org/llvm.sh | sed -e 's/^add-apt-repository.*$/& -y\n& -y/')" "llvm.sh" ${{ inputs.install_clang_llvm_org }}
+      ls /usr/bin/clang*
+    if: -1 != inputs.install_clang_llvm_org && 0 != inputs.install_clang_llvm_org
+    shell: bash