macos_test_runner: add attr post_action_determines_exit_code

Author: kapoorlakshya

apple/testing/default_runner/macos_test_runner.bzl

@@ -57,12 +57,14 @@ def _get_template_substitutions(
         *,
         xctestrun_template,
         pre_action_binary,
-        post_action_binary):
+        post_action_binary,
+        post_action_determines_exit_code):
     """Returns the template substitutions for this runner."""
     subs = {
         "xctestrun_template": xctestrun_template.short_path,
         "pre_action_binary": pre_action_binary,
         "post_action_binary": post_action_binary,
+        "post_action_determines_exit_code": post_action_determines_exit_code,
     }
 
     return {"%(" + k + ")s": subs[k] for k in subs}
@@ -101,8 +103,10 @@ def _macos_test_runner_impl(ctx):
         pre_action_binary = ctx.executable.pre_action.short_path
         runfiles = runfiles.merge(ctx.attr.pre_action[DefaultInfo].default_runfiles)
 
+    post_action_determines_exit_code = False
     if ctx.executable.post_action:
         post_action_binary = ctx.executable.post_action.short_path
+        post_action_determines_exit_code = ctx.attr.post_action_determines_exit_code
         runfiles = runfiles.merge(ctx.attr.post_action[DefaultInfo].default_runfiles)
 
     ctx.actions.expand_template(
@@ -112,6 +116,7 @@ def _macos_test_runner_impl(ctx):
             xctestrun_template = preprocessed_xctestrun_template,
             pre_action_binary = pre_action_binary,
             post_action_binary = post_action_binary,
+            post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
         ),
     )
 
@@ -139,6 +144,12 @@ A binary to run prior to test execution. Sets any environment variables availabl
             cfg = "exec",
             doc = """
 A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$TEST_XCRESULT_BUNDLE_PATH` environment variables, in addition to any other variables available to the test runner.
+""",
+        ),
+        "post_action_determines_exit_code": attr.bool(
+            default = False,
+            doc = """
+When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.
 """,
         ),
         "_test_template": attr.label(

apple/testing/default_runner/macos_test_runner.template.sh

@@ -157,14 +157,23 @@ xcodebuild test-without-building \
 
 # Run a post-action binary, if provided.
 post_action_binary=%(post_action_binary)s
+post_action_determines_exit_code="%(post_action_determines_exit_code)s"
+post_action_exit_code=0
 TEST_EXIT_CODE=$test_exit_code \
   TEST_LOG_FILE="$testlog" \
   TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \
-  "$post_action_binary"
+  "$post_action_binary" || post_action_exit_code=$?
 
-if [[ "$test_exit_code" -ne 0 ]]; then
-  echo "error: tests exited with '$test_exit_code'" >&2
-  exit "$test_exit_code"
+if [[ "$post_action_determines_exit_code" == true ]]; then
+  if [[ "$post_action_exit_code" -ne 0 ]]; then
+    echo "error: post_action exited with '$post_action_exit_code'" >&2
+    exit "$post_action_exit_code"
+  fi
+else
+  if [[ "$test_exit_code" -ne 0 ]]; then
+    echo "error: tests exited with '$test_exit_code'" >&2
+    exit "$test_exit_code"
+  fi
 fi
 
 if [[ "${COVERAGE:-}" -ne 1 ]]; then

test/macos_test_runner_unit_test.sh

@@ -76,6 +76,27 @@ macos_test_runner(
     pre_action = ":pre_action",
     post_action = ":post_action",
 )
+
+
+genrule(
+  name = "post_action_soft_fail_gen",
+  executable = True,
+  outs = ["post_action_soft_fail.bash"],
+  cmd = """
+echo 'echo "POST-ACTION: Soft failing." && exit 0' > \$@
+""",
+)
+
+sh_binary(
+  name = "post_action_soft_fail",
+  srcs = [":post_action_soft_fail_gen"],
+)
+
+macos_test_runner(
+    name = "macos_runner_with_soft_fail",
+    post_action = ":post_action_soft_fail",
+    post_action_determines_exit_code = True,
+)
 EOF
 }
 
@@ -281,6 +302,14 @@ macos_unit_test(
     minimum_os_version = "${MIN_OS_MACOS}",
     runner = ":macos_runner",
 )
+
+macos_unit_test(
+    name = 'SoftFailingUnitTest',
+    infoplists = ["FailUnitTest-Info.plist"],
+    deps = [":fail_unit_test_lib"],
+    minimum_os_version = "${MIN_OS_MACOS}",
+    runner = ":macos_runner_with_soft_fail",
+)
 EOF
 }
 
@@ -518,6 +547,17 @@ function test_macos_unit_test_fail() {
   expect_log "Executed 1 test, with 1 failure"
 }
 
+function test_macos_unit_test_soft_fail() {
+  create_runners
+  create_macos_unit_tests
+  do_macos_test //macos:SoftFailingUnitTest || fail "should pass"
+
+  expect_log "Test Suite 'FailingUnitTest' failed"
+  expect_log "Test Suite 'SoftFailingUnitTest.xctest' failed"
+  expect_log "Executed 1 test, with 1 failure"
+  expect_log "POST-ACTION: Soft failing."
+}
+
 function test_macos_unit_test_with_filter() {
   create_runners
   create_macos_unit_tests