CI: Check that test expectations are up-to-date

Author: sbc100

.circleci/config.yml

@@ -973,58 +973,4 @@ workflows:
   build-test:
     jobs:
       - ruff
-      - mypy
-      - eslint
-      - build-docs
-      - build-linux
-      - test-sanity:
-          requires:
-            - build-linux
-      - test-posixtest:
-          requires:
-            - build-linux
-      - test-core0:
-          requires:
-            - build-linux
-      - test-core2:
-          requires:
-            - build-linux
-      - test-core3:
-          requires:
-            - build-linux
-      - test-wasm64:
-          requires:
-            - build-linux
-      - test-wasm64-4gb:
-          requires:
-            - build-linux
-      - test-wasm2js1:
-          requires:
-            - build-linux
-      - test-other:
-          requires:
-            - build-linux
-      - test-browser-chrome:
-          requires:
-            - build-linux
-      - test-browser-chrome-2gb:
-          requires:
-            - build-linux
-      - test-browser-chrome-wasm64:
-          requires:
-            - build-linux
-      - test-browser-chrome-wasm64-4gb:
-          requires:
-            - build-linux
-      - test-browser-firefox:
-          requires:
-            - build-linux
-      - test-browser-firefox-wasm64
-      - test-sockets-chrome:
-          requires:
-            - build-linux
-      - test-jsc
-      - test-spidermonkey
-      - test-node-compat
-      - test-windows
       - test-mac-arm64

.github/workflows/check-expectations.yml

@@ -0,0 +1,46 @@
+name: CI
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  check-expectations:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # We want access to other branches, specifically `main`
+      - name: pip install
+        run: |
+          which python3
+          python3 --version
+          python3 -m pip install -r requirements-dev.txt
+      - name: Install emsdk
+        run: |
+          EM_CONFIG=$HOME/emsdk/.emscripten
+          echo $EM_CONFIG
+          echo "EM_CONFIG=$EM_CONFIG" >> $GITHUB_ENV
+          curl -# -L -o ~/emsdk-main.tar.gz https://github.com/emscripten-core/emsdk/archive/main.tar.gz
+          tar -C ~ -xf ~/emsdk-main.tar.gz
+          mv ~/emsdk-main ~/emsdk
+          cd ~/emsdk
+          ./emsdk install tot
+          ./emsdk activate tot
+          echo "JS_ENGINES = [NODE_JS]" >> $EM_CONFIG
+          echo "final config:"
+          cat $EM_CONFIG
+      - name: Check test expectaions on main
+        run: |
+          git checkout origin/main
+          git checkout - ./tools/maint/rebaseline_tests.py
+          ./bootstrap
+          if ! ./tools/maint/rebaseline_tests.py --check-only; then
+            echo "Test expectations are out-of-date on the main branch."
+            echo "You can run `./tools/maint/rebaseline_tests.py --new-branch`"
+            echo "and use it to create a seperate PR."
+            exit 1
+          fi

tools/maint/rebaseline_tests.py

@@ -26,7 +26,8 @@
   'browser.test_small_js_flags',
   'other.test_INCOMING_MODULE_JS_API',
   'other.*code_size*',
-  'other.*codesize*'
+  'other.*codesize*',
+  'skip:other.test_jspi_code_size',
 ]
 
 
@@ -63,33 +64,43 @@ def process_changed_file(filename):
 
 def main(argv):
   parser = argparse.ArgumentParser()
-  parser.add_argument('-s', '--skip-tests', action='store_true', help="don't actually run the tests, just analyze the existing results")
-  parser.add_argument('-b', '--new-branch', action='store_true', help='create a new branch containing the updates')
-  parser.add_argument('-c', '--clear-cache', action='store_true', help='clear the cache before rebaselining (useful when working with llvm changes)')
+  parser.add_argument('-s', '--skip-tests', action='store_true', help="Don't actually run the tests, just analyze the existing results")
+  parser.add_argument('-b', '--new-branch', action='store_true', help='Create a new branch containing the updates')
+  parser.add_argument('-c', '--clear-cache', action='store_true', help='Clear the cache before rebaselining (useful when working with llvm changes)')
+  parser.add_argument('-n', '--check-only', dest='check_only', action='store_true', help='Return non-zero if test expectations are out of date, and skip creating a git commit')
   args = parser.parse_args()
 
   if args.clear_cache:
     run(['emcc', '--clear-cache'])
 
   if not args.skip_tests:
-    if run(['git', 'status', '-uno', '--porcelain']).strip():
+    if not args.check_only and run(['git', 'status', '-uno', '--porcelain']).strip():
       print('tree is not clean')
       return 1
 
     subprocess.check_call(['test/runner', '--rebaseline', '--browser=0'] + TESTS, cwd=root_dir)
 
   if not run(['git', 'status', '-uno', '--porcelain']):
-    print('no updates found')
-    return 1
+    print('test expectations are up-to-date')
+    return 0
 
   output = run(['git', 'status', '-uno', '--porcelain'])
   filenames = []
   for line in output.splitlines():
-    status, filename = line.strip().rsplit(' ', 1)
-    if filename.startswith('test/'):
+    filename = line.strip().rsplit(' ', 1)[1]
+    if filename.startswith('test'):
       filenames.append(filename)
 
-  commit_message = f'''
+  if args.check_only:
+    message = f'''Test expectations are out-of-date
+
+The following ({len(filenames)}) test expectation files were updated by
+running the tests with `--rebaseline`:
+
+```
+'''
+  else:
+    message = f'''
 Automatic rebaseline of codesize expectations. NFC
 
 This is an automatic change generated by tools/maint/rebaseline_tests.py.
@@ -101,18 +112,21 @@ def main(argv):
 '''
 
   for file in filenames:
-    commit_message += process_changed_file(file)
+    message += process_changed_file(file)
 
-  commit_message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n'
+  message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n'
 
-  commit_message += '```\n'
+  message += '```\n'
+
+  print(message)
+  if args.check_only:
+    return 1
 
   if args.new_branch:
     run(['git', 'checkout', '-b', 'rebaseline_tests'])
   run(['git', 'add', '-u', '.'])
-  run(['git', 'commit', '-F', '-'], input=commit_message)
+  run(['git', 'commit', '-F', '-'], input=message)
 
-  print(commit_message)
   return 0