Skip to content

Enable more spec tests #6669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tlively merged 3 commits into from
Jun 17, 2024
Merged

Enable more spec tests #6669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
45b9c61
Enable more spec tests
tlively Jun 15, 2024
54a660d
skip custom.wast
tlively Jun 17, 2024
397a634
flake8
tlively Jun 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 20 additions & 28 deletions check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,38 +221,30 @@ def check_expected(actual, expected):

check_expected(actual, expected)

# skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature
if 'exports.wast' in base: # FIXME
continue

run_spec_test(wast)

# check binary format. here we can verify execution of the final
# result, no need for an output verification
# some wast files cannot be split:
# * comments.wast: contains characters that are not valid utf-8,
# so our string splitting code fails there

# FIXME Remove reference type tests from this list after nullref is
# implemented in V8
if base not in ['comments.wast', 'ref_null.wast', 'ref_is_null.wast', 'ref_func.wast', 'old_select.wast']:
split_num = 0
actual = ''
with open('spec.wast', 'w') as transformed_spec_file:
for module, asserts in support.split_wast(wast):
print(' testing split module', split_num)
split_num += 1
support.write_wast('split.wast', module)
run_opt_test('split.wast') # also that our optimizer doesn't break on it
result_wast_file = shared.binary_format_check('split.wast', verify_final_result=False)
with open(result_wast_file) as f:
result_wast = f.read()
# add the asserts, and verify that the test still passes
transformed_spec_file.write(result_wast + '\n' + '\n'.join(asserts))

# compare all the outputs to the expected output
actual = run_spec_test('spec.wast')
check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log'))
split_num = 0
actual = ''
with open('spec.wast', 'w') as transformed_spec_file:
for module, asserts in support.split_wast(wast):
if not module:
# Skip any initial assertions that don't have a module
continue
print(' testing split module', split_num)
split_num += 1
support.write_wast('split.wast', module)
run_opt_test('split.wast') # also that our optimizer doesn't break on it
result_wast_file = shared.binary_format_check('split.wast', verify_final_result=False)
with open(result_wast_file) as f:
result_wast = f.read()
# add the asserts, and verify that the test still passes
transformed_spec_file.write(result_wast + '\n' + '\n'.join(asserts))

# compare all the outputs to the expected output
actual = run_spec_test('spec.wast')
check_expected(actual, os.path.join(shared.get_test_dir('spec'), 'expected-output', base + '.log'))


def run_validator_tests():
Expand Down
60 changes: 18 additions & 42 deletions scripts/test/shared.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,57 +399,33 @@ def get_tests(test_dir, extensions=[], recursive=False):
# delete the old file, make sure you rename the corresponding .wast.log file in
# expected-output/ if any.
SPEC_TESTS_TO_SKIP = [
# Stacky code / notation
'block.wast',
'call.wast',
'float_exprs.wast',
# Malformed module accepted
'globals.wast',
'loop.wast',
'nop.wast',
'select.wast',
'stack.wast',
'unwind.wast',

# Binary module
'binary.wast',
'binary-leb128.wast',
'custom.wast',

# Empty 'then' or 'else' in 'if'
'if.wast',
'local_set.wast',
'store.wast',

# No module in a file
'token.wast',
'utf8-custom-section-id.wast',
'utf8-import-field.wast',
'utf8-import-module.wast',
'utf8-invalid-encoding.wast',
'const.wast',
'address.wast',
'custom.wast', # invalid section ID accepted as Custom, triggering UBSan

# 'register' command
# Unlinkable module accepted
'linking.wast',

# Misc. unsupported constructs
'call_indirect.wast', # Empty (param) and (result)
'const.wast', # Unparenthesized expression
'data.wast', # Various unsupported (data) notations
'elem.wast', # Unsupported 'offset' syntax in (elem)
'exports.wast', # Multiple inlined exports for a function
'func.wast', # Forward named type reference
'skip-stack-guard-page.wast', # Hexadecimal style (0x..) in memory offset

# Untriaged: We don't know the cause of the error yet
'address.wast', # wasm2js 'assert_return' failure
'br_if.wast', # Validation error
'float_literals.wast', # 'assert_return' failure
'int_literals.wast', # 'assert_return' failure
'local_tee.wast', # Validation failure
'memory_grow.wast', # 'assert_return' failure
'start.wast', # Assertion failure
'type.wast', # 'assertion_invalid' failure
'unreachable.wast', # Validation failure
'unreached-invalid.wast' # 'assert_invalid' failure
# Invalid module accepted
'func.wast',
'type.wast',
'unreached-invalid.wast',

# WAT parser error
'unwind.wast',

# WAST parser error
'binary.wast',

# Test invalid
'elem.wast',
]
options.spec_tests = [t for t in options.spec_tests if os.path.basename(t) not
in SPEC_TESTS_TO_SKIP]
Expand Down
13 changes: 12 additions & 1 deletion scripts/test/support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import filecmp
import os
import re
import shutil
import subprocess
import sys
Expand Down Expand Up @@ -87,6 +88,9 @@ def untar(tarfile, outdir):
shutil.rmtree(tmpdir)


QUOTED = re.compile(r'\(module\s*(\$\S*)?\s+(quote|binary)')


def split_wast(wastFile):
# if it's a binary, leave it as is, we can't split it
wast = None
Expand Down Expand Up @@ -124,6 +128,7 @@ def to_end(j):
return j

i = 0
ignoring_quoted = False
while i >= 0:
start = wast.find('(', i)
if start >= 0 and wast[start + 1] == ';':
Expand All @@ -141,11 +146,17 @@ def to_end(j):
break
i = to_end(start + 1)
chunk = wast[start:i]
if QUOTED.match(chunk):
# There may be assertions after this quoted module, but we aren't
# returning the module, so we need to skip the assertions as well.
ignoring_quoted = True
continue
if chunk.startswith('(module'):
ignoring_quoted = False
ret += [(chunk, [])]
elif chunk.startswith('(assert_invalid'):
continue
elif chunk.startswith(('(assert', '(invoke', '(register')):
elif chunk.startswith(('(assert', '(invoke', '(register')) and not ignoring_quoted:
# ret may be empty if there are some asserts before the first
# module. in that case these are asserts *without* a module, which
# are valid (they may check something that doesn't refer to a module
Expand Down
51 changes: 33 additions & 18 deletions test/spec/stack.wast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,15 @@
(i64.eq)
(if
(then (br $done))
(then
(else
(local.get $i)
(local.get $res)
(i64.mul)
(local.set $res)
(local.get $i)
(i64.const 1)
(i64.sub)
(local.set $i)
)
(else
(local.get $i)
(local.get $res)
(i64.mul)
(local.set $res)
(local.get $i)
(i64.const 1)
(i64.sub)
(local.set $i)
)
)
(br $loop)
Expand Down Expand Up @@ -93,13 +91,11 @@
(i64.eq (local.get $i) (i64.const 0))
(if
(then (br $done))
(then
(else
(i64.mul (local.get $i) (local.get $res))
(local.set $res)
(i64.sub (local.get $i) (i64.const 1))
(local.set $i)
)
(else
(i64.mul (local.get $i) (local.get $res))
(local.set $res)
(i64.sub (local.get $i) (i64.const 1))
(local.set $i)
)
)
(br $loop)
Expand Down Expand Up @@ -129,12 +125,31 @@
end
local.get $res
)

(global $temp (mut i32) (i32.const 0))
(func $add_one_to_global (result i32)
(local i32)
(global.set $temp (i32.add (i32.const 1) (global.get $temp)))
(global.get $temp)
)
(func $add_one_to_global_and_drop
(drop (call $add_one_to_global))
)
(func (export "not-quite-a-tree") (result i32)
call $add_one_to_global
call $add_one_to_global
call $add_one_to_global_and_drop
i32.add
)
)

(assert_return (invoke "fac-expr" (i64.const 25)) (i64.const 7034535277573963776))
(assert_return (invoke "fac-stack" (i64.const 25)) (i64.const 7034535277573963776))
(assert_return (invoke "fac-mixed" (i64.const 25)) (i64.const 7034535277573963776))

(assert_return (invoke "not-quite-a-tree") (i32.const 3))
(assert_return (invoke "not-quite-a-tree") (i32.const 9))


;; Syntax of flat call_indirect

Expand Down