Skip to content

Commit 4be2a03

Browse files
BrianHarrisonAMDBrianHarrisonAMD
authored
[WA] Disable MLIR when building debug to workaround sanitizer issue (#3200)
1 parent a6a59c2 commit 4be2a03

9 files changed

+26
-11
lines changed

‎Jenkinsfile‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,21 @@ def cmake_build(Map conf=[:]){
3333
def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "")
3434
def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
3535
def prefixpath = conf.get("prefixpath","/opt/rocm")
36+
def build_type_debug = (conf.get("build_type",'release') == 'debug')
37+
def code_conv_enabled = conf.get("codecov", false)
38+
3639
def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON")
40+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
41+
if (build_type_debug || code_conv_enabled)
42+
{
43+
mlir_args = " -DMIOPEN_USE_MLIR=OFF"
44+
}
45+
3746
def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","")
3847
def build_fin = conf.get("build_fin", "OFF")
3948

4049
setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
4150

42-
def build_type_debug = (conf.get("build_type",'release') == 'debug')
43-
4451
//cmake_env can overwrite default CXX variables.
4552
def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")
4653

@@ -72,7 +79,7 @@ def cmake_build(Map conf=[:]){
7279
test_flags = " --disable-verification-cache " + test_flags
7380
}
7481

75-
if(conf.get("codecov", false)){ //Need
82+
if(code_conv_enabled){ //Need
7683
setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags} -fprofile-arcs -ftest-coverage' -DCODECOV_TEST=On " + setup_args
7784
}else if(build_type_debug){
7885
setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args

‎test/nightlies/JenkinsfileNightlyAux‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){
4646
timeout(time: 8, unit: 'HOURS')
4747
{
4848
sh "echo \$HSA_ENABLE_SDMA"
49-
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
49+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
50+
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
5051
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
5152
sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
5253

‎test/nightlies/JenkinsfileNightlyConv2D‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
4747
timeout(time: 8, unit: 'HOURS')
4848
{
4949
sh "echo \$HSA_ENABLE_SDMA"
50-
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
50+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
51+
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
5152
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
5253
sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
5354

‎test/nightlies/JenkinsfileNightlyConv2Daux‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){
4848
timeout(time: 8, unit: 'HOURS')
4949
{
5050
sh "echo \$HSA_ENABLE_SDMA"
51-
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
51+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
52+
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
5253
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
5354
sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
5455

‎test/nightlies/JenkinsfileNightlyFastFullConv2D‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ def buildJob(compiler, flags, image, test, testargs){
4343
timeout(time: 12, unit: 'HOURS')
4444
{
4545
sh "echo \$HSA_ENABLE_SDMA"
46-
sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
46+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
47+
sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
4748
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
4849
sh "MIOPEN_ENABLE_LOGGING_CMD=1 MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
4950

‎test/nightlies/JenkinsfileNightlyFusions‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){
4646
timeout(time: 8, unit: 'HOURS')
4747
{
4848
sh "echo \$HSA_ENABLE_SDMA"
49-
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
49+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
50+
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
5051
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
5152
sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
5253

‎test/nightlies/JenkinsfileNightlyRNN‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
4747
timeout(time: 8, unit: 'HOURS')
4848
{
4949
sh "echo \$HSA_ENABLE_SDMA"
50-
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
50+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
51+
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
5152
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
5253
sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
5354

‎test/nightlies/JenkinsfileNightlyReduce‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
4747
timeout(time: 6, unit: 'HOURS')
4848
{
4949
sh "echo \$HSA_ENABLE_SDMA"
50-
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
50+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
51+
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
5152
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
5253
sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
5354

‎test/nightlies/JenkinsfileNightlyTensorOps‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){
4848
timeout(time: 6, unit: 'HOURS')
4949
{
5050
sh "echo \$HSA_ENABLE_SDMA"
51-
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
51+
// WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
52+
sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
5253
sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
5354
sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
5455

0 commit comments

Comments
 (0)