Skip to content

Commit cb1141b

Browse files
AaronStGeorgeAaronStGeorge
committed
[Fusilli] Create FUSILLI_CHECK_ERROR macro
This odd [CI failure](https://github.com/nod-ai/shark-ai/actions/runs/18548883676/job/52872578004) might indicate a race on the compile cache (though I don't see how) or a sporadic device issue. It's hard to tell from the logs, and I haven't been able to reproduce this locally. This PR adds some logging so hopefully if this happens in CI again we can get a better picture of what's going on.
1 parent 1b5fbf5 commit cb1141b

17 files changed

+162
-127
lines changed

‎.github/workflows/ci-sharkfuser.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ jobs:
5757
-DCMAKE_C_COMPILER=clang-20
5858
-DCMAKE_CXX_COMPILER=clang++-20
5959
-DCMAKE_BUILD_TYPE=Release
60+
-DFUSILLI_ENABLE_LOGGING=ON
6061
-DFUSILLI_SYSTEMS_AMDGPU=ON
6162
- name: cpu_clang18_debug
6263
runs-on: azure-cpubuilder-linux-scale

‎sharkfuser/include/fusilli/support/logging.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ inline ConditionalStreamer &getLogger() {
416416
do { \
417417
fusilli::ErrorObject _error = (expr); \
418418
if (isError(_error)) { \
419-
FUSILLI_LOG_LABEL_RED("ERROR: "); \
419+
FUSILLI_LOG_LABEL_RED("ERROR: " << _error); \
420420
FUSILLI_LOG_ENDL(#expr << " at " << __FILE__ << ":" << __LINE__); \
421421
return _error; \
422422
} \
@@ -438,7 +438,7 @@ inline ConditionalStreamer &getLogger() {
438438
({ \
439439
auto _errorOr = (expr); \
440440
if (isError(_errorOr)) { \
441-
FUSILLI_LOG_LABEL_RED("ERROR: "); \
441+
FUSILLI_LOG_LABEL_RED("ERROR: " << _errorOr); \
442442
FUSILLI_LOG_ENDL(#expr << " at " << __FILE__ << ":" << __LINE__); \
443443
return fusilli::ErrorObject(_errorOr); \
444444
} \

‎sharkfuser/samples/convolution/conv_fprop_nchw_kcrs.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ TEST_CASE("Convolution fprop; X (NCHW), W (KCRS); 1x1 conv; no padding",
4646
Y->setOutput(true);
4747

4848
// Validate, infer missing properties
49-
REQUIRE(isOk(graph->validate()));
49+
FUSILLI_REQUIRE_OK(graph->validate());
5050

5151
// Compile
52-
REQUIRE(isOk(graph->compile(handle, /*remove=*/true)));
52+
FUSILLI_REQUIRE_OK(graph->compile(handle, /*remove=*/true));
5353

5454
return std::make_tuple(graph, X, W, Y);
5555
};
@@ -98,22 +98,22 @@ TEST_CASE("Convolution fprop; X (NCHW), W (KCRS); 1x1 conv; no padding",
9898
};
9999

100100
// Execute graph once.
101-
REQUIRE(isOk(graph->execute(variantPack)));
101+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
102102

103103
// Read output buffers.
104104
std::vector<half> result;
105-
REQUIRE(isOk(yBuf->read(handle, result)));
105+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
106106
for (auto val : result)
107107
REQUIRE(val == half(128.0f));
108108

109109
// Execute graph a few times.
110110
constexpr size_t numIters = 1;
111111
for (size_t i = 0; i < numIters; i++)
112-
REQUIRE(isOk(graph->execute(variantPack)));
112+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
113113

114114
// Repeat output buffer checks.
115115
result.clear();
116-
REQUIRE(isOk(yBuf->read(handle, result)));
116+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
117117
for (auto val : result)
118118
REQUIRE(val == half(128.0f));
119119
}

‎sharkfuser/samples/convolution/conv_fprop_nhwc_krsc.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 1x1 conv; no padding",
4646
Y->setOutput(true);
4747

4848
// Validate, infer missing properties
49-
REQUIRE(isOk(graph->validate()));
49+
FUSILLI_REQUIRE_OK(graph->validate());
5050

5151
// Compile
52-
REQUIRE(isOk(graph->compile(handle, /*remove=*/true)));
52+
FUSILLI_REQUIRE_OK(graph->compile(handle, /*remove=*/true));
5353

5454
return std::make_tuple(graph, X, W, Y);
5555
};
@@ -98,22 +98,22 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 1x1 conv; no padding",
9898
};
9999

100100
// Execute graph once.
101-
REQUIRE(isOk(graph->execute(variantPack)));
101+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
102102

103103
// Read output buffers.
104104
std::vector<half> result;
105-
REQUIRE(isOk(yBuf->read(handle, result)));
105+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
106106
for (auto val : result)
107107
REQUIRE(val == half(128.0f));
108108

109109
// Execute graph a few times.
110110
constexpr size_t numIters = 1;
111111
for (size_t i = 0; i < numIters; i++)
112-
REQUIRE(isOk(graph->execute(variantPack)));
112+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
113113

114114
// Repeat output buffer checks.
115115
result.clear();
116-
REQUIRE(isOk(yBuf->read(handle, result)));
116+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
117117
for (auto val : result)
118118
REQUIRE(val == half(128.0f));
119119
}

‎sharkfuser/samples/convolution/conv_fprop_nhwc_krsc_with_pad.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 3x3 conv; same padding",
4646
Y->setOutput(true);
4747

4848
// Validate, infer missing properties
49-
REQUIRE(isOk(graph->validate()));
49+
FUSILLI_REQUIRE_OK(graph->validate());
5050

5151
// Compile
52-
REQUIRE(isOk(graph->compile(handle, /*remove=*/true)));
52+
FUSILLI_REQUIRE_OK(graph->compile(handle, /*remove=*/true));
5353

5454
return std::make_tuple(graph, X, W, Y);
5555
};
@@ -98,20 +98,20 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 3x3 conv; same padding",
9898
};
9999

100100
// Execute graph once.
101-
REQUIRE(isOk(graph->execute(variantPack)));
101+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
102102

103103
// Read output buffers.
104104
std::vector<half> result;
105-
REQUIRE(isOk(yBuf->read(handle, result)));
105+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
106106
REQUIRE(result[0] == half(512.0f));
107107

108108
// Execute graph a few times.
109109
constexpr size_t numIters = 1;
110110
for (size_t i = 0; i < numIters; i++)
111-
REQUIRE(isOk(graph->execute(variantPack)));
111+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
112112

113113
// Repeat output buffer checks.
114114
result.clear();
115-
REQUIRE(isOk(yBuf->read(handle, result)));
115+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
116116
REQUIRE(result[0] == half(512.0f));
117117
}

‎sharkfuser/samples/convolution/conv_fprop_with_bias.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 1x1 conv; no padding; bias",
5454
biasResult->setName("result").setOutput(true);
5555

5656
// Validate, infer missing properties
57-
REQUIRE(isOk(graph->validate()));
57+
FUSILLI_REQUIRE_OK(graph->validate());
5858

5959
// Compile
60-
REQUIRE(isOk(graph->compile(handle, /*remove=*/true)));
60+
FUSILLI_REQUIRE_OK(graph->compile(handle, /*remove=*/true));
6161

6262
return std::make_tuple(graph, X, W, B, biasResult);
6363
};
@@ -113,22 +113,22 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 1x1 conv; no padding; bias",
113113
};
114114

115115
// Execute graph once.
116-
REQUIRE(isOk(graph->execute(variantPack)));
116+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
117117

118118
// Read output buffers.
119119
std::vector<half> result;
120-
REQUIRE(isOk(yBuf->read(handle, result)));
120+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
121121
for (auto val : result)
122122
REQUIRE(val == half(129.0f));
123123

124124
// Execute graph a few times.
125125
constexpr size_t numIters = 1;
126126
for (size_t i = 0; i < numIters; i++)
127-
REQUIRE(isOk(graph->execute(variantPack)));
127+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
128128

129129
// Repeat output buffer checks.
130130
result.clear();
131-
REQUIRE(isOk(yBuf->read(handle, result)));
131+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
132132
for (auto val : result)
133133
REQUIRE(val == half(129.0f));
134134
}

‎sharkfuser/samples/convolution/conv_fprop_with_relu.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,10 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 1x1 conv; no padding; relu",
5050
reluResult->setName("result").setOutput(true);
5151

5252
// Validate, infer missing properties
53-
REQUIRE(isOk(graph->validate()));
53+
FUSILLI_REQUIRE_OK(graph->validate());
5454

5555
// Compile
56-
REQUIRE(isOk(graph->compile(handle, /*remove=*/true)));
56+
FUSILLI_REQUIRE_OK(graph->compile(handle, /*remove=*/true));
5757

5858
return std::make_tuple(graph, X, W, reluResult);
5959
};
@@ -102,22 +102,22 @@ TEST_CASE("Convolution fprop; X (NHWC), W (KRSC); 1x1 conv; no padding; relu",
102102
};
103103

104104
// Execute graph once.
105-
REQUIRE(isOk(graph->execute(variantPack)));
105+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
106106

107107
// Read output buffers.
108108
std::vector<half> result;
109-
REQUIRE(isOk(yBuf->read(handle, result)));
109+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
110110
for (auto val : result)
111111
REQUIRE(val == half(128.0f));
112112

113113
// Execute graph a few times.
114114
constexpr size_t numIters = 1;
115115
for (size_t i = 0; i < numIters; i++)
116-
REQUIRE(isOk(graph->execute(variantPack)));
116+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
117117

118118
// Repeat output buffer checks.
119119
result.clear();
120-
REQUIRE(isOk(yBuf->read(handle, result)));
120+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
121121
for (auto val : result)
122122
REQUIRE(val == half(128.0f));
123123
}

‎sharkfuser/samples/convolution/conv_fprop_with_relu_and_bias.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,10 @@ TEST_CASE(
5959
reluResult->setName("result").setOutput(true);
6060

6161
// Validate, infer missing properties
62-
REQUIRE(isOk(graph->validate()));
62+
FUSILLI_REQUIRE_OK(graph->validate());
6363

6464
// Compile
65-
REQUIRE(isOk(graph->compile(handle, /*remove=*/true)));
65+
FUSILLI_REQUIRE_OK(graph->compile(handle, /*remove=*/true));
6666

6767
return std::make_tuple(graph, X, W, B, reluResult);
6868
};
@@ -118,22 +118,22 @@ TEST_CASE(
118118
};
119119

120120
// Execute graph once.
121-
REQUIRE(isOk(graph->execute(variantPack)));
121+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
122122

123123
// Read output buffers.
124124
std::vector<half> result;
125-
REQUIRE(isOk(yBuf->read(handle, result)));
125+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
126126
for (auto val : result)
127127
REQUIRE(val == half(129.0f));
128128

129129
// Execute graph a few times.
130130
constexpr size_t numIters = 1;
131131
for (size_t i = 0; i < numIters; i++)
132-
REQUIRE(isOk(graph->execute(variantPack)));
132+
FUSILLI_REQUIRE_OK(graph->execute(variantPack));
133133

134134
// Repeat output buffer checks.
135135
result.clear();
136-
REQUIRE(isOk(yBuf->read(handle, result)));
136+
FUSILLI_REQUIRE_OK(yBuf->read(handle, result));
137137
for (auto val : result)
138138
REQUIRE(val == half(129.0f));
139139
}

‎sharkfuser/tests/test_buffer.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ TEST_CASE("Buffer allocation, move semantics and lifetime", "[buffer]") {
3737

3838
// Read buffer and check contents.
3939
std::vector<float> result;
40-
REQUIRE(isOk(buf.read(handle, result)));
40+
FUSILLI_REQUIRE_OK(buf.read(handle, result));
4141
for (auto val : result)
4242
REQUIRE(val == 1.0f);
4343

@@ -51,7 +51,7 @@ TEST_CASE("Buffer allocation, move semantics and lifetime", "[buffer]") {
5151

5252
// Read moved buffer and check contents.
5353
result.clear();
54-
REQUIRE(isOk(movedBuf.read(handle, result)));
54+
FUSILLI_REQUIRE_OK(movedBuf.read(handle, result));
5555
for (auto val : result)
5656
REQUIRE(val == 1.0f);
5757
}
@@ -79,7 +79,7 @@ TEST_CASE("Buffer import and lifetimes", "[buffer]") {
7979

8080
// Read buffer and check contents.
8181
std::vector<half> result;
82-
REQUIRE(isOk(buf.read(handle, result)));
82+
FUSILLI_REQUIRE_OK(buf.read(handle, result));
8383
for (auto val : result)
8484
REQUIRE(val == half(1.0f));
8585

@@ -92,7 +92,7 @@ TEST_CASE("Buffer import and lifetimes", "[buffer]") {
9292

9393
// Read imported buffer and check contents.
9494
result.clear();
95-
REQUIRE(isOk(importedBuf.read(handle, result)));
95+
FUSILLI_REQUIRE_OK(importedBuf.read(handle, result));
9696
for (auto val : result)
9797
REQUIRE(val == half(1.0f));
9898
}
@@ -102,7 +102,7 @@ TEST_CASE("Buffer import and lifetimes", "[buffer]") {
102102

103103
// Read original buffer and check contents.
104104
result.clear();
105-
REQUIRE(isOk(buf.read(handle, result)));
105+
FUSILLI_REQUIRE_OK(buf.read(handle, result));
106106
for (auto val : result)
107107
REQUIRE(val == 1.0f);
108108
}
@@ -138,7 +138,7 @@ TEST_CASE("Buffer errors", "[buffer]") {
138138

139139
// Read buffer into an empty vector should work.
140140
result.clear();
141-
REQUIRE(isOk(buf.read(handle, result)));
141+
FUSILLI_REQUIRE_OK(buf.read(handle, result));
142142
for (auto val : result)
143143
REQUIRE(val == 0.0f);
144144
}

‎sharkfuser/tests/test_cache.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ TEST_CASE("CacheFile::create remove = true", "[CacheFile]") {
2727
REQUIRE(std::filesystem::exists(cacheFilePath));
2828

2929
// Roundtrip writing and reading.
30-
REQUIRE(isOk(cf.write("test content")));
30+
FUSILLI_REQUIRE_OK(cf.write("test content"));
3131
std::string content = FUSILLI_REQUIRE_UNWRAP(cf.read());
3232
REQUIRE(content == "test content");
3333
}
@@ -46,8 +46,8 @@ TEST_CASE("CacheFile::create remove = true", "[CacheFile]") {
4646
/*remove=*/true));
4747

4848
// Write different content to each file.
49-
REQUIRE(isOk(cf1.write("content1")));
50-
REQUIRE(isOk(cf2.write("content2")));
49+
FUSILLI_REQUIRE_OK(cf1.write("content1"));
50+
FUSILLI_REQUIRE_OK(cf2.write("content2"));
5151

5252
// Store paths before move.
5353
std::filesystem::path path1 = cf1.path;
@@ -109,8 +109,8 @@ TEST_CASE("CacheFile::create remove = false", "[CacheFile]") {
109109
/*remove=*/false));
110110

111111
// Write different content to each file.
112-
REQUIRE(isOk(cf1.write("content1")));
113-
REQUIRE(isOk(cf2.write("content2")));
112+
FUSILLI_REQUIRE_OK(cf1.write("content1"));
113+
FUSILLI_REQUIRE_OK(cf2.write("content2"));
114114

115115
// Store paths before move.
116116
std::filesystem::path path1 = cf1.path;
@@ -156,7 +156,7 @@ TEST_CASE("CacheFile::open", "[CacheFile]") {
156156
/*graphName=*/"test_graph",
157157
/*filename=*/"test_file.txt",
158158
/*remove=*/true));
159-
REQUIRE(isOk(cacheFile.write("test data")));
159+
FUSILLI_REQUIRE_OK(cacheFile.write("test data"));
160160

161161
// Now open the existing file.
162162
CacheFile opened =

0 commit comments

Comments
 (0)