Fix SbomValidator.ValidateSbomAsync false positive when outputPath is directory

src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs

@@ -96,6 +96,11 @@ public interface IRecorder
 
     public IList<FileValidationResult> Errors { get; }
 
+    /// <summary>
+    /// Gets the list of exceptions that were recorded during the execution of the SBOM tool.
+    /// </summary>
+    public IList<Exception> Exceptions { get; }
+
     /// <summary>
     /// Record telemetry for an AggregationSource.
     /// </summary>

src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs

@@ -122,6 +122,8 @@ public async Task LogException(Exception exception)
 
     public IList<FileValidationResult> Errors => errors;
 
+    public IList<Exception> Exceptions => exceptions;
+
     /// <summary>
     /// Start recording the duration of exeuction of the given event.
     /// </summary>

src/Microsoft.Sbom.Api/SbomValidator.cs

@@ -104,7 +104,8 @@ public async Task<SbomValidationResult> ValidateSbomAsync(
         await recorder.FinalizeAndLogTelemetryAsync();
 
         var errors = recorder.Errors.Select(error => error.ToEntityError()).ToList();
-        return new SbomValidationResult(!errors.Any(), errors);
+        var hasExceptions = recorder.Exceptions.Any();
+        return new SbomValidationResult(!errors.Any() && !hasExceptions, errors);
     }
 
     private InputConfiguration ValidateConfig(InputConfiguration config)

test/Microsoft.Sbom.Api.Tests/Output/Telemetry/TelemetryRecorderTests.cs

@@ -91,4 +91,32 @@ public void RecordAggregationSource_Duplicate_Throws()
             telemetryRecorder.RecordAggregationSource(testKey, testPackageCount, testRelationshipCount);
         });
     }
+
+    [TestMethod]
+    public void TelemetryRecorder_Exceptions_Property_ReturnsRecordedExceptions()
+    {
+        var telemetryRecorder = new TelemetryRecorder(fileSystemUtilsMock.Object, configMock.Object, loggerMock.Object);
+        var testException1 = new InvalidOperationException("Test exception 1");
+        var testException2 = new ArgumentException("Test exception 2");
+
+        Assert.AreEqual(0, telemetryRecorder.Exceptions.Count);
+
+        telemetryRecorder.RecordException(testException1);
+        telemetryRecorder.RecordException(testException2);
+
+        Assert.AreEqual(2, telemetryRecorder.Exceptions.Count);
+        Assert.IsTrue(telemetryRecorder.Exceptions.Contains(testException1));
+        Assert.IsTrue(telemetryRecorder.Exceptions.Contains(testException2));
+    }
+
+    [TestMethod]
+    public void TelemetryRecorder_RecordException_NullException_Throws()
+    {
+        var telemetryRecorder = new TelemetryRecorder(fileSystemUtilsMock.Object, configMock.Object, loggerMock.Object);
+
+        Assert.ThrowsException<ArgumentNullException>(() =>
+        {
+            telemetryRecorder.RecordException(null);
+        });
+    }
 }

test/Microsoft.Sbom.Api.Tests/SbomValidatorTests.cs

@@ -0,0 +1,225 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Sbom.Api.Entities;
+using Microsoft.Sbom.Api.Output.Telemetry;
+using Microsoft.Sbom.Api.Workflows;
+using Microsoft.Sbom.Common;
+using Microsoft.Sbom.Common.Config;
+using Microsoft.Sbom.Common.Config.Validators;
+using Microsoft.Sbom.Contracts;
+using Microsoft.Sbom.Extensions;
+using Microsoft.Sbom.Extensions.Entities;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using Constants = Microsoft.Sbom.Api.Utils.Constants;
+
+namespace Microsoft.Sbom.Api.Tests;
+
+[TestClass]
+public class SbomValidatorTests
+{
+    private Mock<IWorkflow<SbomParserBasedValidationWorkflow>> workflowMock;
+    private Mock<IRecorder> recorderMock;
+    private Mock<IEnumerable<ConfigValidator>> configValidatorsMock;
+    private Mock<IConfiguration> configurationMock;
+    private Mock<ISbomConfigProvider> sbomConfigProviderMock;
+    private Mock<IFileSystemUtils> fileSystemUtilsMock;
+    private Mock<ISbomConfig> sbomConfigMock;
+
+    // Common test data
+    private readonly string buildDropPath = "/test/drop";
+    private readonly string outputPathFile = "/test/output.json";
+    private readonly string outputPathDirectory = "/test/output";
+    private readonly List<SbomSpecification> specifications = new List<SbomSpecification> { new SbomSpecification("SPDX", "2.2") };
+    private readonly string manifestDirPath = "/test/manifest";
+    private readonly ManifestInfo manifestInfo = Constants.TestManifestInfo;
+    private readonly string manifestJsonPath = "/test/manifest/manifest.json";
+
+    [TestInitialize]
+    public void Init()
+    {
+        workflowMock = new Mock<IWorkflow<SbomParserBasedValidationWorkflow>>(MockBehavior.Strict);
+        recorderMock = new Mock<IRecorder>(MockBehavior.Strict);
+        configValidatorsMock = new Mock<IEnumerable<ConfigValidator>>(MockBehavior.Strict);
+        configurationMock = new Mock<IConfiguration>(MockBehavior.Strict);
+        sbomConfigProviderMock = new Mock<ISbomConfigProvider>(MockBehavior.Strict);
+        fileSystemUtilsMock = new Mock<IFileSystemUtils>(MockBehavior.Strict);
+        sbomConfigMock = new Mock<ISbomConfig>(MockBehavior.Strict);
+    }
+
+    [TestCleanup]
+    public void AfterEachTest()
+    {
+        workflowMock.VerifyAll();
+        recorderMock.VerifyAll();
+        configValidatorsMock.VerifyAll();
+        configurationMock.VerifyAll();
+        sbomConfigProviderMock.VerifyAll();
+        fileSystemUtilsMock.VerifyAll();
+        sbomConfigMock.VerifyAll();
+    }
+
+    [TestMethod]
+    public async Task ValidateSbomAsync_WithNoErrorsAndNoExceptions_ReturnsTrue()
+    {
+        var errors = new List<FileValidationResult>();
+        var exceptions = new List<Exception>();
+
+        configValidatorsMock.Setup(cv => cv.GetEnumerator()).Returns(new List<ConfigValidator>().GetEnumerator());
+
+        configurationMock.Setup(c => c.ManifestInfo).Returns(new ConfigurationSetting<IList<ManifestInfo>>
+        {
+            Value = new List<ManifestInfo> { manifestInfo }
+        });
+
+        sbomConfigProviderMock.Setup(scp => scp.Get(manifestInfo)).Returns(sbomConfigMock.Object);
+        sbomConfigMock.Setup(sc => sc.ManifestJsonFilePath).Returns(manifestJsonPath);
+
+        fileSystemUtilsMock.Setup(fs => fs.FileExists(manifestJsonPath)).Returns(true);
+        workflowMock.Setup(w => w.RunAsync()).ReturnsAsync(true);
+
+        recorderMock.Setup(r => r.FinalizeAndLogTelemetryAsync()).Returns(Task.CompletedTask);
+        recorderMock.Setup(r => r.Errors).Returns(errors);
+        recorderMock.Setup(r => r.Exceptions).Returns(exceptions);
+
+        var validator = new SbomValidator(
+            workflowMock.Object,
+            recorderMock.Object,
+            configValidatorsMock.Object,
+            configurationMock.Object,
+            sbomConfigProviderMock.Object,
+            fileSystemUtilsMock.Object);
+
+        var result = await validator.ValidateSbomAsync(buildDropPath, outputPathFile, specifications, manifestDirPath);
+
+        Assert.IsTrue(result.IsSuccess);
+        Assert.AreEqual(0, result.Errors.Count);
+    }
+
+    [TestMethod]
+    public async Task ValidateSbomAsync_WithErrorsButNoExceptions_ReturnsFalse()
+    {
+        var errors = new List<FileValidationResult>
+        {
+            new FileValidationResult { ErrorType = ErrorType.MissingFile, Path = "/test/missing.txt" }
+        };
+        var exceptions = new List<Exception>();
+
+        configValidatorsMock.Setup(cv => cv.GetEnumerator()).Returns(new List<ConfigValidator>().GetEnumerator());
+
+        configurationMock.Setup(c => c.ManifestInfo).Returns(new ConfigurationSetting<IList<ManifestInfo>>
+        {
+            Value = new List<ManifestInfo> { manifestInfo }
+        });
+
+        sbomConfigProviderMock.Setup(scp => scp.Get(manifestInfo)).Returns(sbomConfigMock.Object);
+        sbomConfigMock.Setup(sc => sc.ManifestJsonFilePath).Returns(manifestJsonPath);
+
+        fileSystemUtilsMock.Setup(fs => fs.FileExists(manifestJsonPath)).Returns(true);
+        workflowMock.Setup(w => w.RunAsync()).ReturnsAsync(true);
+
+        recorderMock.Setup(r => r.FinalizeAndLogTelemetryAsync()).Returns(Task.CompletedTask);
+        recorderMock.Setup(r => r.Errors).Returns(errors);
+        recorderMock.Setup(r => r.Exceptions).Returns(exceptions);
+
+        var validator = new SbomValidator(
+            workflowMock.Object,
+            recorderMock.Object,
+            configValidatorsMock.Object,
+            configurationMock.Object,
+            sbomConfigProviderMock.Object,
+            fileSystemUtilsMock.Object);
+
+        var result = await validator.ValidateSbomAsync(buildDropPath, outputPathFile, specifications, manifestDirPath);
+
+        Assert.IsFalse(result.IsSuccess);
+        Assert.AreEqual(1, result.Errors.Count);
+    }
+
+    [TestMethod]
+    public async Task ValidateSbomAsync_WithNoErrorsButWithExceptions_ReturnsFalse()
+    {
+        var errors = new List<FileValidationResult>();
+        var exceptions = new List<Exception>
+        {
+            new InvalidOperationException("Cannot write to directory path")
+        };
+
+        configValidatorsMock.Setup(cv => cv.GetEnumerator()).Returns(new List<ConfigValidator>().GetEnumerator());
+
+        configurationMock.Setup(c => c.ManifestInfo).Returns(new ConfigurationSetting<IList<ManifestInfo>>
+        {
+            Value = new List<ManifestInfo> { manifestInfo }
+        });
+
+        sbomConfigProviderMock.Setup(scp => scp.Get(manifestInfo)).Returns(sbomConfigMock.Object);
+        sbomConfigMock.Setup(sc => sc.ManifestJsonFilePath).Returns(manifestJsonPath);
+
+        fileSystemUtilsMock.Setup(fs => fs.FileExists(manifestJsonPath)).Returns(true);
+        workflowMock.Setup(w => w.RunAsync()).ReturnsAsync(true);
+
+        recorderMock.Setup(r => r.FinalizeAndLogTelemetryAsync()).Returns(Task.CompletedTask);
+        recorderMock.Setup(r => r.Errors).Returns(errors);
+        recorderMock.Setup(r => r.Exceptions).Returns(exceptions);
+
+        var validator = new SbomValidator(
+            workflowMock.Object,
+            recorderMock.Object,
+            configValidatorsMock.Object,
+            configurationMock.Object,
+            sbomConfigProviderMock.Object,
+            fileSystemUtilsMock.Object);
+
+        var result = await validator.ValidateSbomAsync(buildDropPath, outputPathDirectory, specifications, manifestDirPath);
+
+        Assert.IsFalse(result.IsSuccess);
+        Assert.AreEqual(0, result.Errors.Count); // No validation errors, but should still fail due to exception
+    }
+
+    [TestMethod]
+    public async Task ValidateSbomAsync_WithBothErrorsAndExceptions_ReturnsFalse()
+    {
+        var errors = new List<FileValidationResult>
+        {
+            new FileValidationResult { ErrorType = ErrorType.MissingFile, Path = "/test/missing.txt" }
+        };
+        var exceptions = new List<Exception>
+        {
+            new InvalidOperationException("Cannot write to directory path")
+        };
+
+        configValidatorsMock.Setup(cv => cv.GetEnumerator()).Returns(new List<ConfigValidator>().GetEnumerator());
+
+        configurationMock.Setup(c => c.ManifestInfo).Returns(new ConfigurationSetting<IList<ManifestInfo>>
+        {
+            Value = new List<ManifestInfo> { manifestInfo }
+        });
+
+        sbomConfigProviderMock.Setup(scp => scp.Get(manifestInfo)).Returns(sbomConfigMock.Object);
+        sbomConfigMock.Setup(sc => sc.ManifestJsonFilePath).Returns(manifestJsonPath);
+
+        fileSystemUtilsMock.Setup(fs => fs.FileExists(manifestJsonPath)).Returns(true);
+        workflowMock.Setup(w => w.RunAsync()).ReturnsAsync(true);
+
+        recorderMock.Setup(r => r.FinalizeAndLogTelemetryAsync()).Returns(Task.CompletedTask);
+        recorderMock.Setup(r => r.Errors).Returns(errors);
+        recorderMock.Setup(r => r.Exceptions).Returns(exceptions);
+
+        var validator = new SbomValidator(
+            workflowMock.Object,
+            recorderMock.Object,
+            configValidatorsMock.Object,
+            configurationMock.Object,
+            sbomConfigProviderMock.Object,
+            fileSystemUtilsMock.Object);
+
+        var result = await validator.ValidateSbomAsync(buildDropPath, outputPathDirectory, specifications, manifestDirPath);
+
+        Assert.IsFalse(result.IsSuccess);
+        Assert.AreEqual(1, result.Errors.Count);
+    }
+}

test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_4_0/InterfaceConcretionTests.cs

@@ -128,6 +128,8 @@ private class PinnedIRecorder : IRecorder
     {
         public IList<FileValidationResult> Errors => throw new NotImplementedException();
 
+        public IList<Exception> Exceptions => throw new NotImplementedException();
+
         public void AddResult(string propertyName, string value) => throw new NotImplementedException();
         public void AddToTotalCountOfLicenses(int count) => throw new NotImplementedException();
         public void AddToTotalNumberOfPackageDetailsEntries(int count) => throw new NotImplementedException();