feat: Add filter support for TestAdapter

samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj

@@ -7,6 +7,8 @@
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net462;net8.0</TargetFrameworks>
     <GenerateProgramFile>false</GenerateProgramFile>
+    <!-- Disable parallel tests between TargetFrameworks -->
+    <TestTfmsInParallel>false</TestTfmsInParallel>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="Program.fs" />

samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj

@@ -13,6 +13,8 @@
     <NoWarn>$(NoWarn);CA1018;CA5351;CA1825</NoWarn>
     <!-- Disable entry point generation as this project has it's own entry point -->
     <GenerateProgramFile>false</GenerateProgramFile>
+    <!-- Disable parallel tests between TargetFrameworks -->
+    <TestTfmsInParallel>false</TestTfmsInParallel>
   </PropertyGroup>
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
     <Reference Include="System.Reflection" />

src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs

@@ -0,0 +1,57 @@
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
+using System.Diagnostics;
+using System.IO;
+
+namespace BenchmarkDotNet.TestAdapter;
+
+internal class LoggerHelper
+{
+    public LoggerHelper(IMessageLogger logger, Stopwatch stopwatch)
+    {
+        InnerLogger = logger;
+        Stopwatch = stopwatch;
+    }
+
+    public IMessageLogger InnerLogger { get; private set; }
+
+    public Stopwatch Stopwatch { get; private set; }
+
+    public void Log(string format, params object[] args)
+    {
+        SendMessage(TestMessageLevel.Informational, null, string.Format(format, args));
+    }
+
+    public void LogWithSource(string source, string format, params object[] args)
+    {
+        SendMessage(TestMessageLevel.Informational, source, string.Format(format, args));
+    }
+
+    public void LogError(string format, params object[] args)
+    {
+        SendMessage(TestMessageLevel.Error, null, string.Format(format, args));
+    }
+
+    public void LogErrorWithSource(string source, string format, params object[] args)
+    {
+        SendMessage(TestMessageLevel.Error, source, string.Format(format, args));
+    }
+
+    public void LogWarning(string format, params object[] args)
+    {
+        SendMessage(TestMessageLevel.Warning, null, string.Format(format, args));
+    }
+
+    public void LogWarningWithSource(string source, string format, params object[] args)
+    {
+        SendMessage(TestMessageLevel.Warning, source, string.Format(format, args));
+    }
+
+    private void SendMessage(TestMessageLevel level, string? assemblyName, string message)
+    {
+        var assemblyText = assemblyName == null
+            ? "" :
+            $"{Path.GetFileNameWithoutExtension(assemblyName)}: ";
+
+        InnerLogger.SendMessage(level, $"[BenchmarkDotNet {Stopwatch.Elapsed:hh\\:mm\\:ss\\.ff}] {assemblyText}{message}");
+    }
+}

src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs

@@ -0,0 +1,166 @@
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+
+namespace BenchmarkDotNet.TestAdapter;
+
+internal class TestCaseFilter
+{
+    private const string DisplayNameString = "DisplayName";
+    private const string FullyQualifiedNameString = "FullyQualifiedName";
+
+    private readonly HashSet<string> knownTraits;
+    private List<string> supportedPropertyNames;
+    private readonly ITestCaseFilterExpression? filterExpression;
+    private readonly bool successfullyGotFilter;
+    private readonly bool isDiscovery;
+
+    public TestCaseFilter(IDiscoveryContext discoveryContext, LoggerHelper logger)
+    {
+        // Traits are not known at discovery time because we load them from benchmarks
+        isDiscovery = true;
+        knownTraits = [];
+        supportedPropertyNames = GetSupportedPropertyNames();
+        successfullyGotFilter = GetTestCaseFilterExpressionFromDiscoveryContext(discoveryContext, logger, out filterExpression);
+    }
+
+    public TestCaseFilter(IRunContext runContext, LoggerHelper logger, string assemblyFileName, HashSet<string> knownTraits)
+    {
+        this.knownTraits = knownTraits;
+        supportedPropertyNames = GetSupportedPropertyNames();
+        successfullyGotFilter = GetTestCaseFilterExpression(runContext, logger, assemblyFileName, out filterExpression);
+    }
+
+    public string GetTestCaseFilterValue()
+    {
+        return successfullyGotFilter
+            ? filterExpression?.TestCaseFilterValue ?? ""
+            : "";
+    }
+
+    public bool MatchTestCase(TestCase testCase)
+    {
+        if (!successfullyGotFilter)
+        {
+            // Had an error while getting filter, match no testcase to ensure discovered test list is empty
+            return false;
+        }
+        else if (filterExpression == null)
+        {
+            // No filter specified, keep every testcase
+            return true;
+        }
+
+        return filterExpression.MatchTestCase(testCase, p => PropertyProvider(testCase, p));
+    }
+
+    public object? PropertyProvider(TestCase testCase, string name)
+    {
+        // Traits filtering
+        if (isDiscovery || knownTraits.Contains(name))
+        {
+            var result = new List<string>();
+
+            foreach (var trait in GetTraits(testCase))
+                if (string.Equals(trait.Key, name, StringComparison.OrdinalIgnoreCase))
+                    result.Add(trait.Value);
+
+            if (result.Count > 0)
+                return result.ToArray();
+        }
+
+        // Property filtering
+        switch (name.ToLowerInvariant())
+        {
+            // FullyQualifiedName
+            case "fullyqualifiedname":
+                return testCase.FullyQualifiedName;
+            // DisplayName
+            case "displayname":
+                return testCase.DisplayName;
+            default:
+                return null;
+        }
+    }
+
+    private bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression? filter)
+    {
+        filter = null;
+
+        try
+        {
+            filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!);
+            return true;
+        }
+        catch (TestPlatformFormatException e)
+        {
+            logger.LogWarning("{0}: Exception filtering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.Message);
+            return false;
+        }
+    }
+
+    private bool GetTestCaseFilterExpressionFromDiscoveryContext(IDiscoveryContext discoveryContext, LoggerHelper logger, out ITestCaseFilterExpression? filter)
+    {
+        filter = null;
+
+        if (discoveryContext is IRunContext runContext)
+        {
+            try
+            {
+                filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!);
+                return true;
+            }
+            catch (TestPlatformException e)
+            {
+                logger.LogWarning("Exception filtering tests: {0}", e.Message);
+                return false;
+            }
+        }
+        else
+        {
+            try
+            {
+                // GetTestCaseFilter is present on DiscoveryContext but not in IDiscoveryContext interface
+                var method = discoveryContext.GetType().GetRuntimeMethod("GetTestCaseFilter", [typeof(IEnumerable<string>), typeof(Func<string, TestProperty>)]);
+                filter = (ITestCaseFilterExpression)method?.Invoke(discoveryContext, [supportedPropertyNames, null])!;
+
+                return true;
+            }
+            catch (TargetInvocationException e)
+            {
+                if (e?.InnerException is TestPlatformException ex)
+                {
+                    logger.LogWarning("Exception filtering tests: {0}", ex.InnerException.Message ?? "");
+                    return false;
+                }
+
+                throw e!.InnerException;
+            }
+        }
+    }
+
+    private List<string> GetSupportedPropertyNames()
+    {
+        // Returns the set of well-known property names usually used with the Test Plugins (Used Test Traits + DisplayName + FullyQualifiedName)
+        if (supportedPropertyNames == null)
+        {
+            supportedPropertyNames = knownTraits.ToList();
+            supportedPropertyNames.Add(DisplayNameString);
+            supportedPropertyNames.Add(FullyQualifiedNameString);
+        }
+
+        return supportedPropertyNames;
+    }
+
+    private static IEnumerable<KeyValuePair<string, string>> GetTraits(TestCase testCase)
+    {
+        var traitProperty = TestProperty.Find("TestObject.Traits");
+        return traitProperty != null
+            ? testCase.GetPropertyValue(traitProperty, Array.Empty<KeyValuePair<string, string>>())
+            : [];
+    }
+}

src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs

@@ -4,6 +4,7 @@
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Reflection;
@@ -42,11 +43,18 @@ public void DiscoverTests(
             IMessageLogger logger,
             ITestCaseDiscoverySink discoverySink)
         {
+            var stopwatch = Stopwatch.StartNew();
+            var loggerHelper = new LoggerHelper(logger, stopwatch);
+            var testCaseFilter = new TestCaseFilter(discoveryContext, loggerHelper);
+
             foreach (var source in sources)
             {
                 ValidateSourceIsAssemblyOrThrow(source);
                 foreach (var testCase in GetVsTestCasesFromAssembly(source, logger))
                 {
+                    if (!testCaseFilter.MatchTestCase(testCase))
+                        continue;
+
                     discoverySink.SendTestCase(testCase);
                 }
             }
@@ -67,14 +75,19 @@ public void RunTests(IEnumerable<TestCase>? tests, IRunContext? runContext, IFra
 
             cts ??= new CancellationTokenSource();
 
+            var stopwatch = Stopwatch.StartNew();
+            var logger = new LoggerHelper(frameworkHandle, stopwatch);
+
             foreach (var testsPerAssembly in tests.GroupBy(t => t.Source))
+            {
                 RunBenchmarks(testsPerAssembly.Key, frameworkHandle, testsPerAssembly);
+            }
 
             cts = null;
         }
 
         /// <summary>
-        /// Runs all benchmarks in the given set of sources (assemblies).
+        /// Runs all/filtered benchmarks in the given set of sources (assemblies).
         /// </summary>
         /// <param name="sources">The assemblies to run.</param>
         /// <param name="runContext">A context that the run is performed in.</param>
@@ -88,8 +101,31 @@ public void RunTests(IEnumerable<string>? sources, IRunContext? runContext, IFra
 
             cts ??= new CancellationTokenSource();
 
+            var stopwatch = Stopwatch.StartNew();
+            var logger = new LoggerHelper(frameworkHandle, stopwatch);
+
             foreach (var source in sources)
-                RunBenchmarks(source, frameworkHandle);
+            {
+                var filter = new TestCaseFilter(runContext!, logger, source, ["Category"]);
+                if (filter.GetTestCaseFilterValue() != "")
+                {
+                    var discoveredBenchmarks = GetVsTestCasesFromAssembly(source, frameworkHandle);
+                    var filteredTestCases = discoveredBenchmarks.Where(x => filter.MatchTestCase(x))
+                                                                .ToArray();
+
+                    if (filteredTestCases.Length == 0)
+                        continue;
+
+                    // Run filtered tests.
+                    RunBenchmarks(source, frameworkHandle, filteredTestCases);
+                }
+                else
+                {
+                    // Run all benchmarks
+                    RunBenchmarks(source, frameworkHandle);
+                }
+            }
+
 
             cts = null;
         }

src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props

@@ -1,6 +1,8 @@
 <Project InitialTargets="GenerateBDNEntryPoint" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <EntryPointSourceDirectory>$(MSBuildThisFileDirectory)..\entrypoints\</EntryPointSourceDirectory>
+    <!-- Disable parallel tests between TargetFrameworks, if it's not explicitly specified. -->
+    <TestTfmsInParallel Condition="'$(TestTfmsInParallel)' = ''">false</TestTfmsInParallel>
   </PropertyGroup>
   <!-- 
     Microsoft.NET.Test.Sdk uses a property called "GenerateProgramFile" to determine whether it generates an entry