Skip to content

Commit fac944b

Browse files
mehrandvdmehrandvd
authored
Merge pull request #41 from mehrandvd/copilot/fix-37
Add MSTest support with comprehensive documentation and demo project
2 parents dbcfb39 + b8bb30c commit fac944b

File tree

10 files changed

+641
-0
lines changed

10 files changed

+641
-0
lines changed

‎README.md‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,51 @@ That's it! ✨ skUnit handles the conversation, calls your AI, and verifies the
5959

6060
> **💡 Pro Tip:** For better alignment with Microsoft Extensions AI (MEAI), you can use `[ASSISTANT]` instead of `[AGENT]` - both work identically!
6161
62+
## 🧪 Works with Any Test Framework
63+
64+
skUnit is completely test-framework agnostic! Here's the same test with different frameworks:
65+
66+
### xUnit
67+
```csharp
68+
[Fact]
69+
public async Task TestGreeting()
70+
{
71+
var markdown = File.ReadAllText("greeting.md");
72+
var scenarios = ChatScenario.LoadFromText(markdown);
73+
74+
await ScenarioAssert.PassAsync(scenarios, myChatClient);
75+
}
76+
```
77+
78+
### MSTest
79+
```csharp
80+
[TestMethod]
81+
public async Task TestGreeting()
82+
{
83+
var scenarioAssert = new ScenarioAssert(myChatClient, TestContext.WriteLine);
84+
85+
var scenarios = await ChatScenario.LoadFromResourceAsync(
86+
"MyProject.Scenarios.greeting.md",
87+
typeof(MyTestClass).Assembly);
88+
89+
await scenarioAssert.PassAsync(scenarios);
90+
}
91+
```
92+
93+
### NUnit
94+
```csharp
95+
[Test]
96+
public async Task TestGreeting()
97+
{
98+
var markdown = File.ReadAllText("greeting.md");
99+
var scenarios = ChatScenario.LoadFromText(markdown);
100+
101+
await ScenarioAssert.PassAsync(scenarios, myChatClient);
102+
}
103+
```
104+
105+
The core difference is just the logging integration - use `TestContext.WriteLine` for MSTest, `ITestOutputHelper.WriteLine` for xUnit, or `TestContext.WriteLine` for NUnit.
106+
62107
## 🎯 Key Features
63108

64109
### 1. Start Simple: Basic Chat Scenarios
@@ -271,6 +316,7 @@ var chatClient = new ChatClientBuilder(baseChatClient)
271316

272317
- **[Chat Scenario Spec](docs/chat-scenario-spec.md)** - Complete guide to writing chat scenarios
273318
- **[CHECK Statement Spec](docs/check-statements-spec.md)** - All available assertion types
319+
- **[Test Framework Integration](docs/test-framework-integration.md)** - How to use skUnit with xUnit, MSTest, NUnit, and more
274320
- **[MCP Testing Guide](docs/mcp-testing-guide.md)** - How to test Model Context Protocol servers
275321
- **[Multi-Modal Support](docs/multi-modal-support.md)** - Working with images and other media
276322
- **[Scenario Run Options](docs/scenario-run-options.md)** - Mitigate hallucinations with multi-run success thresholds

‎demos/Demo.MSTest/ChatScenarioTests.cs‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
using Azure.AI.OpenAI;
2+
using Microsoft.Extensions.AI;
3+
using Microsoft.Extensions.Configuration;
4+
using skUnit;
5+
using skUnit.Scenarios;
6+
7+
namespace Demo.MSTest;
8+
9+
[TestClass]
10+
public class ChatScenarioTests
11+
{
12+
private static IChatClient _chatClient = null!;
13+
private static ScenarioAssert ScenarioAssert { get; set; } = null!;
14+
15+
public TestContext TestContext { get; set; } = null!;
16+
17+
[ClassInitialize]
18+
public static void ClassInitialize(TestContext context)
19+
{
20+
var configuration = new ConfigurationBuilder()
21+
.AddUserSecrets<ChatScenarioTests>()
22+
.Build();
23+
24+
var apiKey = configuration["AzureOpenAI_ApiKey"]
25+
?? throw new InvalidOperationException("AzureOpenAI_ApiKey not found in user secrets");
26+
var endpoint = configuration["AzureOpenAI_Endpoint"]
27+
?? throw new InvalidOperationException("AzureOpenAI_Endpoint not found in user secrets");
28+
var deployment = configuration["AzureOpenAI_Deployment"]
29+
?? throw new InvalidOperationException("AzureOpenAI_Deployment not found in user secrets");
30+
31+
_chatClient = new AzureOpenAIClient(new Uri(endpoint), new System.ClientModel.ApiKeyCredential(apiKey))
32+
.GetChatClient(deployment)
33+
.AsIChatClient();
34+
35+
// This uses Console.WriteLine for class-level initialization
36+
ScenarioAssert = new ScenarioAssert(_chatClient, context.WriteLine);
37+
}
38+
39+
[TestMethod]
40+
public async Task SimpleGreeting_ShouldPass()
41+
{
42+
// Create per-test instance with TestContext logging
43+
44+
var scenarios = await ChatScenario.LoadFromResourceAsync(
45+
"Demo.MSTest.Scenarios.SimpleGreeting.md",
46+
typeof(ChatScenarioTests).Assembly);
47+
48+
await ScenarioAssert.PassAsync(scenarios, _chatClient);
49+
}
50+
51+
[TestMethod]
52+
public async Task GetCurrentTimeChat_ShouldPass()
53+
{
54+
var scenarios = await ChatScenario.LoadFromResourceAsync(
55+
"Demo.MSTest.Scenarios.GetCurrentTimeChat.md",
56+
typeof(ChatScenarioTests).Assembly);
57+
58+
await ScenarioAssert.PassAsync(scenarios, _chatClient);
59+
}
60+
61+
[TestMethod]
62+
public async Task JsonUserInfo_ShouldPass()
63+
{
64+
var scenarios = await ChatScenario.LoadFromResourceAsync(
65+
"Demo.MSTest.Scenarios.JsonUserInfo.md",
66+
typeof(ChatScenarioTests).Assembly);
67+
68+
await ScenarioAssert.PassAsync(scenarios, _chatClient);
69+
}
70+
71+
[DataTestMethod]
72+
[DataRow("SimpleGreeting")]
73+
[DataRow("GetCurrentTimeChat")]
74+
[DataRow("JsonUserInfo")]
75+
public async Task ScenarioMatrix_ShouldPass(string scenarioName)
76+
{
77+
var scenarios = await ChatScenario.LoadFromResourceAsync(
78+
$"Demo.MSTest.Scenarios.{scenarioName}.md",
79+
typeof(ChatScenarioTests).Assembly);
80+
81+
await ScenarioAssert.PassAsync(scenarios, _chatClient);
82+
}
83+
}

‎demos/Demo.MSTest/Demo.MSTest.csproj‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net8.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
8+
<IsPackable>false</IsPackable>
9+
<IsTestProject>true</IsTestProject>
10+
<UserSecretsId>3207994b-8ef0-4e63-b359-cefc9b9fa5be</UserSecretsId>
11+
</PropertyGroup>
12+
13+
<ItemGroup>
14+
<EmbeddedResource Include="Scenarios\*.md">
15+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
16+
</EmbeddedResource>
17+
</ItemGroup>
18+
19+
<ItemGroup>
20+
<PackageReference Include="coverlet.collector" Version="6.0.4" />
21+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
22+
<PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
23+
<PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
24+
<PackageReference Include="Azure.AI.OpenAI" Version="2.3.0-beta.2" />
25+
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.5" />
26+
<PackageReference Include="Microsoft.Extensions.AI" Version="9.8.0" />
27+
<PackageReference Include="skUnit" Version="0.56.0-beta" />
28+
</ItemGroup>
29+
30+
<ItemGroup>
31+
<Using Include="Microsoft.VisualStudio.TestTools.UnitTesting" />
32+
</ItemGroup>
33+
34+
</Project>

‎demos/Demo.MSTest/Demo.MSTest.sln‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
Microsoft Visual Studio Solution File, Format Version 12.00
2+
# Visual Studio Version 17
3+
VisualStudioVersion = 17.0.31903.59
4+
MinimumVisualStudioVersion = 10.0.40219.1
5+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Demo.MSTest", "Demo.MSTest.csproj", "{A1B2C3D4-E5F6-7890-ABCD-123456789ABC}"
6+
EndProject
7+
Global
8+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
9+
Debug|Any CPU = Debug|Any CPU
10+
Release|Any CPU = Release|Any CPU
11+
EndGlobalSection
12+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
13+
{A1B2C3D4-E5F6-7890-ABCD-123456789ABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
14+
{A1B2C3D4-E5F6-7890-ABCD-123456789ABC}.Debug|Any CPU.Build.0 = Debug|Any CPU
15+
{A1B2C3D4-E5F6-7890-ABCD-123456789ABC}.Release|Any CPU.ActiveCfg = Release|Any CPU
16+
{A1B2C3D4-E5F6-7890-ABCD-123456789ABC}.Release|Any CPU.Build.0 = Release|Any CPU
17+
EndGlobalSection
18+
EndGlobal

‎demos/Demo.MSTest/README.md‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# Demo.MSTest - MSTest Integration Example
2+
3+
This demo shows how to use skUnit with MSTest framework for semantic testing of chat applications.
4+
5+
## Key Features Demonstrated
6+
7+
- **MSTest Attributes**: Using `[TestClass]`, `[TestMethod]`, `[DataTestMethod]`, and `[ClassInitialize]`
8+
- **TestContext Integration**: Proper logging integration using `TestContext.WriteLine`
9+
- **Data-Driven Tests**: Using `[DataRow]` for parameterized tests
10+
- **Scenario Loading**: Loading test scenarios from embedded markdown files
11+
- **Function-Style Testing**: Alternative approach using `getAnswerFunc` delegate
12+
13+
## MSTest vs xUnit Key Differences
14+
15+
### Test Class Setup
16+
```csharp
17+
// MSTest
18+
[TestClass]
19+
public class ChatScenarioTests
20+
{
21+
public TestContext TestContext { get; set; } = null!;
22+
23+
[ClassInitialize]
24+
public static void ClassInitialize(TestContext context) { }
25+
26+
[TestMethod]
27+
public async Task MyTest() { }
28+
}
29+
30+
// xUnit equivalent
31+
public class ChatScenarioTests
32+
{
33+
public ChatScenarioTests(ITestOutputHelper output) { }
34+
35+
[Fact]
36+
public async Task MyTest() { }
37+
}
38+
```
39+
40+
### Logging Integration
41+
```csharp
42+
// MSTest
43+
var scenarioAssert = new ScenarioAssert(_chatClient, TestContext.WriteLine);
44+
45+
// xUnit
46+
var scenarioAssert = new ScenarioAssert(_chatClient, output.WriteLine);
47+
```
48+
49+
### Data-Driven Tests
50+
```csharp
51+
// MSTest
52+
[DataTestMethod]
53+
[DataRow("Scenario1")]
54+
[DataRow("Scenario2")]
55+
public async Task TestScenarios(string scenarioName) { }
56+
57+
// xUnit
58+
[Theory]
59+
[InlineData("Scenario1")]
60+
[InlineData("Scenario2")]
61+
public async Task TestScenarios(string scenarioName) { }
62+
```
63+
64+
## Running the Demo
65+
66+
1. **Configure Azure OpenAI secrets**:
67+
```bash
68+
cd Demo.MSTest
69+
dotnet user-secrets set "AzureOpenAI_ApiKey" "your-key"
70+
dotnet user-secrets set "AzureOpenAI_Endpoint" "https://your-endpoint.openai.azure.com/"
71+
dotnet user-secrets set "AzureOpenAI_Deployment" "your-deployment-name"
72+
```
73+
74+
2. **Build and test**:
75+
```bash
76+
dotnet restore Demo.MSTest.sln
77+
dotnet build Demo.MSTest.sln --no-restore
78+
dotnet test Demo.MSTest.sln --no-build
79+
```
80+
81+
## Scenarios Included
82+
83+
- **SimpleGreeting.md**: Basic semantic condition testing
84+
- **GetCurrentTimeChat.md**: Multi-turn conversation validation
85+
- **JsonUserInfo.md**: JSON structure and content validation
86+
87+
This demo proves that skUnit works seamlessly with MSTest - the core library is completely test-framework agnostic!

‎demos/Demo.MSTest/Scenarios/GetCurrentTimeChat.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# SCENARIO Current Time Request
2+
3+
## [USER]
4+
What time is it?
5+
6+
## [AGENT]
7+
I don't have access to real-time information, but I can help you find the current time.
8+
9+
### CHECK SemanticCondition
10+
It acknowledges the time request and provides a helpful response

‎demos/Demo.MSTest/Scenarios/JsonUserInfo.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# SCENARIO JSON User Info Response
2+
3+
## [USER]
4+
Give me user info as JSON
5+
6+
## [AGENT]
7+
{"name": "John", "age": 30, "city": "New York"}
8+
9+
### CHECK JsonCheck
10+
{
11+
"name": ["NotEmpty"],
12+
"age": ["GreaterThan", 0],
13+
"city": ["SemanticCondition", "It's a real city name"]
14+
}

‎demos/Demo.MSTest/Scenarios/SimpleGreeting.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# SCENARIO Simple Greeting
2+
3+
## [USER]
4+
Hello!
5+
6+
## [AGENT]
7+
Hi there! How can I help you today?
8+
9+
### CHECK SemanticCondition
10+
It's a friendly greeting response

‎demos/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@ Complex multi-scenario testing for e-commerce chat applications.
2929
- **Features**: Multi-turn conversations, complex business logic testing
3030
- **Key Learning**: How to test sophisticated chat workflows with multiple scenarios
3131

32+
## 🧪 Demo.MSTest
33+
**MSTest Framework Integration**
34+
35+
Shows how to use skUnit with MSTest framework for semantic testing.
36+
37+
- **Location**: `Demo.MSTest/`
38+
- **Features**: MSTest attributes, TestContext integration, data-driven tests
39+
- **Key Learning**: How to use skUnit with MSTest instead of xUnit (proving framework agnosticism)
40+
3241
## Running the Demos
3342

3443
Each demo is a complete .NET project that you can run independently:

0 commit comments

Comments
 (0)