Skip to content

Commit 0a56da6

Browse files
nohwndnohwnd
authored
Cross platform acceptance tests (#2653)
Run acceptance tests on Windows, Linux and MacOS.
1 parent 5354492 commit 0a56da6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+269
-78
lines changed

‎azure-pipelines.yml‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
displayName: 'Run script build.cmd'
1313
inputs:
1414
filename: build.cmd
15-
arguments: '-verbose -configuration $(buildConfiguration)'
15+
arguments: '-verbose -configuration $(buildConfiguration) -steps "InstallDotnet, Restore, UpdateLocalization, Build, Publish"'
1616
modifyEnvironment: false
1717
failOnStandardError: true
1818

@@ -32,7 +32,48 @@ jobs:
3232
arguments: '-verbose -configuration $(buildConfiguration)'
3333
modifyEnvironment: false
3434
failOnStandardError: true
35+
36+
- task: PublishPipelineArtifact@1
37+
inputs:
38+
targetPath: '$(Build.SourcesDirectory)\artifacts'
39+
artifact: 'testArtifacts'
40+
publishLocation: 'pipeline'
41+
42+
- job: WindowsAcceptance
43+
dependsOn: Windows
44+
timeoutInMinutes: 120
45+
pool:
46+
vmImage: 'vs2017-win2016'
47+
variables:
48+
buildConfiguration: 'Release'
49+
steps:
50+
- task: DownloadPipelineArtifact@2
51+
inputs:
52+
buildType: 'current'
53+
artifactName: 'testArtifacts'
54+
targetPath: '$(Build.SourcesDirectory)\artifacts'
55+
56+
- task: PowerShell@2
57+
inputs:
58+
targetType: 'inline'
59+
script: 'ls "$(Build.SourcesDirectory)\artifacts\" -Recurse -Dir'
60+
61+
- task: BatchScript@1
62+
displayName: 'Run script build.cmd'
63+
inputs:
64+
filename: build.cmd
65+
# build should not be needed in the end, we are copying the build task from the build folder, but we should get
66+
# it either from artifacts, if present, or from extracted package file
67+
# Adding it at the moment to avoid this error: #[error]Copy-Item : Cannot find path 'D:\a\1\s\src\Microsoft.TestPlatform.Build\bin\Release\netstandard2.0' because it does not exist.
68+
arguments: '-verbose -configuration $(buildConfiguration) -steps "InstallDotnet, Build, PrepareAcceptanceTests"'
69+
modifyEnvironment: false
70+
failOnStandardError: true
3571

72+
- task: PowerShell@2
73+
inputs:
74+
targetType: 'inline'
75+
script: 'ls "$(Build.SourcesDirectory)\artifacts\Release\packages"'
76+
3677
- task: BatchScript@1
3778
displayName: 'Run Acceptance Tests'
3879
inputs:
@@ -64,7 +105,8 @@ jobs:
64105
testResultsFiles: '**\*.trx'
65106
condition: succeededOrFailed()
66107

67-
- job:
108+
- job: OtherOSes
109+
dependsOn: Windows
68110
workspace:
69111
clean: all
70112
strategy:
@@ -81,7 +123,17 @@ jobs:
81123
buildConfiguration: 'Release'
82124
steps:
83125
- script: ./build.sh -c $(buildConfiguration)
84-
displayName: './build.sh -c $(buildConfiguration)'
85-
- script: ./test.sh -c $(buildConfiguration)
86-
displayName: './test.sh -c $(buildConfiguration)'
126+
displayName: 'Build'
127+
- script: ./test.sh -c $(buildConfiguration) -p Unit
128+
displayName: 'Unit tests'
129+
- task: DownloadPipelineArtifact@2
130+
inputs:
131+
buildType: 'current'
132+
artifactName: 'testArtifacts'
133+
targetPath: '$(Build.SourcesDirectory)/artifacts'
134+
- script: ./tools/dotnet-linux/dotnet build -c $(buildConfiguration) ./test/TestAssets/TestAssets.sln
135+
displayName: 'Build test assets'
136+
- script: ./test.sh -c $(buildConfiguration) -p Acceptance
137+
displayName: 'Acceptance tests'
138+
87139

‎global.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
{
22
"sdk": {
3-
"version": "5.0.100-rc.1.20453.7",
3+
"version": "5.0.100",
44
"rollForward": "minor",
55
"allowPrerelease": false,
66
"architecture": "x64"
77
},
88
"tools": {
9-
"dotnet": "5.0.100-rc.1.20453.7"
9+
"dotnet": "5.0.100"
1010
},
1111
"msbuild-sdks": {
1212
"Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.20570.10",

‎scripts/build.ps1‎

Lines changed: 40 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,13 @@ Param(
4545
# Build specific projects
4646
[Parameter(Mandatory=$false)]
4747
[Alias("p")]
48-
[System.String[]] $ProjectNamePatterns = @()
48+
[System.String[]] $ProjectNamePatterns = @(),
49+
50+
[Alias("f")]
51+
[Switch] $Force,
52+
53+
[Alias("s")]
54+
[String[]] $Steps = @("InstallDotnet", "Restore", "UpdateLocalization", "Build", "Publish", "PrepareAcceptanceTests")
4955
)
5056

5157
$ErrorActionPreference = "Stop"
@@ -82,7 +88,7 @@ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1
8288
# Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712
8389
$env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR
8490
$env:NUGET_EXE_Version = "3.4.3"
85-
$env:DOTNET_CLI_VERSION = "5.0.100-rc.1.20453.7"
91+
$env:DOTNET_CLI_VERSION = "5.0.100"
8692
# $env:DOTNET_RUNTIME_VERSION = "LATEST"
8793
$env:VSWHERE_VERSION = "2.0.2"
8894
$env:MSBUILD_VERSION = "15.0"
@@ -270,6 +276,7 @@ function Copy-PackageIntoStaticDirectory {
270276
# need to put them in folder that is not changing it's name based on config
271277
$tpPackagesPath = "$env:TP_OUT_DIR\$TPB_Configuration\packages\"
272278
$tpPackagesDestination = "$env:TP_TESTARTIFACTS"
279+
New-Item -ItemType Directory -Force $tpPackagesDestination | Out-Null
273280
Copy-Item $tpPackagesPath $tpPackagesDestination -Force -Filter *.nupkg -Verbose -Recurse
274281
}
275282

@@ -1103,19 +1110,37 @@ Write-Log "Test platform environment variables: "
11031110
Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table
11041111
Write-Log "Test platform build variables: "
11051112
Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table
1106-
Install-DotNetCli
1107-
Clear-Package
1108-
Restore-Package
1109-
Update-LocalizedResources
1110-
Invoke-Build
1111-
Publish-Package
1112-
Create-VsixPackage
1113-
Create-NugetPackages
1114-
Generate-Manifest
1115-
Publish-PatchedDotnet
1116-
Copy-PackageIntoStaticDirectory
1117-
Invoke-TestAssetsBuild
1118-
Publish-Tests
1113+
1114+
if ($Force -or $Steps -contains "InstallDotnet") {
1115+
Install-DotNetCli
1116+
}
1117+
1118+
if ($Force -or $Steps -contains "Restore") {
1119+
Clear-Package
1120+
Restore-Package
1121+
}
1122+
1123+
if ($Force -or $Steps -contains "UpdateLocalization") {
1124+
Update-LocalizedResources
1125+
}
1126+
1127+
if ($Force -or $Steps -contains "Build") {
1128+
Invoke-Build
1129+
}
1130+
1131+
if ($Force -or $Steps -contains "Publish") {
1132+
Publish-Package
1133+
Create-VsixPackage
1134+
Create-NugetPackages
1135+
Generate-Manifest
1136+
Copy-PackageIntoStaticDirectory
1137+
}
1138+
1139+
if ($Force -or $Steps -contains "PrepareAcceptanceTests") {
1140+
Publish-PatchedDotnet
1141+
Invoke-TestAssetsBuild
1142+
Publish-Tests
1143+
}
11191144

11201145
Write-Log "Build complete. {$(Get-ElapsedTime($timer))}"
11211146
if ($Script:ScriptFailed) { Exit 1 } else { Exit 0 }

‎scripts/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ VERSION=$(test -z $VERSION && grep TPVersionPrefix $TP_ROOT_DIR/scripts/build/Te
111111
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
112112
# Dotnet build doesnt support --packages yet. See https://github.com/dotnet/cli/issues/2712
113113
export NUGET_PACKAGES=$TP_PACKAGES_DIR
114-
DOTNET_CLI_VERSION="5.0.100-rc.1.20453.7"
114+
DOTNET_CLI_VERSION="5.0.100"
115115
#DOTNET_RUNTIME_VERSION="LATEST"
116116

117117
#

‎scripts/perf/perf.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ function Get-TestAdapterPath($testadapter)
119119
{
120120
if($testadapter -eq "MsTest")
121121
{
122-
return "$env:TP_PACKAGES_DIR\MSTest.TestAdapter\$($Script:TPT_DependencyProps.Project.PropertyGroup.MSTestAdapterVersion)\build\_common"
122+
return "$env:TP_PACKAGES_DIR\mstest.testadapter\$($Script:TPT_DependencyProps.Project.PropertyGroup.MSTestAdapterVersion)\build\_common"
123123
}
124124
if($testadapter -eq "xUnit")
125125
{

‎scripts/test.sh‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ NOCOLOR='\033[0m'
1515
# Parse options
1616
#
1717
CONFIGURATION="Debug"
18-
TARGET_RUNTIME="ubuntu.16.04-x64"
18+
TARGET_RUNTIME="ubuntu.18.04-x64"
1919
FAIL_FAST=false
2020
VERBOSE=false
2121

22-
while [ $# -gt 0 ]; do
22+
while [ $# -gt 0 ]
23+
do
2324
lowerI="$(echo ${1:-} | awk '{print tolower($0)}')"
2425
case $lowerI in
2526
-h | --help)
@@ -28,27 +29,35 @@ while [ $# -gt 0 ]; do
2829
;;
2930
-c)
3031
CONFIGURATION=$2
32+
shift
33+
shift
3134
;;
3235
-r)
3336
TARGET_RUNTIME=$2
37+
shift
38+
shift
3439
;;
3540
-p)
3641
PROJECT_NAME_PATTERNS=$2
42+
shift
43+
shift
3744
;;
3845
-verbose)
3946
VERBOSE=$2
47+
shift
48+
shift
4049
;;
4150
*)
42-
break
51+
echo Unknown parameter $key
52+
shift
4353
;;
4454
esac
45-
shift
4655
done
4756

4857
#
4958
# Variables
5059
#
51-
PROJECT_NAME_PATTERNS=**Unit*bin*$CONFIGURATION*netcoreapp2.1*UnitTests*dll
60+
PROJECT_NAME_PATTERNS=**$PROJECT_NAME_PATTERNS*bin*$CONFIGURATION*netcoreapp2.1*${PROJECT_NAME_PATTERNS}Tests*dll
5261
TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P)
5362
TP_TOOLS_DIR="$TP_ROOT_DIR/tools"
5463
TP_PACKAGES_DIR="$TP_ROOT_DIR/packages"
@@ -111,7 +120,7 @@ function invoke_test()
111120
local dotnet=$(_get_dotnet_path)
112121
local vstest=$TP_OUT_DIR/$TPB_Configuration/$TPB_TargetFrameworkCore/vstest.console.dll
113122

114-
find ./test -path $PROJECT_NAME_PATTERNS | xargs $dotnet $vstest --parallel --testcasefilter:"TestCategory!=Windows"
123+
find ./test -path $PROJECT_NAME_PATTERNS | xargs $dotnet $vstest --parallel --testcasefilter:"TestCategory!=Windows&TestCategory!=Windows-Review"
115124
}
116125

117126
#

‎src/package/nuspec/Microsoft.NET.Test.Sdk.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
1414

1515
<PropertyGroup>
16+
<TestProject>true</TestProject>
1617
<IsTestProject>true</IsTestProject>
1718
</PropertyGroup>
1819

‎test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests
1414
using Microsoft.VisualStudio.TestTools.UnitTesting;
1515

1616
[TestClass]
17+
[TestCategory("Windows-Review")]
1718
public class AppDomainTests : AcceptanceTestBase
1819
{
1920
[TestMethod]
21+
[TestCategory("Windows-Review")]
2022
[NetFullTargetFrameworkDataSource]
2123
public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo)
2224
{

‎test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests
66
using Microsoft.VisualStudio.TestTools.UnitTesting;
77

88
[TestClass]
9+
[TestCategory("Windows-Review")]
910
public class ArgumentProcessorTests : AcceptanceTestBase
1011
{
1112

1213
[TestMethod]
14+
[TestCategory("Windows-Review")]
1315
[NetFullTargetFrameworkDataSource]
1416
public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo runnerInfo)
1517
{
@@ -30,6 +32,7 @@ public void PassingNoArgumentsToVsTestConsoleShouldPrintHelpMessage(RunnerInfo r
3032
}
3133

3234
[TestMethod]
35+
[TestCategory("Windows-Review")]
3336
[NetFullTargetFrameworkDataSource]
3437
public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(RunnerInfo runnerInfo)
3538
{

0 commit comments

Comments
 (0)