[WASI] Acquire the WASI SDK for runtime build automatically and centrally

eng/AcquireWasiSdk.targets

@@ -0,0 +1,62 @@
+<!--
+  Import these targets to acquire the WASI SDK.
+  - Use DependsOnTargets="AcquireWasiSdk" on the target that depends on WASI SDK.
+    - By default, this target is conditioned on $(TargetsWasi).
+    - You can depend on "AcquireWasiSdkUnconditional" instead if this doesn't work for your project.
+  - Use $(RuntimeBuildWasiSdkPath) (set by "AcquireWasiSdk") to refer to the SDK root in your target.
+-->
+<Project>
+  <PropertyGroup>
+    <_WasiSdkVersion>25.0</_WasiSdkVersion>
+    <_RuntimeLocalWasiSdkPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'wasi-sdk'))</_RuntimeLocalWasiSdkPath>    
+  </PropertyGroup>
+
+  <!-- If the user wants us to use an explicit SDK, validate it matches our expectations. -->
+  <Target Name="_ValidateWasiSdk">
+    <PropertyGroup>
+      <_ActualWasiSdkVersion Condition="'$(WASI_SDK_PATH)' != '' and Exists('$(WASI_SDK_PATH)/VERSION')">$([System.IO.File]::ReadAllText('$(WASI_SDK_PATH)/VERSION').Split()[0])</_ActualWasiSdkVersion>
+      <_UseRuntimeLocalWasiSdk Condition="'$(_WasiSdkVersion)' != '$(_ActualWasiSdkVersion)'">true</_UseRuntimeLocalWasiSdk>
+    </PropertyGroup>
+  </Target>
+
+  <!-- Otherwise, download our own SDK. Use this file as the input since that's what defines the version. -->
+  <Target Name="_AcquireLocalWasiSdk"
+          Condition="'$(_UseRuntimeLocalWasiSdk)' == 'true'"
+          Inputs="$(MSBuildThisFileFullPath)"
+          Outputs="$(_RuntimeLocalWasiSdkPath)VERSION">
+
+    <Message Text="Downloading a runtime-local WASI SDK $(_WasiSdkVersion) to '$(_RuntimeLocalWasiSdkPath)'" Importance="High" />
+
+    <PropertyGroup>
+      <_WasiSdkMajorVersion>$(_WasiSdkVersion.Split('.')[0])</_WasiSdkMajorVersion>
+      <_WasiSdkUrl>https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$(_WasiSdkMajorVersion)/wasi-sdk-$(_WasiSdkVersion)-x86_64-linux.tar.gz</_WasiSdkUrl>
+      <_WasiSdkUrl Condition="'$(HostOS)' == 'osx'" >https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$(_WasiSdkMajorVersion)/wasi-sdk-$(_WasiSdkVersion)-x86_64-macos.tar.gz</_WasiSdkUrl>
+      <_WasiSdkUrl Condition="'$(HostOS)' == 'windows'" >https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$(_WasiSdkMajorVersion)/wasi-sdk-$(_WasiSdkVersion)-x86_64-windows.tar.gz</_WasiSdkUrl>
+    </PropertyGroup>
+
+    <RemoveDir Directories="$(_RuntimeLocalWasiSdkPath)" />
+    <MakeDir Directories="$(_RuntimeLocalWasiSdkPath)" />
+
+    <Exec Command="curl -L -o wasi-sdk-$(_WasiSdkVersion).tar.gz $(_WasiSdkUrl) &amp;&amp; tar --strip-components=1 -xzmf wasi-sdk-$(_WasiSdkVersion).tar.gz -C $(_RuntimeLocalWasiSdkPath)"
+          Condition="'$(HostOS)' != 'windows'"
+          WorkingDirectory="$(ArtifactsObjDir)"
+          IgnoreStandardErrorWarningFormat="true" />
+
+    <Exec Command="powershell -NonInteractive -command &quot;&amp; $(MSBuildThisFileDirectory)\download-wasi-sdk.ps1 -WasiSdkUrl $(_WasiSdkUrl) -WasiSdkVersion $(_WasiSdkVersion) -WasiSdkPath $(_RuntimeLocalWasiSdkPath); Exit $LastExitCode &quot;"
+          Condition="'$(HostOS)' == 'windows'"
+          WorkingDirectory="$(ArtifactsObjDir)"
+          IgnoreStandardErrorWarningFormat="true" />
+  </Target>
+
+  <Target Name="AcquireWasiSdkUnconditional"
+          DependsOnTargets="_ValidateWasiSdk;_AcquireLocalWasiSdk">
+    <PropertyGroup>
+      <RuntimeBuildWasiSdkPath Condition="'$(_UseRuntimeLocalWasiSdk)' != 'true'">$([MSBuild]::NormalizeDirectory('$(WASI_SDK_PATH)'))</RuntimeBuildWasiSdkPath>
+      <RuntimeBuildWasiSdkPath Condition="'$(_UseRuntimeLocalWasiSdk)' == 'true'">$(_RuntimeLocalWasiSdkPath)</RuntimeBuildWasiSdkPath>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="AcquireWasiSdk"
+          Condition="'$(TargetsWasi)' == 'true'"
+          DependsOnTargets="AcquireWasiSdkUnconditional" />
+</Project>

src/mono/wasi/provision.ps1 → eng/download-wasi-sdk.ps1

@@ -4,17 +4,13 @@ param(
      [Parameter()]
      [string]$WasiSdkVersion,
      [Parameter()]
-     [string]$WasiSdkPath,
-     [Parameter()]
-     [string]$WasiLocalPath
+     [string]$WasiSdkPath
 )
 
 Set-StrictMode -version 2.0
 $ErrorActionPreference='Stop'
 $ProgressPreference = 'SilentlyContinue'
 
-New-Item -Path $WasiSdkPath -ItemType "directory"
 Invoke-WebRequest -Uri $WasiSdkUrl -OutFile ./wasi-sdk-$WasiSdkVersion-x86_64-windows.tar.gz
-tar --strip-components=1 -xzf ./wasi-sdk-$WasiSdkVersion-x86_64-windows.tar.gz -C $WasiSdkPath
-Copy-Item $WasiLocalPath/wasi-sdk-version.txt $WasiSdkPath/wasi-sdk-version.txt
+tar --strip-components=1 -xzmf ./wasi-sdk-$WasiSdkVersion-x86_64-windows.tar.gz -C $WasiSdkPath
 Remove-Item ./wasi-sdk-$WasiSdkVersion-x86_64-windows.tar.gz -fo

eng/native/gen-buildsys.cmd

@@ -59,12 +59,12 @@ if /i "%__Arch%" == "wasm" (
     )
     if /i "%__Os%" == "wasi" (
         if "%WASI_SDK_PATH%" == "" (
-            if not exist "%__repoRoot%\src\mono\wasi\wasi-sdk" (
+            if not exist "%__repoRoot%\artifacts\wasi-sdk" (
                 echo Error: Should set WASI_SDK_PATH environment variable pointing to WASI SDK root.
                 exit /B 1
             )
 
-            set "WASI_SDK_PATH=%__repoRoot%\src\mono\wasi\wasi-sdk"
+            set "WASI_SDK_PATH=%__repoRoot%\artifacts\wasi-sdk"
         )
         set __CmakeGenerator=Ninja
         set __ExtraCmakeParams=%__ExtraCmakeParams% -DCLR_CMAKE_TARGET_OS=wasi "-DCMAKE_TOOLCHAIN_FILE=!WASI_SDK_PATH!/share/cmake/wasi-sdk-p2.cmake" "-DCMAKE_CROSSCOMPILING_EMULATOR=node --experimental-wasm-bigint --experimental-wasi-unstable-preview1"

eng/native/gen-buildsys.sh

@@ -109,8 +109,8 @@ if [[ "$host_arch" == "wasm" ]]; then
         cmake_command="emcmake $cmake_command"
     elif [[ "$target_os" == "wasi" ]]; then
         if [[ -z "$WASI_SDK_PATH" ]]; then
-            if [[ -d "$reporoot"/src/mono/wasi/wasi-sdk ]]; then
-                export WASI_SDK_PATH="$reporoot"/src/mono/wasi/wasi-sdk
+            if [[ -d "$reporoot"/artifacts/wasi-sdk ]]; then
+                export WASI_SDK_PATH="$reporoot"/artifacts/wasi-sdk
             else
                 echo "Error: You need to set the WASI_SDK_PATH environment variable pointing to the WASI SDK root."
                 exit 1

eng/testing/tests.wasi.targets

@@ -13,8 +13,6 @@
                                         >true</InstallWasmtimeForTests>
 
     <!--<InstallWorkloadUsingArtifactsDependsOn>_GetWorkloadsToInstall;$(InstallWorkloadUsingArtifactsDependsOn)</InstallWorkloadUsingArtifactsDependsOn>-->
-    <WASI_SDK_PATH Condition="'$(WASI_SDK_PATH)' == '' or !Exists('$(WASI_SDK_PATH)/WASI-SDK-VERSION-25.0')">$([MSBuild]::NormalizeDirectory($(MonoProjectRoot), 'wasi', 'wasi-sdk'))</WASI_SDK_PATH>
-    <WASI_SDK_PATH>$([MSBuild]::EnsureTrailingSlash('$(WASI_SDK_PATH)').Replace('\', '/'))</WASI_SDK_PATH>
 
     <_BundleAOTTestWasmAppForHelixDependsOn>$(_BundleAOTTestWasmAppForHelixDependsOn);PrepareForWasiBuildApp;_PrepareForAOTOnHelix</_BundleAOTTestWasmAppForHelixDependsOn>
   </PropertyGroup>

src/libraries/sendtohelix-wasi.targets

@@ -46,8 +46,6 @@
     <IncludeHelixCorrelationPayload>false</IncludeHelixCorrelationPayload>
     <EnableDefaultBuildHelixWorkItems>false</EnableDefaultBuildHelixWorkItems>
 
-    <WASI_SDK_PATH Condition="'$(WASI_SDK_PATH)' == '' or !Exists('$(WASI_SDK_PATH)/WASI-SDK-VERSION-25.0')">$([MSBuild]::NormalizeDirectory($(RepoRoot), 'src', 'mono', 'wasi', 'wasi-sdk'))</WASI_SDK_PATH>
-    <WASI_SDK_PATH>$([MSBuild]::EnsureTrailingSlash('$(WASI_SDK_PATH)').Replace('\', '/'))</WASI_SDK_PATH>
     <WasiBuildTargetsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono', 'wasi', 'build'))</WasiBuildTargetsDir>
     <WasiSdkDirForHelixPayload>$(HelixDependenciesStagingPath)$(WorkItemPrefix)wasi-sdk</WasiSdkDirForHelixPayload>
     <WasmtimeDirForHelixPayload>$(HelixDependenciesStagingPath)$(WorkItemPrefix)wasmtime</WasmtimeDirForHelixPayload>
@@ -63,11 +61,6 @@
     <SdkForWorkloadTestingDirName Condition="'$(NeedsWorkload)' == 'true' and '$(TestUsingWorkloads)' != 'true'">dotnet-none</SdkForWorkloadTestingDirName>
   </PropertyGroup>
 
-  <ItemGroup>
-    <HelixDependenciesToStage Condition="'$(NeedsWasiSdk)' == 'true'"  SourcePath="$(WASI_SDK_PATH)" Include="$(WasiSdkDirForHelixPayload)" />
-    <HelixDependenciesToStage Condition="'$(NeedsWasmtime)' == 'true'" SourcePath="$(WasmtimeDir)"   Include="$(WasmtimeDirForHelixPayload)" />
-  </ItemGroup>
-
   <ItemGroup Condition="'$(WindowsShell)' != 'true'">
     <HelixPreCommand Condition="'$(Scenario)' != ''" Include="export SCENARIO=$(Scenario)" />
     <HelixPreCommand Include="export XHARNESS_DISABLE_COLORED_OUTPUT=true" />
@@ -104,7 +97,20 @@
     <EnableXHarnessTelemetry>false</EnableXHarnessTelemetry>
   </PropertyGroup>
 
+  <ItemGroup>
+    <HelixDependenciesToStage Condition="'$(NeedsWasmtime)' == 'true'" SourcePath="$(WasmtimeDir)" Include="$(WasmtimeDirForHelixPayload)" />
+  </ItemGroup>
+
   <Import Project="$(RepositoryEngineeringDir)testing\wasi-provisioning.targets" />
+  <Import Project="$(RepositoryEngineeringDir)AcquireWasiSdk.targets" />
+
+  <Target Name="IncludeWasiSdkForHelixStaging" Condition="'$(NeedsWasiSdk)' == 'true'"
+          DependsOnTargets="AcquireWasiSdk"
+          BeforeTargets="StageDependenciesForHelix">
+    <ItemGroup>
+      <HelixDependenciesToStage SourcePath="$(RuntimeBuildWasiSdkPath)" Include="$(WasiSdkDirForHelixPayload)" />
+    </ItemGroup>
+  </Target>
 
   <Target Name="PrepareHelixCorrelationPayload_Wasi">
     <Error Condition="'$(Scenario)' != 'WasmTestOnWasmtime' and 

src/libraries/sendtohelixhelp.proj

@@ -108,11 +108,6 @@
     <TestEnvFileName Condition=" '$(Scenario)' != '' and '$(TargetOS)' != 'windows' and '$(TargetOS)' != 'browser' and '$(TargetOS)' != 'wasi'">SetStressModes_$(Scenario).sh</TestEnvFileName>
   </PropertyGroup>
 
-  <ItemGroup>
-    <__HelixDependenciesToStageThatDontExist Include="@(HelixDependenciesToStage)" />
-    <__HelixDependenciesToStageThatDontExist Remove="@(HelixDependenciesToStage->Exists())" />
-  </ItemGroup>
-
   <!-- HelixPreCommands is a set of commands run before the work item command. We use it here to inject
        setting up the per-scenario environment.
   -->
@@ -362,7 +357,12 @@
         `WASI_SDK_PATH=/usr/local/wasi-sdk`
       .. then we need to stage that before passing the path to helix like under `artifacts/obj/helix-staging`
   -->
-  <Target Name="StageDependenciesForHelix" Condition="@(__HelixDependenciesToStageThatDontExist->Count()) > 0">
+  <Target Name="StageDependenciesForHelix"
+          Condition="'@(HelixDependenciesToStage->Exists())' != '@(HelixDependenciesToStage)'">
+    <ItemGroup>
+      <__HelixDependenciesToStageThatDontExist Include="@(HelixDependenciesToStage)" Condition="!Exists('%(Identity)')" />
+    </ItemGroup>
+
     <Error Condition="'%(__HelixDependenciesToStageThatDontExist.SourcePath)' == '' or !Exists(%(__HelixDependenciesToStageThatDontExist.SourcePath))"
            Text="Could not find %(__HelixDependenciesToStageThatDontExist.Name) at %(__HelixDependenciesToStageThatDontExist.SourcePath), needed to provision for running tests on helix" />
 

src/mono/Directory.Build.props

@@ -32,13 +32,6 @@
     <EMSDK_PATH Condition="Exists('$(ProvisionEmscriptenDir)') and '$(EMSDK_PATH)' == ''">$(ProvisionEmscriptenDir.Replace('\', '/'))</EMSDK_PATH>
   </PropertyGroup>
 
-  <!-- Directory to provision and use WASI sdk if WASI_SDK_PATH env variable is not set -->
-  <PropertyGroup Condition="'$(TargetsWasi)' == 'true'">
-    <WASI_SDK_PATH Condition="'$(WASI_SDK_PATH)' == '' or !Exists('$(WASI_SDK_PATH)/WASI-SDK-VERSION-25.0')">$([MSBuild]::NormalizeDirectory($(MSBuildThisFileDirectory), 'wasi', 'wasi-sdk'))</WASI_SDK_PATH>
-    <WASI_SDK_PATH>$([MSBuild]::EnsureTrailingSlash('$(WASI_SDK_PATH)').Replace('\', '/'))</WASI_SDK_PATH>
-    <ShouldProvisionWasiSdk Condition="!Exists('$(WASI_SDK_PATH)/WASI-SDK-VERSION-25.0')">true</ShouldProvisionWasiSdk>
-  </PropertyGroup>
-
   <PropertyGroup>
     <PlatformConfigPathPart>$(TargetOS).$(Platform).$(Configuration)</PlatformConfigPathPart>
     <RuntimeBinDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'mono', '$(TargetOS).$(Platform).$(Configuration)'))</RuntimeBinDir>

src/mono/browser/README.md

@@ -393,7 +393,7 @@ Tests are run with V8, Chrome, node, and wasmtime for the various jobs.
 - V8: the version used is from `eng/testing/BrowserVersions.props`. This is used for all the library tests, and WBT, but *not* runtime tests.
 - Chrome: Same as V8.
 - Node: fixed version from emsdk
-- wasmtime - fixed version in `src/mono/wasi/wasi-sdk-version.txt`.
+- wasmtime - fixed version in `src/mono/wasi/wasmtime-version.txt`.
 
 ### `eng/testing/BrowserVersions.props`