[invalid] Test uploads with azcopy

scripts/azure-pipelines/android/azure-pipelines.yml

@@ -98,7 +98,7 @@ jobs:
          -File scripts/azure-pipelines/test-modified-ports.ps1 \
          -Triplet ${{ replace(parameters.jobName, '_', '-') }} \
          -BuildReason $(Build.Reason) \
-         -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" \
+         -BinarySourceStub "x-azcopy-sas,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" \
          -WorkingRoot ${{ variables.WORKING_ROOT }}
   - task: PublishPipelineArtifact@1
     displayName: "Publish Artifact: failure logs for ${{ replace(parameters.jobName, '_', '-') }}"

scripts/azure-pipelines/azure-pipelines.yml

@@ -5,11 +5,11 @@ parameters:
   - name: vcpkgToolSha
     displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap'
     type: string
-    default:  'use default'
+    default:  'refs/pull/1738/merge'
   - name: tripletPattern
-    displayName: 'Enable triplets which contain this substring'
+    displayName: 'Enable triplets which contain this substring (^ matches begin, $ matches end)'
     type: string
-    default: '-'
+    default: '^'
 
 jobs:
 - template: windows/azure-pipelines.yml

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -94,7 +94,7 @@ jobs:
          -File scripts/azure-pipelines/test-modified-ports.ps1 \
          -Triplet ${{ replace(parameters.jobName, '_', '-') }} \
          -BuildReason $(Build.Reason) \
-         -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" \
+         -BinarySourceStub "x-azcopy-sas,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" \
          -WorkingRoot ${{ variables.WORKING_ROOT }}
   - task: PublishPipelineArtifact@1
     displayName: "Publish Artifact: failure logs for ${{ replace(parameters.jobName, '_', '-') }}"

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -57,7 +57,7 @@ jobs:
           $binarySas = az storage container generate-sas --name cache --account-name vcpkgbinarycachewus --as-user --auth-mode login --https-only --permissions rclw --expiry $end -o tsv | Out-String
           $binarySas = $binarySas.Trim()
           $env:X_VCPKG_ASSET_SOURCES = "x-azurl,https://vcpkgassetcachewus.blob.core.windows.net/cache,$assetSas,readwrite"
-          & scripts/azure-pipelines/test-modified-ports.ps1 -Triplet ${{ replace(parameters.jobName, '_', '-') }} -BuildReason $(Build.Reason) -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" -WorkingRoot $env:WORKING_ROOT -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)
+          & scripts/azure-pipelines/test-modified-ports.ps1 -Triplet ${{ replace(parameters.jobName, '_', '-') }} -BuildReason $(Build.Reason) -BinarySourceStub "x-azcopy-sas,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" -WorkingRoot $env:WORKING_ROOT -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)
   - task: PublishPipelineArtifact@1
     displayName: 'Publish Artifact: failure logs for x64-osx'
     inputs:

scripts/azure-pipelines/test-modified-ports.ps1

@@ -84,6 +84,12 @@ if ((-Not [string]::IsNullOrWhiteSpace($ArchivesRoot))) {
 $buildtreesRoot = Join-Path $WorkingRoot 'b'
 $installRoot = Join-Path $WorkingRoot 'installed'
 $packagesRoot = Join-Path $WorkingRoot 'p'
+$failureLogs = Join-Path $ArtifactStagingDirectory 'failure-logs'
+
+$env:AZCOPY_LOG_LOCATION = Join-Path $failureLogs 'azcopy-logs'
+$env:AZCOPY_JOB_PLAN_LOCATION = Join-Path $WorkingRoot 'azcopy-plans'
+
+New-Item -Type Directory -Path $env:AZCOPY_LOG_LOCATION -Force | Out-Null
 
 $commonArgs = @(
     "--x-buildtrees-root=$buildtreesRoot",
@@ -128,7 +134,38 @@ if ($Triplet -eq 'x64-windows-release') {
     $tripletArg = "--triplet=$Triplet"
 }
 
-$failureLogs = Join-Path $ArtifactStagingDirectory 'failure-logs'
+# Build large artifacts with unique ABI hashes
+Get-Date | Out-File "scripts/manual-tests/azcopy/test-upload-artifacts/mutable"
+# 3 GB < single max write (5 GB) < 6 GB, in files of 1.5 GB random data
+$data = New-Object byte[] 1.5GB
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/core-1.dat", $data)
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/core-2.dat", $data)
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/large-1.dat", $data)
+(New-Object System.Random).NextBytes($data)
+[System.IO.File]::WriteAllBytes("scripts/manual-tests/azcopy/large-2.dat", $data)
+Get-ChildItem -File scripts/manual-tests/azcopy
+# Build and upload [core], [core,large]
+$env:VCPKG_DEFAULT_BINARY_CACHE = Join-Path $WorkingRoot 'archives-azcopy'
+New-Item -Type Directory -Path $env:VCPKG_DEFAULT_BINARY_CACHE -Force | Out-Null
+& $vcpkgExe x-test-features test-upload-artifacts --overlay-ports=scripts/manual-tests/azcopy $tripletArg @commonArgs @cachingArgs
+$lastLastExitCode = $LASTEXITCODE
+$env:VCPKG_DEFAULT_BINARY_CACHE = $null
+if ($lastLastExitCode -eq 0)
+{
+    # Try restore from cache
+    & $vcpkgExe remove test-upload-artifacts $tripletArg @commonArgs @cachingArgs
+    $xunitFile = Join-Path $ArtifactStagingDirectory "$Triplet-results.xml"
+    $xunitArg = "--x-xunit=$xunitFile"
+    & $vcpkgExe install --only-binarycaching "test-upload-artifacts[*]" --overlay-ports=scripts/manual-tests/azcopy $tripletArg $xunitArg @commonArgs @cachingArgs
+    $lastLastExitCode = $LASTEXITCODE
+    Write-Host "##vso[task.setvariable variable=XML_RESULTS_FILE]$xunitFile"
+}
+Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$true"
+exit $lastLastExitCode
+
 $failureLogsArg = "--failure-logs=$failureLogs"
 $knownFailuresFromArgs = @()
 if ($testFeatures) {

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -73,7 +73,7 @@ jobs:
           $binarySas = $binarySas.Trim()
           $env:X_VCPKG_ASSET_SOURCES = "x-azurl,https://vcpkgassetcachewus.blob.core.windows.net/cache,$assetSas,readwrite"
           ./vcpkg.exe fetch python3
-          & scripts/azure-pipelines/test-modified-ports.ps1 -Triplet ${{ replace(parameters.jobName, '_', '-') }} -BuildReason $(Build.Reason) -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" -WorkingRoot $env:WORKING_ROOT -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)
+          & scripts/azure-pipelines/test-modified-ports.ps1 -Triplet ${{ replace(parameters.jobName, '_', '-') }} -BuildReason $(Build.Reason) -BinarySourceStub "x-azcopy-sas,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" -WorkingRoot $env:WORKING_ROOT -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory)
   - task: PowerShell@2
     displayName: 'Validate version files'
     condition: eq('${{ replace(parameters.jobName, '_', '-') }}', '${{ variables.ExtraChecksTriplet }}')

scripts/manual-tests/azcopy/core-1.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/core-2.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/large-1.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/large-2.dat

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/test-upload-artifacts/mutable

@@ -0,0 +1 @@
+placeholder

scripts/manual-tests/azcopy/test-upload-artifacts/portfile.cmake

@@ -0,0 +1,7 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
+
+file(INSTALL "${CURRENT_PORT_DIR}/mutable" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")
+foreach(name IN LISTS FEATURES)
+    file(GLOB blobs "${CURRENT_PORT_DIR}/../${name}-*.dat")
+    file(INSTALL ${blobs} DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")
+endforeach()

scripts/manual-tests/azcopy/test-upload-artifacts/vcpkg.json

@@ -0,0 +1,15 @@
+{
+  "name": "test-upload-artifacts",
+  "version-string": "ci",
+  "description": [
+    "Tests uploading artifacts of varying size.",
+    "Write suitable amounts of random data to the datafiles before installing the port."
+  ],
+  "homepage": "https://github.com/microsoft/vcpkg",
+  "license": "MIT",
+  "features": {
+    "large" : {
+      "description": "Install large.dat"
+    }
+  }
+}

scripts/vcpkg-tools.json

@@ -306,7 +306,7 @@
     {
       "name": "node",
       "os": "windows",
-      "arch":"x64",
+      "arch": "x64",
       "version": "24.3.0",
       "executable": "node-v24.3.0-win-x64/node.exe",
       "url": "https://nodejs.org/dist/v24.3.0/node-v24.3.0-win-x64.7z",
@@ -316,7 +316,7 @@
     {
       "name": "node",
       "os": "windows",
-      "arch":"arm64",
+      "arch": "arm64",
       "version": "24.3.0",
       "executable": "node-v24.3.0-win-arm64/node.exe",
       "url": "https://nodejs.org/dist/v24.3.0/node-v24.3.0-win-arm64.7z",
@@ -326,7 +326,7 @@
     {
       "name": "node",
       "os": "linux",
-      "arch":"x64",
+      "arch": "x64",
       "version": "24.3.0",
       "executable": "node-v24.3.0-linux-x64/bin/node",
       "url": "https://nodejs.org/dist/v24.3.0/node-v24.3.0-linux-x64.tar.gz",
@@ -346,7 +346,7 @@
     {
       "name": "node",
       "os": "osx",
-      "arch":"x64",
+      "arch": "x64",
       "version": "24.3.0",
       "executable": "node-v24.3.0-darwin-x64/bin/node",
       "url": "https://nodejs.org/dist/v24.3.0/node-v24.3.0-darwin-x64.tar.gz",
@@ -367,41 +367,41 @@
       "name": "azcopy",
       "os": "linux",
       "arch": "amd64",
-      "version": "10.29.1",
-      "executable": "azcopy_linux_amd64_10.29.1/azcopy",
-      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.1/azcopy_linux_amd64_10.29.1.tar.gz",
-      "sha512": "e2fe66658d790784e5d32f397254aad2bb3fa3faec6929e5a9800111d8feee8353db0250feb7d9e49a77118a5437513162704506e0324903b5aeb96f8d85e8fb",
-      "archive": "azcopy_linux_amd64_10.29.1.tar.gz"
+      "version": "10.28.0",
+      "executable": "azcopy_linux_amd64_10.28.0/azcopy",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.28.0/azcopy_linux_amd64_10.28.0.tar.gz",
+      "sha512": "3fb8f6c902f9f89535f6f50efbf57032db31d26e3ad4a3d0fe1c973a3715983b14e26fecfc8f1e65bd57a60b71e282ecde05208309e6db875877bacb0e875566",
+      "archive": "azcopy_linux_amd64_10.28.0.tar.gz"
     },
     {
       "name": "azcopy",
       "os": "osx",
       "arch": "amd64",
-      "version": "10.29.1",
-      "executable": "azcopy_darwin_amd64_10.29.1/azcopy",
-      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.1/azcopy_darwin_amd64_10.29.1.zip",
-      "sha512": "12481665136e6a10c8b1f784a097420c76d47336e8d2d5dc8b004012ab75e0e91d846fbd8cd0fb75bd6e9603db660cd5eb5ce71a8e003994a4356f19e3c02ec2",
-      "archive": "azcopy_darwin_amd64_10.29.1.zip"
+      "version": "10.28.0",
+      "executable": "azcopy_darwin_amd64_10.28.0/azcopy",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.28.0/azcopy_darwin_amd64_10.28.0.zip",
+      "sha512": "e49da705aaa2399a5aec2ff3ae26d8776d338a3c32874abe6d9888393a080960ba1b8ca9add352323b048d82b88a4afc38b26b5c39ec25ecd253544a309ec252",
+      "archive": "azcopy_darwin_amd64_10.28.0.zip"
     },
     {
       "name": "azcopy",
       "os": "osx",
       "arch": "arm64",
-      "version": "10.29.1",
-      "executable": "azcopy_darwin_arm64_10.29.1/azcopy",
-      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.1/azcopy_darwin_arm64_10.29.1.zip",
-      "sha512": "025bbc13b6617ab341fd0344a7a6342b44e6d10e5b7fb009a35988bfcebc442e0d3c6040f7245ea4e3c0643410a0342eff17aab295dea7667441f1e78801b845",
-      "archive": "azcopy_darwin_arm64_10.29.1.zip"
+      "version": "10.28.0",
+      "executable": "azcopy_darwin_arm64_10.28.0/azcopy",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.28.0/azcopy_darwin_arm64_10.28.0.zip",
+      "sha512": "53923b9379c0532c889b9b5f806e1bbdfef05362dadc3e060972e4de34c8a6a1bb44084171991aaeb5792965ef6670956f1cbf078c755368f3349d356b734364",
+      "archive": "azcopy_darwin_arm64_10.28.0.zip"
     },
     {
       "name": "azcopy",
       "os": "windows",
       "arch": "amd64",
-      "version": "10.29.1",
-      "executable": "azcopy_windows_amd64_10.29.1/azcopy.exe",
-      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.29.1/azcopy_windows_amd64_10.29.1.zip",
-      "sha512": "f3f30b51d7447e27912bb42241ff187cc89596a0ca0c7138367891ecb01282e947a70f6ba1f248f79ef4f4707e03cceeabcc7b3bc8187cb661e32c2195ddfdd2",
-      "archive": "azcopy_windows_amd64_10.29.1.zip"
+      "version": "10.28.0",
+      "executable": "azcopy_windows_amd64_10.28.0/azcopy.exe",
+      "url": "https://github.com/Azure/azure-storage-azcopy/releases/download/v10.28.0/azcopy_windows_amd64_10.28.0.zip",
+      "sha512": "67aa0a03fc5d71c318025447ef6de612956191e0523cb16e6e1472c35994e26090a2f9f063c408b146bf6209de0215a65e7790b1cfc694538189925878d8f5cb",
+      "archive": "azcopy_windows_amd64_10.28.0.zip"
     }
   ]
 }