Skip to content

Add x-azcopy and x-azcopy-sas binary caching sources #1738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
+578 −32
Open

Add x-azcopy and x-azcopy-sas binary caching sources #1738

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
18f4be4
Add azcopy provider
vicroms Jul 29, 2025
7710e07
Generate message map
vicroms Jul 29, 2025
f538301
Add parser tests
vicroms Jul 31, 2025
df0bbca
Refactor repeated azcopy sources handling
vicroms Aug 1, 2025
29d9966
Silence INFO output on azcopy list invocation
vicroms Aug 1, 2025
faf56e3
Make error messages match source name
vicroms Aug 1, 2025
cfe3e13
Extract functions from handle_azcopy_segments, delambdaize, split.
BillyONeal Aug 1, 2025
614c8bf
Reimplement azcopy providers
vicroms Aug 2, 2025
1dd3bdc
Update src/vcpkg/binarycaching.cpp
vicroms Aug 2, 2025
5b2457a
Fix redefinition warning
vicroms Aug 2, 2025
0228ce0
Merge branch 'azcopy-provider' of github.com:vicroms/vcpkg-tool into …
vicroms Aug 2, 2025
db5f025
Fix build error
vicroms Aug 2, 2025
c6c50a1
Download multiple files in single invocation
vicroms Aug 2, 2025
4b1399e
Break down long azcopy invocations in batches
vicroms Aug 2, 2025
1065502
Add metric for azcopy-sas
vicroms Aug 3, 2025
5a21789
Print error output
vicroms Aug 5, 2025
e3681cd
Undo changes to IObjectStorageTool
vicroms Aug 7, 2025
3e09823
Parse cache files from azcopy list output
vicroms Aug 11, 2025
6ec4c06
Apply suggestions from code review
vicroms Aug 11, 2025
ea49e5c
Apply Billy's suggestions
vicroms Aug 11, 2025
8143199
Fix precheck and apply Thomas1664 suggestion
vicroms Aug 12, 2025
23a0c46
PR review coments
vicroms Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions include/vcpkg/base/message-data.inc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2706,6 +2706,10 @@ DECLARE_MESSAGE(RestoredPackagesFromAWS,
(msg::count, msg::elapsed),
"",
"Restored {count} package(s) from AWS in {elapsed}. Use --debug to see more details.")
DECLARE_MESSAGE(RestoredPackagesFromAzureStorage,
(msg::count, msg::elapsed),
"",
"Restored {count} package(s) from Azure Storage in {elapsed}. Use --debug to see more details.")
DECLARE_MESSAGE(RestoredPackagesFromAZUPKG,
(msg::count, msg::elapsed),
"",
Expand Down
3 changes: 3 additions & 0 deletions include/vcpkg/binarycaching.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ namespace vcpkg

std::vector<UrlTemplate> azblob_templates_to_put;

std::vector<std::string> azcopy_read_templates;
std::vector<std::string> azcopy_write_templates;

std::vector<std::string> gcs_read_prefixes;
std::vector<std::string> gcs_write_prefixes;

Expand Down
1 change: 1 addition & 0 deletions include/vcpkg/metrics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ namespace vcpkg
AssetSource,
BinaryCachingAws,
BinaryCachingAzBlob,
BinaryCachingAzCopy,
BinaryCachingCos,
BinaryCachingDefault,
BinaryCachingFiles,
Expand Down
2 changes: 2 additions & 0 deletions locales/messages.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1419,6 +1419,8 @@
"_RestoredPackagesFromAWS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
"RestoredPackagesFromAZUPKG": "Restored {count} package(s) from Universal Packages in {elapsed}. Use --debug to see more details.",
"_RestoredPackagesFromAZUPKG.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
"RestoredPackagesFromAzureStorage": "Restored {count} package(s) from Azure Storage in {elapsed}. Use --debug to see more details.",
"_RestoredPackagesFromAzureStorage.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
"RestoredPackagesFromCOS": "Restored {count} package(s) from COS in {elapsed}. Use --debug to see more details.",
"_RestoredPackagesFromCOS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.",
"RestoredPackagesFromFiles": "Restored {count} package(s) from {path} in {elapsed}. Use --debug to see more details.",
Expand Down
149 changes: 149 additions & 0 deletions src/vcpkg-test/configparser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,155 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]")
}
}

TEST_CASE ("BinaryConfigParser azcopy providers", "[binaryconfigparser]")
{
SECTION ("azcopy no SAS token")
{
{
auto parsed = parse_binary_provider_configs("x-azcopy,https://azure/container", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);

REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
REQUIRE(state.azcopy_read_templates.size() == 1);
CHECK(state.azcopy_read_templates.front() == "https://azure/container/{sha}.zip");
CHECK(state.azcopy_write_templates.empty());
REQUIRE(state.secrets.empty());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,https://azure/container,read", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);

REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
REQUIRE(state.azcopy_read_templates.size() == 1);
CHECK(state.azcopy_read_templates.front() == "https://azure/container/{sha}.zip");
CHECK(state.azcopy_write_templates.empty());
REQUIRE(state.secrets.empty());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,https://azure/container,write", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);

REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
CHECK(state.azcopy_read_templates.empty());
REQUIRE(state.azcopy_write_templates.size() == 1);
CHECK(state.azcopy_write_templates.front() == "https://azure/container/{sha}.zip");
REQUIRE(state.secrets.empty());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,https://azure/container,readwrite", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);

REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
REQUIRE(state.azcopy_read_templates.size() == 1);
CHECK(state.azcopy_read_templates.front() == "https://azure/container/{sha}.zip");
REQUIRE(state.azcopy_write_templates.size() == 1);
CHECK(state.azcopy_write_templates.front() == "https://azure/container/{sha}.zip");
REQUIRE(state.secrets.empty());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,http://not/container", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,,readwrite", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,https://azure/container,", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,https://azure/container,?sas", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy,https://azure/container,sas,readwrite", {});
REQUIRE(!parsed.has_value());
}
}

SECTION ("azcopy with SAS token")
{
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,sas", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
REQUIRE(state.azcopy_read_templates.size() == 1);
CHECK(state.azcopy_read_templates.front() == "https://azure/container/{sha}.zip?sas");
CHECK(state.azcopy_write_templates.empty());
REQUIRE(state.secrets == std::vector<std::string>{"sas"});
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,sas,read", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
REQUIRE(state.azcopy_read_templates.size() == 1);
CHECK(state.azcopy_read_templates.front() == "https://azure/container/{sha}.zip?sas");
CHECK(state.azcopy_write_templates.empty());
REQUIRE(state.secrets == std::vector<std::string>{"sas"});
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,sas,write", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
CHECK(state.azcopy_read_templates.empty());
REQUIRE(state.azcopy_write_templates.size() == 1);
CHECK(state.azcopy_write_templates.front() == "https://azure/container/{sha}.zip?sas");
REQUIRE(state.secrets == std::vector<std::string>{"sas"});
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,sas,readwrite", {});
auto state = parsed.value_or_exit(VCPKG_LINE_INFO);
REQUIRE(state.binary_cache_providers == std::set<StringLiteral>{{"azcopy"}, {"default"}});
REQUIRE(state.azcopy_read_templates.size() == 1);
CHECK(state.azcopy_read_templates.front() == "https://azure/container/{sha}.zip?sas");
REQUIRE(state.azcopy_write_templates.size() == 1);
CHECK(state.azcopy_write_templates.front() == "https://azure/container/{sha}.zip?sas");
REQUIRE(state.secrets == std::vector<std::string>{"sas"});
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,,sas,readwrite", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,http://not/container", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,?sas", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,,readwrite", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,sas,invalid", {});
REQUIRE(!parsed.has_value());
}
{
auto parsed = parse_binary_provider_configs("x-azcopy-sas,https://azure/container,sas,readwrite,extra", {});
REQUIRE(!parsed.has_value());
}
}
}

TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]")
{
{
Expand Down
Loading
Loading