Merge pull request #14170 from lovesegfault/curl-based-s3-pieces

feat(libstore/filetransfer): add S3 signing support

Author: Ericson2314

src/libstore/aws-creds.cc

@@ -24,6 +24,22 @@ namespace nix {
 
 namespace {
 
+// Global credential provider cache using boost's concurrent map
+// Key: profile name (empty string for default profile)
+using CredentialProviderCache =
+    boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>;
+
+static CredentialProviderCache credentialProviderCache;
+
+/**
+ * Clear all cached credential providers.
+ * Called automatically by CrtWrapper destructor during static destruction.
+ */
+static void clearAwsCredentialsCache()
+{
+    credentialProviderCache.clear();
+}
+
 static void initAwsCrt()
 {
     struct CrtWrapper
@@ -95,13 +111,6 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth:
     return fut.get(); // This will throw if set_exception was called
 }
 
-// Global credential provider cache using boost's concurrent map
-// Key: profile name (empty string for default profile)
-using CredentialProviderCache =
-    boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>;
-
-static CredentialProviderCache credentialProviderCache;
-
 } // anonymous namespace
 
 AwsCredentials getAwsCredentials(const std::string & profile)
@@ -160,11 +169,6 @@ void invalidateAwsCredentials(const std::string & profile)
     credentialProviderCache.erase(profile);
 }
 
-void clearAwsCredentialsCache()
-{
-    credentialProviderCache.clear();
-}
-
 AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url)
 {
     std::string profile = s3Url.profile.value_or("");

src/libstore/filetransfer.cc

@@ -9,9 +9,14 @@
 #include "nix/util/signals.hh"
 
 #include "store-config-private.hh"
+#include <optional>
 #if NIX_WITH_S3_SUPPORT
 #  include <aws/core/client/ClientConfiguration.h>
 #endif
+#if NIX_WITH_CURL_S3
+#  include "nix/store/aws-creds.hh"
+#  include "nix/store/s3-url.hh"
+#endif
 
 #ifdef __linux__
 #  include "nix/util/linux-namespaces.hh"
@@ -426,6 +431,24 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf);
             errbuf[0] = 0;
 
+            // Set up username/password authentication if provided
+            if (request.usernameAuth) {
+                curl_easy_setopt(req, CURLOPT_USERNAME, request.usernameAuth->username.c_str());
+                if (request.usernameAuth->password) {
+                    curl_easy_setopt(req, CURLOPT_PASSWORD, request.usernameAuth->password->c_str());
+                }
+            }
+
+#if NIX_WITH_CURL_S3
+            // Set up AWS SigV4 signing if this is an S3 request
+            // Note: AWS SigV4 support guaranteed available (curl >= 7.75.0 checked at build time)
+            // The username/password (access key ID and secret key) are set via the general
+            // usernameAuth mechanism above.
+            if (request.awsSigV4Provider) {
+                curl_easy_setopt(req, CURLOPT_AWS_SIGV4, request.awsSigV4Provider->c_str());
+            }
+#endif
+
             result.data.clear();
             result.bodySize = 0;
         }
@@ -800,7 +823,11 @@ struct curlFileTransfer : public FileTransfer
 
     void enqueueItem(std::shared_ptr<TransferItem> item)
     {
-        if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https")
+        if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
+#if NIX_WITH_CURL_S3
+            && item->request.uri.scheme() != "s3"
+#endif
+        )
             throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string());
 
         {
@@ -818,9 +845,15 @@ struct curlFileTransfer : public FileTransfer
     {
         /* Ugly hack to support s3:// URIs. */
         if (request.uri.scheme() == "s3") {
+#if NIX_WITH_CURL_S3
+            // New curl-based S3 implementation
+            auto modifiedRequest = request;
+            modifiedRequest.setupForS3();
+            enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
+#elif NIX_WITH_S3_SUPPORT
+            // Old AWS SDK-based implementation
             // FIXME: do this on a worker thread
             try {
-#if NIX_WITH_S3_SUPPORT
                 auto parsed = ParsedS3URL::parse(request.uri.parsed());
 
                 std::string profile = parsed.profile.value_or("");
@@ -838,13 +871,12 @@ struct curlFileTransfer : public FileTransfer
                 res.data = std::move(*s3Res.data);
                 res.urls.push_back(request.uri.to_string());
                 callback(std::move(res));
-#else
-                throw nix::Error(
-                    "cannot download '%s' because Nix is not built with S3 support", request.uri.to_string());
-#endif
             } catch (...) {
                 callback.rethrow();
             }
+#else
+            throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri.to_string());
+#endif
             return;
         }
 
@@ -872,6 +904,41 @@ ref<FileTransfer> makeFileTransfer()
     return makeCurlFileTransfer();
 }
 
+#if NIX_WITH_CURL_S3
+void FileTransferRequest::setupForS3()
+{
+    auto parsedS3 = ParsedS3URL::parse(uri.parsed());
+    // Update the request URI to use HTTPS
+    uri = parsedS3.toHttpsUrl();
+    // This gets used later in a curl setopt
+    awsSigV4Provider = "aws:amz:" + parsedS3.region.value_or("us-east-1") + ":s3";
+    // check if the request already has pre-resolved credentials
+    std::optional<std::string> sessionToken;
+    if (usernameAuth) {
+        debug("Using pre-resolved AWS credentials from parent process");
+        sessionToken = preResolvedAwsSessionToken;
+    } else {
+        std::string profile = parsedS3.profile.value_or("");
+        try {
+            auto creds = getAwsCredentials(profile);
+            usernameAuth = UsernameAuth{
+                .username = creds.accessKeyId,
+                .password = creds.secretAccessKey,
+            };
+            sessionToken = creds.sessionToken;
+        } catch (const AwsAuthError & e) {
+            warn("AWS authentication failed for S3 request %s: %s", uri, e.what());
+            // Invalidate the cached credentials so next request will retry
+            invalidateAwsCredentials(profile);
+            // Continue without authentication - might be a public bucket
+            return;
+        }
+    }
+    if (sessionToken)
+        headers.emplace_back("x-amz-security-token", *sessionToken);
+}
+#endif
+
 std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
 {
     auto promise = std::make_shared<std::promise<FileTransferResult>>();

src/libstore/include/nix/store/aws-creds.hh

@@ -57,12 +57,6 @@ AwsCredentials getAwsCredentials(const std::string & profile = "");
  */
 void invalidateAwsCredentials(const std::string & profile);
 
-/**
- * Clear all cached credential providers.
- * Typically called during application cleanup.
- */
-void clearAwsCredentialsCache();
-
 /**
  * Pre-resolve AWS credentials for S3 URLs.
  * Used to cache credentials in parent process before forking.

src/libstore/include/nix/store/filetransfer.hh

@@ -11,6 +11,11 @@
 #include "nix/util/serialise.hh"
 #include "nix/util/url.hh"
 
+#include "nix/store/config.hh"
+#if NIX_WITH_CURL_S3
+#  include "nix/store/aws-creds.hh"
+#endif
+
 namespace nix {
 
 struct FileTransferSettings : Config
@@ -77,6 +82,17 @@ extern FileTransferSettings fileTransferSettings;
 
 extern const unsigned int RETRY_TIME_MS_DEFAULT;
 
+/**
+ * Username and optional password for HTTP basic authentication.
+ * These are used with curl's CURLOPT_USERNAME and CURLOPT_PASSWORD options
+ * for various protocols including HTTP, FTP, and others.
+ */
+struct UsernameAuth
+{
+    std::string username;
+    std::optional<std::string> password;
+};
+
 struct FileTransferRequest
 {
     ValidURL uri;
@@ -92,6 +108,18 @@ struct FileTransferRequest
     std::optional<std::string> data;
     std::string mimeType;
     std::function<void(std::string_view data)> dataCallback;
+    /**
+     * Optional username and password for HTTP basic authentication.
+     * When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option.
+     */
+    std::optional<UsernameAuth> usernameAuth;
+#if NIX_WITH_CURL_S3
+    /**
+     * Pre-resolved AWS session token for S3 requests.
+     * When provided along with usernameAuth, this will be used instead of fetching fresh credentials.
+     */
+    std::optional<std::string> preResolvedAwsSessionToken;
+#endif
 
     FileTransferRequest(ValidURL uri)
         : uri(std::move(uri))
@@ -103,6 +131,13 @@ struct FileTransferRequest
     {
         return data ? "upload" : "download";
     }
+
+#if NIX_WITH_CURL_S3
+private:
+    friend struct curlFileTransfer;
+    void setupForS3();
+    std::optional<std::string> awsSigV4Provider;
+#endif
 };
 
 struct FileTransferResult