Hedging: Adds support for writes on multi region accounts

Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/AvailabilityStrategy.cs

@@ -39,11 +39,17 @@ internal static AvailabilityStrategy DisabledStrategy()
         /// </summary>
         /// <param name="threshold"> how long before SDK begins hedging</param>
         /// <param name="thresholdStep">Period of time between first hedge and next hedging attempts</param>
+        /// <param name="enableMultiWriteRegionHedge">Whether hedging for write requests on accounts with multi-region writes are enabled
+        /// Note that this does come with the caveat that there will be more 409 errors thrown by the SDK.
+        /// This is expected and applications that adapt this feature should be prepared to handle these exceptions.
+        /// Application might not be able to be deterministic on Create vs Replace in the case of Upsert Operations</param>
         /// <returns>something</returns>
-        public static AvailabilityStrategy CrossRegionHedgingStrategy(TimeSpan threshold,
-            TimeSpan? thresholdStep)
+        public static AvailabilityStrategy CrossRegionHedgingStrategy(
+            TimeSpan threshold,
+            TimeSpan? thresholdStep,
+            bool enableMultiWriteRegionHedge = false)
         {
-            return new CrossRegionHedgingAvailabilityStrategy(threshold, thresholdStep);
+            return new CrossRegionHedgingAvailabilityStrategy(threshold, thresholdStep, enableMultiWriteRegionHedge);
         }
     }
 }

Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs

@@ -36,14 +36,24 @@ internal class CrossRegionHedgingAvailabilityStrategy : AvailabilityStrategyInte
         /// </summary>
         public TimeSpan ThresholdStep { get; private set; }
 
+        /// <summary>
+        /// Whether hedging for write requests on accounts with multi-region writes are enabled
+        /// Note that this does come with the caveat that there will be more 409 errors thrown by the SDK.
+        /// This is expected and applications that adapt this feature should be prepared to handle these exceptions.
+        /// Application might not be able to be deterministic on Create vs Replace in the case of Upsert Operations
+        /// </summary>
+        public bool EnableMultiWriteRegionHedge { get; private set; }
+
         /// <summary>
         /// Constructor for hedging availability strategy
         /// </summary>
         /// <param name="threshold"></param>
         /// <param name="thresholdStep"></param>
+        /// <param name="enableMultiWriteRegionHedge"></param>
         public CrossRegionHedgingAvailabilityStrategy(
             TimeSpan threshold,
-            TimeSpan? thresholdStep)
+            TimeSpan? thresholdStep,
+            bool enableMultiWriteRegionHedge = false)
         {
             if (threshold <= TimeSpan.Zero)
             {
@@ -57,6 +67,7 @@ public CrossRegionHedgingAvailabilityStrategy(
 
             this.Threshold = threshold;
             this.ThresholdStep = thresholdStep ?? TimeSpan.FromMilliseconds(-1);
+            this.EnableMultiWriteRegionHedge = enableMultiWriteRegionHedge;
         }
 
         /// <inheritdoc/>
@@ -70,18 +81,24 @@ internal override bool Enabled()
         /// This availability strategy can only be used if the request is a read-only request on a document request.
         /// </summary>
         /// <param name="request"></param>
+        /// <param name="client"></param>
         /// <returns>whether the request should be a hedging request.</returns>
-        internal bool ShouldHedge(RequestMessage request)
+        internal bool ShouldHedge(RequestMessage request, CosmosClient client)
         {
             //Only use availability strategy for document point operations
             if (request.ResourceType != ResourceType.Document)
             {
                 return false;
             }
 
-            //check to see if it is a not a read-only request
+            //check to see if it is a not a read-only request/ if multimaster writes are enabled
             if (!OperationTypeExtensions.IsReadOperation(request.OperationType))
             {
+                if (this.EnableMultiWriteRegionHedge
+                    && client.DocumentClient.GlobalEndpointManager.CanSupportMultipleWriteLocations(request))
+                {
+                    return true;
+                }
                 return false;
             }
 
@@ -102,7 +119,7 @@ internal override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
             RequestMessage request,
             CancellationToken cancellationToken)
         {
-            if (!this.ShouldHedge(request)
+            if (!this.ShouldHedge(request, client)
                 || client.DocumentClient.GlobalEndpointManager.ReadEndpoints.Count == 1)
             {
                 return await sender(request, cancellationToken);
@@ -113,7 +130,7 @@ internal override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
             using (CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
             {
                 using (CloneableStream clonedBody = (CloneableStream)(request.Content == null
-                    ? null//new CloneableStream(new MemoryStream())
+                    ? null
                     : await StreamExtension.AsClonableStreamAsync(request.Content)))
                 {
                     IReadOnlyCollection<string> hedgeRegions = client.DocumentClient.GlobalEndpointManager

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -555,6 +555,14 @@ public bool CanSupportMultipleWriteLocations(DocumentServiceRequest request)
                 && (request.ResourceType == ResourceType.Document ||
                 (request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.Execute));
         }
+
+        public bool CanSupportMultipleWriteLocations(RequestMessage request)
+        {
+            return this.locationCache.CanUseMultipleWriteLocations()
+                && this.locationCache.GetAvailableAccountLevelWriteLocations()?.Count > 1
+                && (request.ResourceType == ResourceType.Document ||
+                (request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.Execute));
+        }
 
 #pragma warning disable VSTHRD100 // Avoid async void methods
         private async void StartLocationBackgroundRefreshLoop()