Letting cassandra max retry delay be configured

Author: rkargMsft

src/Cassandra/Orleans.Clustering.Cassandra/CassandraClusteringTable.cs

@@ -49,10 +49,10 @@ async Task IMembershipTable.InitializeMembershipTable(bool tryInitTableVersion)
 
         _queries = await OrleansQueries.CreateInstance(_session);
 
-        await _queries.EnsureTableExistsAsync(_ttlSeconds);
+        await _queries.EnsureTableExistsAsync(_options.InitializeRetryMaxDelay, _ttlSeconds);
 
         if (tryInitTableVersion)
-            await _queries.EnsureClusterVersionExistsAsync(_identifier);
+            await _queries.EnsureClusterVersionExistsAsync(_options.InitializeRetryMaxDelay, _identifier);
     }
 
     async Task IMembershipTable.DeleteMembershipTableEntries(string clusterId)

src/Cassandra/Orleans.Clustering.Cassandra/CassandraGatewayListProvider.cs

@@ -57,7 +57,7 @@ async Task IGatewayListProvider.InitializeGatewayListProvider()
 
         _queries = await OrleansQueries.CreateInstance(_session);
 
-        await _queries.EnsureTableExistsAsync(_ttlSeconds);
+        await _queries.EnsureTableExistsAsync(_options.InitializeRetryMaxDelay, _ttlSeconds);
     }
 
     async Task<IList<Uri>> IGatewayListProvider.GetGateways()

src/Cassandra/Orleans.Clustering.Cassandra/Hosting/CassandraClusteringOptions.cs

@@ -27,6 +27,14 @@ public class CassandraClusteringOptions
     /// </remarks>
     public bool UseCassandraTtl { get; set; }
 
+    /// <summary>
+    /// Specifies the maximum amount of time to wait after encountering
+    /// contention during initialization before retrying.
+    /// </summary>
+    /// <remarks>This is generally only encountered with large numbers of silos connecting
+    /// in a short time period and using multi-datacenter Cassandara clusters</remarks>
+    public TimeSpan InitializeRetryMaxDelay { get; set; } = TimeSpan.FromSeconds(20);
+
     internal int? GetCassandraTtlSeconds(ClusterMembershipOptions clusterMembershipOptions) =>
         UseCassandraTtl
             ? Convert.ToInt32(

src/Cassandra/Orleans.Clustering.Cassandra/OrleansQueries.cs

@@ -40,7 +40,7 @@ private OrleansQueries(ISession session)
         Session = session;
     }
 
-    internal async Task EnsureTableExistsAsync(int? ttl)
+    internal async Task EnsureTableExistsAsync(TimeSpan maxRetryDelay, int? ttl)
     {
         if (!await DoesTableAlreadyExistAsync())
         {
@@ -52,7 +52,7 @@ internal async Task EnsureTableExistsAsync(int? ttl)
             {
                 // Randomize the delay to avoid contention, preferring that more instances will wait longer
                 var nextSingle = Random.Shared.NextSingle();
-                await Task.Delay(TimeSpan.FromSeconds(20) * Math.Sqrt(nextSingle));
+                await Task.Delay(maxRetryDelay * Math.Sqrt(nextSingle));
 
                 if (!await DoesTableAlreadyExistAsync())
                 {
@@ -62,7 +62,7 @@ internal async Task EnsureTableExistsAsync(int? ttl)
         }
     }
 
-    internal async Task EnsureClusterVersionExistsAsync(string clusterIdentifier)
+    internal async Task EnsureClusterVersionExistsAsync(TimeSpan maxRetryDelay, string clusterIdentifier)
     {
         if (!await DoesClusterVersionAlreadyExistAsync(clusterIdentifier))
         {
@@ -74,7 +74,7 @@ internal async Task EnsureClusterVersionExistsAsync(string clusterIdentifier)
             {
                 // Randomize the delay to avoid contention, preferring that more instances will wait longer
                 var nextSingle = Random.Shared.NextSingle();
-                await Task.Delay(TimeSpan.FromSeconds(20) * Math.Sqrt(nextSingle));
+                await Task.Delay(maxRetryDelay * Math.Sqrt(nextSingle));
 
                 if (!await DoesClusterVersionAlreadyExistAsync(clusterIdentifier))
                 {