Skip to content

Commit 79ea5be

Browse files
author
Achint Agrawal
committed
Merged with master
2 parents 9807af2 + 1bbe101 commit 79ea5be

File tree

52 files changed

+2698
-1474
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+2698
-1474
lines changed

‎Directory.Build.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<ClientOfficialVersion>3.37.1</ClientOfficialVersion>
44
<ClientPreviewVersion>3.37.1</ClientPreviewVersion>
55
<ClientPreviewSuffixVersion>preview</ClientPreviewSuffixVersion>
6-
<DirectVersion>3.31.5</DirectVersion>
6+
<DirectVersion>3.32.0</DirectVersion>
77
<EncryptionOfficialVersion>2.0.4</EncryptionOfficialVersion>
88
<EncryptionPreviewVersion>2.1.0</EncryptionPreviewVersion>
99
<EncryptionPreviewSuffixVersion>preview4</EncryptionPreviewSuffixVersion>

‎Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -898,6 +898,19 @@ public async Task EncryptionReadManyItemAsync()
898898
}
899899
}
900900

901+
// ISSUE-TODO-VipulVishal - This test passes locally, but often fails in pre-checkin validation.
902+
// Re-enable once the test is stabilized.
903+
// Here's one example failure for reference:
904+
// Test method Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.EncryptionChangeFeedDecryptionSuccessful threw exception:
905+
// System.NullReferenceException: Object reference not set to an instance of an object.
906+
// at Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.<>c__DisplayClass51_0.<ValidateChangeFeedProcessorResponse>b__2(TestDoc doc) in D:\a\1\s\Microsoft.Azure.Cosmos.Encryption\tests\EmulatorTests\MdeEncryptionTests.cs:line 3152
907+
// at System.Linq.Enumerable.WhereListIterator`1.MoveNext()
908+
// at System.Linq.Enumerable.TryGetFirst[TSource](IEnumerable`1 source, Boolean& found)
909+
// at System.Linq.Enumerable.FirstOrDefault[TSource] (IEnumerable`1 source)
910+
// at Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.ValidateChangeFeedProcessorResponse(Container container, TestDoc testDoc1, TestDoc testDoc2) in D:\a\1\s\Microsoft.Azure.Cosmos.Encryption\tests\EmulatorTests\MdeEncryptionTests.cs:line 3152
911+
// at Microsoft.Azure.Cosmos.Encryption.EmulatorTests.MdeEncryptionTests.EncryptionChangeFeedDecryptionSuccessful() in D:\a\1\s\Microsoft.Azure.Cosmos.Encryption\tests\EmulatorTests\MdeEncryptionTests.cs:line 911
912+
// at Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.ThreadOperations.ExecuteWithAbortSafety(Action action)
913+
[Ignore]
901914
[TestMethod]
902915
public async Task EncryptionChangeFeedDecryptionSuccessful()
903916
{

‎Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,17 @@ private async Task ProcessPartitionAsync(DocumentServiceLease lease)
161161
{
162162
DefaultTrace.TraceVerbose("Lease with token {0}: processing canceled", lease.CurrentLeaseToken);
163163
}
164+
catch (LeaseLostException leaseLostException)
165+
{
166+
// LeaseLostException by itself is not loggable, unless it contains a related inner exception
167+
// For cases when the lease or container has been deleted or the lease has been stolen
168+
if (leaseLostException.InnerException != null)
169+
{
170+
await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, leaseLostException.InnerException);
171+
}
172+
173+
DefaultTrace.TraceVerbose("Lease with token {0}: lease was lost", lease.CurrentLeaseToken);
174+
}
164175
catch (Exception ex)
165176
{
166177
await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex);

‎Microsoft.Azure.Cosmos/src/CosmosClient.cs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,9 @@ protected CosmosClient()
183183
/// </code>
184184
/// </example>
185185
/// <remarks>
186-
/// The returned reference doesn't guarantee credentials or connectivity validations because creation doesn't make any network calls.
186+
/// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;".
187+
/// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
188+
/// NOTE: DO NOT use this flag in production (only for emulator)
187189
/// </remarks>
188190
/// <seealso cref="CosmosClientOptions"/>
189191
/// <seealso cref="Fluent.CosmosClientBuilder"/>
@@ -195,7 +197,7 @@ public CosmosClient(
195197
: this(
196198
CosmosClientOptions.GetAccountEndpoint(connectionString),
197199
CosmosClientOptions.GetAccountKey(connectionString),
198-
clientOptions)
200+
CosmosClientOptions.GetCosmosClientOptionsWithCertificateFlag(connectionString, clientOptions))
199201
{
200202
}
201203

@@ -495,6 +497,11 @@ public static async Task<CosmosClient> CreateAndInitializeAsync(string accountEn
495497
/// ]]>
496498
/// </code>
497499
/// </example>
500+
/// <remarks>
501+
/// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;".
502+
/// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
503+
/// NOTE: DO NOT use this flag in production (only for emulator)
504+
/// </remarks>
498505
public static async Task<CosmosClient> CreateAndInitializeAsync(string connectionString,
499506
IReadOnlyList<(string databaseId, string containerId)> containers,
500507
CosmosClientOptions cosmosClientOptions = null,
@@ -504,6 +511,7 @@ public static async Task<CosmosClient> CreateAndInitializeAsync(string connectio
504511
{
505512
throw new ArgumentNullException(nameof(containers));
506513
}
514+
cosmosClientOptions = CosmosClientOptions.GetCosmosClientOptionsWithCertificateFlag(connectionString, cosmosClientOptions);
507515

508516
CosmosClient cosmosClient = new CosmosClient(connectionString,
509517
cosmosClientOptions);

‎Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ public class CosmosClientOptions
5151

5252
private const string ConnectionStringAccountEndpoint = "AccountEndpoint";
5353
private const string ConnectionStringAccountKey = "AccountKey";
54+
private const string ConnectionStringDisableServerCertificateValidation = "DisableServerCertificateValidation";
5455

5556
private const ApiType DefaultApiType = ApiType.None;
5657

@@ -278,8 +279,8 @@ public ConnectionMode ConnectionMode
278279
/// If this is not set the database account consistency level will be used for all requests.
279280
/// </summary>
280281
public ConsistencyLevel? ConsistencyLevel { get; set; }
281-
282-
/// <summary>
282+
283+
/// <summary>
283284
/// Sets the priority level for requests created using cosmos client.
284285
/// </summary>
285286
/// <remarks>
@@ -666,7 +667,9 @@ internal Protocol ConnectionProtocol
666667
/// </summary>
667668
/// <remarks>
668669
/// <para>
669-
/// Customizing SSL verification is not recommended in production environments.
670+
/// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;".
671+
/// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
672+
/// NOTE: DO NOT use this flag in production (only for emulator)
670673
/// </para>
671674
/// </remarks>
672675
public Func<X509Certificate2, X509Chain, SslPolicyErrors, bool> ServerCertificateCustomValidationCallback { get; set; }
@@ -744,6 +747,11 @@ internal Protocol ConnectionProtocol
744747
/// </summary>
745748
internal bool? EnableCpuMonitor { get; set; }
746749

750+
/// <summary>
751+
/// Flag indicates the value of DisableServerCertificateValidation flag set at connection string level.Default it is false.
752+
/// </summary>
753+
internal bool DisableServerCertificateValidation { get; set; }
754+
747755
/// <summary>
748756
/// Gets or sets Client Telemetry Options like feature flags and corresponding options
749757
/// </summary>
@@ -770,6 +778,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
770778
this.ValidateDirectTCPSettings();
771779
this.ValidateLimitToEndpointSettings();
772780
this.ValidatePartitionLevelFailoverSettings();
781+
this.ValidateAndSetServerCallbackSettings();
773782

774783
ConnectionPolicy connectionPolicy = new ConnectionPolicy()
775784
{
@@ -857,18 +866,34 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
857866

858867
return (Documents.ConsistencyLevel)this.ConsistencyLevel.Value;
859868
}
860-
869+
861870
internal static string GetAccountEndpoint(string connectionString)
862871
{
863-
return CosmosClientOptions.GetValueFromConnectionString(connectionString, CosmosClientOptions.ConnectionStringAccountEndpoint);
872+
return CosmosClientOptions.GetValueFromConnectionString<string>(connectionString, CosmosClientOptions.ConnectionStringAccountEndpoint, null);
864873
}
865874

866875
internal static string GetAccountKey(string connectionString)
867876
{
868-
return CosmosClientOptions.GetValueFromConnectionString(connectionString, CosmosClientOptions.ConnectionStringAccountKey);
877+
return CosmosClientOptions.GetValueFromConnectionString<string>(connectionString, CosmosClientOptions.ConnectionStringAccountKey, null);
878+
}
879+
880+
internal static bool IsConnectionStringDisableServerCertificateValidationFlag(string connectionString)
881+
{
882+
return Convert.ToBoolean(CosmosClientOptions.GetValueFromConnectionString<bool>(connectionString, CosmosClientOptions.ConnectionStringDisableServerCertificateValidation, false));
883+
}
884+
885+
internal static CosmosClientOptions GetCosmosClientOptionsWithCertificateFlag(string connectionString, CosmosClientOptions clientOptions)
886+
{
887+
clientOptions ??= new CosmosClientOptions();
888+
if (CosmosClientOptions.IsConnectionStringDisableServerCertificateValidationFlag(connectionString))
889+
{
890+
clientOptions.DisableServerCertificateValidation = true;
891+
}
892+
893+
return clientOptions;
869894
}
870895

871-
private static string GetValueFromConnectionString(string connectionString, string keyName)
896+
private static T GetValueFromConnectionString<T>(string connectionString, string keyName, T defaultValue)
872897
{
873898
if (connectionString == null)
874899
{
@@ -881,8 +906,20 @@ private static string GetValueFromConnectionString(string connectionString, stri
881906
string keyNameValue = value as string;
882907
if (!string.IsNullOrEmpty(keyNameValue))
883908
{
884-
return keyNameValue;
885-
}
909+
try
910+
{
911+
return (T)Convert.ChangeType(value, typeof(T));
912+
}
913+
catch (InvalidCastException)
914+
{
915+
throw new ArgumentException("The connection string contains invalid property: " + keyName);
916+
}
917+
}
918+
}
919+
920+
if (defaultValue != null)
921+
{
922+
return defaultValue;
886923
}
887924

888925
throw new ArgumentException("The connection string is missing a required property: " + keyName);
@@ -915,6 +952,19 @@ private void ValidatePartitionLevelFailoverSettings()
915952
}
916953
}
917954

955+
private void ValidateAndSetServerCallbackSettings()
956+
{
957+
if (this.DisableServerCertificateValidation && this.ServerCertificateCustomValidationCallback != null)
958+
{
959+
throw new ArgumentException($"Cannot specify {nameof(this.DisableServerCertificateValidation)} flag in Connection String and {nameof(this.ServerCertificateCustomValidationCallback)}. Only one can be set.");
960+
}
961+
962+
if (this.DisableServerCertificateValidation)
963+
{
964+
this.ServerCertificateCustomValidationCallback = (_, _, _) => true;
965+
}
966+
}
967+
918968
private void ValidateDirectTCPSettings()
919969
{
920970
string settingName = string.Empty;

‎Microsoft.Azure.Cosmos/src/DocumentClient.cs

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,9 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider
115115
private readonly bool IsLocalQuorumConsistency = false;
116116
private readonly bool isReplicaAddressValidationEnabled;
117117

118-
private readonly IChaosInterceptor chaosInterceptor;
118+
//Fault Injection
119+
private readonly IChaosInterceptorFactory chaosInterceptorFactory;
120+
private IChaosInterceptor chaosInterceptor;
119121

120122
//Auth
121123
internal readonly AuthorizationTokenProvider cosmosAuthorization;
@@ -487,7 +489,7 @@ internal DocumentClient(Uri serviceEndpoint,
487489
this.transportClientHandlerFactory = transportClientHandlerFactory;
488490
this.IsLocalQuorumConsistency = isLocalQuorumConsistency;
489491
this.initTaskCache = new AsyncCacheNonBlocking<string, bool>(cancellationToken: this.cancellationTokenSource.Token);
490-
this.chaosInterceptor = chaosInterceptorFactory?.CreateInterceptor(this);
492+
this.chaosInterceptorFactory = chaosInterceptorFactory;
491493

492494
this.Initialize(
493495
serviceEndpoint: serviceEndpoint,
@@ -1015,6 +1017,14 @@ internal virtual void Initialize(Uri serviceEndpoint,
10151017
// Always called from under the lock except when called from Intilialize method during construction.
10161018
private async Task<bool> GetInitializationTaskAsync(IStoreClientFactory storeClientFactory)
10171019
{
1020+
//Create the chaos interceptor if using fault injection
1021+
//Creating the chaos interceptor requires async calls, so we do it here instead of in the constructor
1022+
//This must also be done before creating the storeClientFactory for direct mode
1023+
if (this.chaosInterceptorFactory != null)
1024+
{
1025+
this.chaosInterceptor = await this.chaosInterceptorFactory.CreateInterceptorAsync(this);
1026+
}
1027+
10181028
await this.InitializeGatewayConfigurationReaderAsync();
10191029

10201030
if (this.desiredConsistencyLevel.HasValue)

‎Microsoft.Azure.Cosmos/src/FaultInjection/IChaosInterceptorFactory.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
//------------------------------------------------------------
44
namespace Microsoft.Azure.Cosmos
55
{
6+
using System.Threading.Tasks;
67
using Microsoft.Azure.Documents.FaultInjection;
78

89
/// <summary>
@@ -15,6 +16,6 @@ internal interface IChaosInterceptorFactory
1516
/// Creates the IChaosInterceptor interceptor that will be used to inject fault injection rules.
1617
/// </summary>
1718
/// <param name="documentClient"></param>
18-
public IChaosInterceptor CreateInterceptor(DocumentClient documentClient);
19+
public Task<IChaosInterceptor> CreateInterceptorAsync(DocumentClient documentClient);
1920
}
2021
}

‎Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ public CosmosClientBuilder(
124124
/// </summary>
125125
/// <example>"AccountEndpoint=https://mytestcosmosaccount.documents.azure.com:443/;AccountKey={SecretAccountKey};"</example>
126126
/// <param name="connectionString">The connection string must contain AccountEndpoint and AccountKey or ResourceToken.</param>
127+
/// <remarks>
128+
/// Emulator: To ignore SSL Certificate please suffix connectionstring with "DisableServerCertificateValidation=True;".
129+
/// When CosmosClientOptions.HttpClientFactory is used, SSL certificate needs to be handled appropriately.
130+
/// NOTE: DO NOT use this flag in production (only for emulator)
131+
/// </remarks>
127132
public CosmosClientBuilder(string connectionString)
128133
{
129134
if (connectionString == null)
@@ -133,6 +138,8 @@ public CosmosClientBuilder(string connectionString)
133138

134139
this.accountEndpoint = CosmosClientOptions.GetAccountEndpoint(connectionString);
135140
this.accountKey = CosmosClientOptions.GetAccountKey(connectionString);
141+
142+
this.clientOptions = CosmosClientOptions.GetCosmosClientOptionsWithCertificateFlag(connectionString, this.clientOptions);
136143
}
137144

138145
/// <summary>

‎Microsoft.Azure.Cosmos/src/Handler/ResponseMessage.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ internal ResponseMessage(
8383
this.CosmosException = cosmosException;
8484
this.Headers = headers ?? new Headers();
8585

86-
this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(this.Headers, true);
86+
this.IndexUtilizationText = ResponseMessage.DecodeIndexMetrics(this.Headers, isBase64Encoded: true);
8787

8888
if (requestMessage != null && requestMessage.Trace != null)
8989
{
@@ -269,9 +269,9 @@ static internal Lazy<string> DecodeIndexMetrics(Headers responseMessageHeaders,
269269

270270
return stringBuilder.ToString();
271271
}
272-
273-
// Return the JSON from the response header
274-
return responseMessageHeaders.IndexUtilizationText;
272+
273+
// Return the JSON from the response header after url decode
274+
return System.Web.HttpUtility.UrlDecode(responseMessageHeaders.IndexUtilizationText, Encoding.UTF8);
275275
});
276276
}
277277

‎Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Aggregate/Aggregators/AggregateItem.cs

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.Aggregate.Aggregators
66
{
77
using System;
8+
using System.Text.RegularExpressions;
89
using Microsoft.Azure.Cosmos.CosmosElements;
910

1011
internal readonly struct AggregateItem
@@ -16,17 +17,31 @@ internal readonly struct AggregateItem
1617

1718
public AggregateItem(CosmosElement cosmosElement)
1819
{
19-
if (cosmosElement == null)
20-
{
21-
throw new ArgumentNullException($"{nameof(cosmosElement)} must not be null.");
22-
}
20+
// If the query is not a select value query then the top level is a an object
21+
CosmosObject cosmosObject = cosmosElement as CosmosObject;
2322

24-
if (!(cosmosElement is CosmosObject cosmosObject))
23+
if (cosmosObject == null)
2524
{
26-
throw new ArgumentException($"{nameof(cosmosElement)} must not be an object.");
25+
// In case of Aggregate query with VALUE query plan, the top level is an array of one item after it is rewritten
26+
// For example, if the query is
27+
// SELECT VALUE {"age": c.age}
28+
// FROM c
29+
// GROUP BY c.age
30+
// Fhe rewritten query is
31+
// SELECT [{"item": c.age}] AS groupByItems, {"age": c.age} AS payload
32+
// FROM c
33+
// GROUP BY c.age
34+
35+
// In this case, the top level is an array of one item [{"item": c.age}]
36+
CosmosArray cosmosArray = cosmosElement as CosmosArray;
37+
if (cosmosArray.Count == 1)
38+
{
39+
cosmosObject = cosmosArray[0] as CosmosObject;
40+
}
2741
}
2842

29-
this.cosmosObject = cosmosObject;
43+
// If the object is still null, then we have an invalid aggregate item
44+
this.cosmosObject = cosmosObject ?? throw new ArgumentException($"Unsupported aggregate item. Expected CosmosObject");
3045
}
3146

3247
public CosmosElement Item

0 commit comments

Comments
 (0)