Skip to content

Commit aab02e5

Browse files
philipthomas-MSFTealsur
philipthomas-MSFT
and
ealsur
authored
Change Feed / Processor AVAD: Fixes timeToLiveExpired missing from metadata (#4523)
* longrunning cfp avad test for ttl * add more to comment * internal set * other tests fail with higher ms * run updatecontracts * Update Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs Co-authored-by: Matias Quaranta <[email protected]> * Update Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs Co-authored-by: Matias Quaranta <[email protected]> * Update Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs Co-authored-by: Matias Quaranta <[email protected]> * Update Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs Co-authored-by: Matias Quaranta <[email protected]> * using Logger.LogLine * change back over to ManualResetEvent. fixed assert to look at Previous. keeping stopwatch just for logging. timeout at 5 minutes. * Update GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs Co-authored-by: Matias Quaranta <[email protected]> * try/finally * removing ctor. making all set internal to address serialization issue. later PRs to test STJ de/serialization. * fixing de/serialziation issue * Update GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs change timeout * Update ReEncryption.csproj 1.11.4 * internal set change to CanWrite:True; --------- Co-authored-by: Matias Quaranta <[email protected]>
1 parent c5ac103 commit aab02e5

File tree

3 files changed

+125
-31
lines changed

3 files changed

+125
-31
lines changed

‎Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,55 +19,36 @@ namespace Microsoft.Azure.Cosmos
1919
#endif
2020
class ChangeFeedMetadata
2121
{
22-
/// <summary>
23-
/// New instance of meta data for <see cref="ChangeFeedItem{T}"/> created.
24-
/// </summary>
25-
/// <param name="conflictResolutionTimestamp"></param>
26-
/// <param name="lsn"></param>
27-
/// <param name="operationType"></param>
28-
/// <param name="previousLsn"></param>
29-
public ChangeFeedMetadata(
30-
DateTime conflictResolutionTimestamp,
31-
long lsn,
32-
ChangeFeedOperationType operationType,
33-
long previousLsn)
34-
{
35-
this.ConflictResolutionTimestamp = conflictResolutionTimestamp;
36-
this.Lsn = lsn;
37-
this.OperationType = operationType;
38-
this.PreviousLsn = previousLsn;
39-
}
40-
4122
/// <summary>
4223
/// The conflict resolution timestamp.
4324
/// </summary>
4425
[JsonProperty(PropertyName = "crts", NullValueHandling = NullValueHandling.Ignore)]
4526
[JsonConverter(typeof(UnixDateTimeConverter))]
46-
public DateTime ConflictResolutionTimestamp { get; }
27+
public DateTime ConflictResolutionTimestamp { get; internal set; }
4728

4829
/// <summary>
4930
/// The current logical sequence number.
5031
/// </summary>
5132
[JsonProperty(PropertyName = "lsn", NullValueHandling = NullValueHandling.Ignore)]
52-
public long Lsn { get; }
33+
public long Lsn { get; internal set; }
5334

5435
/// <summary>
5536
/// The change feed operation type.
5637
/// </summary>
5738
[JsonProperty(PropertyName = "operationType")]
5839
[JsonConverter(typeof(StringEnumConverter))]
59-
public ChangeFeedOperationType OperationType { get; }
40+
public ChangeFeedOperationType OperationType { get; internal set; }
6041

6142
/// <summary>
6243
/// The previous logical sequence number.
6344
/// </summary>
6445
[JsonProperty(PropertyName = "previousImageLSN", NullValueHandling = NullValueHandling.Ignore)]
65-
public long PreviousLsn { get; }
46+
public long PreviousLsn { get; internal set; }
6647

6748
/// <summary>
6849
/// Used to distinquish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents).
6950
/// </summary>
7051
[JsonProperty(PropertyName = "timeToLiveExpired", NullValueHandling= NullValueHandling.Ignore)]
71-
public bool IsTimeToLiveExpired { get; }
52+
public bool IsTimeToLiveExpired { get; internal set; }
7253
}
7354
}

‎Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.cs

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,14 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests.ChangeFeed
66
{
77
using System;
88
using System.Collections.Generic;
9+
using System.Diagnostics;
910
using System.Linq;
1011
using System.Threading;
1112
using System.Threading.Tasks;
1213
using Microsoft.Azure.Cosmos.ChangeFeed.Utils;
14+
using Microsoft.Azure.Cosmos.Services.Management.Tests;
1315
using Microsoft.VisualStudio.TestTools.UnitTesting;
16+
using Newtonsoft.Json;
1417
using Newtonsoft.Json.Linq;
1518

1619
[TestClass]
@@ -29,6 +32,115 @@ public async Task Cleanup()
2932
await base.TestCleanup();
3033
}
3134

35+
[TestMethod]
36+
[Timeout(300000)]
37+
[TestCategory("LongRunning")]
38+
[Owner("philipthomas-MSFT")]
39+
[Description("Scenario: When a document is created with ttl set, there should be 1 create and 1 delete that will appear for that " +
40+
"document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode.")]
41+
public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsAsync()
42+
{
43+
ContainerInternal monitoredContainer = await this.CreateMonitoredContainer(ChangeFeedMode.AllVersionsAndDeletes);
44+
Exception exception = default;
45+
int ttlInSeconds = 5;
46+
Stopwatch stopwatch = new();
47+
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
48+
49+
ChangeFeedProcessor processor = monitoredContainer
50+
.GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(processorName: "processor", onChangesDelegate: (ChangeFeedProcessorContext context, IReadOnlyCollection<ChangeFeedItem<dynamic>> docs, CancellationToken token) =>
51+
{
52+
// NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator.
53+
54+
Logger.LogLine($"@ {DateTime.Now}, {nameof(stopwatch)} -> CFP AVAD took '{stopwatch.ElapsedMilliseconds}' to read document CRUD in feed.");
55+
Logger.LogLine($"@ {DateTime.Now}, {nameof(docs)} -> {JsonConvert.SerializeObject(docs)}");
56+
57+
foreach (ChangeFeedItem<dynamic> change in docs)
58+
{
59+
if (change.Metadata.OperationType == ChangeFeedOperationType.Create)
60+
{
61+
// current
62+
Assert.AreEqual(expected: "1", actual: change.Current.id.ToString());
63+
Assert.AreEqual(expected: "1", actual: change.Current.pk.ToString());
64+
Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Current.description.ToString());
65+
Assert.AreEqual(expected: ttlInSeconds, actual: change.Current.ttl.ToObject<int>());
66+
67+
// metadata
68+
Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value.");
69+
Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value.");
70+
Assert.IsFalse(change.Metadata.IsTimeToLiveExpired);
71+
72+
// previous
73+
Assert.IsNull(change.Previous);
74+
}
75+
else if (change.Metadata.OperationType == ChangeFeedOperationType.Delete)
76+
{
77+
// current
78+
Assert.IsNull(change.Current.id);
79+
80+
// metadata
81+
Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value.");
82+
Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value.");
83+
Assert.IsTrue(change.Metadata.IsTimeToLiveExpired);
84+
85+
// previous
86+
Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString());
87+
Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString());
88+
Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString());
89+
Assert.AreEqual(expected: ttlInSeconds, actual: change.Previous.ttl.ToObject<int>());
90+
91+
// stop after reading delete since it is the last document in feed.
92+
stopwatch.Stop();
93+
allDocsProcessed.Set();
94+
}
95+
else
96+
{
97+
Assert.Fail("Invalid operation.");
98+
}
99+
}
100+
101+
return Task.CompletedTask;
102+
})
103+
.WithInstanceName(Guid.NewGuid().ToString())
104+
.WithLeaseContainer(this.LeaseContainer)
105+
.WithErrorNotification((leaseToken, error) =>
106+
{
107+
exception = error.InnerException;
108+
109+
return Task.CompletedTask;
110+
})
111+
.Build();
112+
113+
stopwatch.Start();
114+
115+
// NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator.
116+
117+
Logger.LogLine($"@ {DateTime.Now}, CFProcessor starting...");
118+
119+
await processor.StartAsync();
120+
121+
try
122+
{
123+
await Task.Delay(GetChangeFeedProcessorBuilderWithAllVersionsAndDeletesTests.ChangeFeedSetupTime);
124+
await monitoredContainer.CreateItemAsync<dynamic>(new { id = "1", pk = "1", description = "Testing TTL on CFP.", ttl = ttlInSeconds }, partitionKey: new PartitionKey("1"));
125+
126+
// NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator.
127+
128+
Logger.LogLine($"@ {DateTime.Now}, Document created.");
129+
130+
bool receivedDelete = allDocsProcessed.WaitOne(250000);
131+
Assert.IsTrue(receivedDelete, "Timed out waiting for docs to process");
132+
133+
if (exception != default)
134+
{
135+
Assert.Fail(exception.ToString());
136+
}
137+
}
138+
finally
139+
{
140+
await processor.StopAsync();
141+
}
142+
}
143+
32144
[TestMethod]
33145
[Owner("philipthomas-MSFT")]
34146
[Description("Scenario: When a document is created, then updated, and finally deleted, there should be 3 changes that will appear for that " +
@@ -467,6 +579,7 @@ private async Task<ContainerInternal> CreateMonitoredContainer(ChangeFeedMode ch
467579
if (changeFeedMode == ChangeFeedMode.AllVersionsAndDeletes)
468580
{
469581
properties.ChangeFeedPolicy.FullFidelityRetention = TimeSpan.FromMinutes(5);
582+
properties.DefaultTimeToLive = -1;
470583
}
471584

472585
ContainerResponse response = await this.database.CreateContainerAsync(properties,

‎Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@
8989
"Attributes": [
9090
"JsonPropertyAttribute"
9191
],
92-
"MethodInfo": "Boolean IsTimeToLiveExpired;CanRead:True;CanWrite:False;Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
92+
"MethodInfo": "Boolean IsTimeToLiveExpired;CanRead:True;CanWrite:True;Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
9393
},
9494
"Int64 get_Lsn()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": {
9595
"Type": "Method",
@@ -110,14 +110,14 @@
110110
"Attributes": [
111111
"JsonPropertyAttribute"
112112
],
113-
"MethodInfo": "Int64 Lsn;CanRead:True;CanWrite:False;Int64 get_Lsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
113+
"MethodInfo": "Int64 Lsn;CanRead:True;CanWrite:True;Int64 get_Lsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
114114
},
115115
"Int64 PreviousLsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"previousImageLSN\")]": {
116116
"Type": "Property",
117117
"Attributes": [
118118
"JsonPropertyAttribute"
119119
],
120-
"MethodInfo": "Int64 PreviousLsn;CanRead:True;CanWrite:False;Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
120+
"MethodInfo": "Int64 PreviousLsn;CanRead:True;CanWrite:True;Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
121121
},
122122
"Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": {
123123
"Type": "Method",
@@ -132,15 +132,15 @@
132132
"JsonConverterAttribute",
133133
"JsonPropertyAttribute"
134134
],
135-
"MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:False;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
135+
"MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
136136
},
137137
"System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": {
138138
"Type": "Property",
139139
"Attributes": [
140140
"JsonConverterAttribute",
141141
"JsonPropertyAttribute"
142142
],
143-
"MethodInfo": "System.DateTime ConflictResolutionTimestamp;CanRead:True;CanWrite:False;System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
143+
"MethodInfo": "System.DateTime ConflictResolutionTimestamp;CanRead:True;CanWrite:True;System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
144144
},
145145
"System.DateTime get_ConflictResolutionTimestamp()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": {
146146
"Type": "Method",
@@ -149,10 +149,10 @@
149149
],
150150
"MethodInfo": "System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
151151
},
152-
"Void .ctor(System.DateTime, Int64, Microsoft.Azure.Cosmos.ChangeFeedOperationType, Int64)": {
152+
"Void .ctor()": {
153153
"Type": "Constructor",
154154
"Attributes": [],
155-
"MethodInfo": "[Void .ctor(System.DateTime, Int64, Microsoft.Azure.Cosmos.ChangeFeedOperationType, Int64), Void .ctor(System.DateTime, Int64, Microsoft.Azure.Cosmos.ChangeFeedOperationType, Int64)]"
155+
"MethodInfo": "[Void .ctor(), Void .ctor()]"
156156
}
157157
},
158158
"NestedTypes": {}

0 commit comments

Comments
 (0)