Skip to content

feature: add config client undo data validation ignore column compare #4317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

feature: add config client undo data validation ignore column compare #4317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions core/src/main/java/io/seata/core/constants/ConfigurationKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,10 @@ public interface ConfigurationKeys {
* The constant TRANSACTION_UNDO_DATA_VALIDATION.
*/
String TRANSACTION_UNDO_DATA_VALIDATION = CLIENT_UNDO_PREFIX + "dataValidation";
/**
* The constant TRANSACTION_UNDO_DATA_VALIDATION_IGNORE_COLUMNS.
*/
String TRANSACTION_UNDO_DATA_VALIDATION_IGNORE_COLUMNS = CLIENT_UNDO_PREFIX + "dataValidationIgnoreColumns";
/**
* The constant TRANSACTION_UNDO_LOG_SERIALIZATION.
*/
Expand Down
74 changes: 67 additions & 7 deletions rm-datasource/src/main/java/io/seata/rm/datasource/DataCompareUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,7 @@
import java.sql.Timestamp;
import java.sql.Types;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.HashMap;
import java.util.Comparator;
import java.util.*;
import java.util.stream.Collectors;

/**
Expand Down Expand Up @@ -87,7 +83,6 @@ public static Result<Boolean> isFieldEquals(Field f0, Field f1) {
}
}
}

private static void convertType(Field f0, Field f1) {
int f0Type = f0.getType();
int f1Type = f1.getType();
Expand Down Expand Up @@ -141,7 +136,33 @@ public static Result<Boolean> isRecordsEquals(TableRecords beforeImage, TableRec
}
}
}

/**
* Is records equals result.
*
* @param beforeImage the before image
* @param afterImage the after image
* @param ignoreColumns field compare ignore columns,comma separated string
* @return the result
*/
public static Result<Boolean> isRecordsEquals(TableRecords beforeImage, TableRecords afterImage,String ignoreColumns) {
if (beforeImage == null) {
return Result.build(afterImage == null, null);
} else {
if (afterImage == null) {
return Result.build(false, null);
}
if (beforeImage.getTableName().equalsIgnoreCase(afterImage.getTableName())
&& CollectionUtils.isSizeEquals(beforeImage.getRows(), afterImage.getRows())) {
//when image is EmptyTableRecords, getTableMeta will throw an exception
if (CollectionUtils.isEmpty(beforeImage.getRows())) {
return Result.ok();
}
return compareRows(beforeImage.getTableMeta(), beforeImage.getRows(), afterImage.getRows(),ignoreColumns);
} else {
return Result.build(false, null);
}
}
}
/**
* Is rows equals result.
*
Expand Down Expand Up @@ -185,6 +206,45 @@ private static Result<Boolean> compareRows(TableMeta tableMetaData, List<Row> ol
}
return Result.ok();
}
private static boolean isCompareField(String field,String ignoreColumns){
if(ignoreColumns == null || ignoreColumns.isEmpty()){
return true;
}
if(Arrays.asList(ignoreColumns.toLowerCase().split(",")).contains(field.toLowerCase())){
return false;
}
return true;
}
private static Result<Boolean> compareRows(TableMeta tableMetaData, List<Row> oldRows, List<Row> newRows,String ignoreColumns) {
// old row to map
Map<String, Map<String, Field>> oldRowsMap = rowListToMap(oldRows, tableMetaData.getPrimaryKeyOnlyName());
// new row to map
Map<String, Map<String, Field>> newRowsMap = rowListToMap(newRows, tableMetaData.getPrimaryKeyOnlyName());
// compare data
for (Map.Entry<String, Map<String, Field>> oldEntry : oldRowsMap.entrySet()) {
String key = oldEntry.getKey();
Map<String, Field> oldRow = oldEntry.getValue();
Map<String, Field> newRow = newRowsMap.get(key);
if (newRow == null) {
return Result.buildWithParams(false, "compare row failed, rowKey {}, reason [newRow is null]", key);
}
for (Map.Entry<String, Field> oldRowEntry : oldRow.entrySet()) {
String fieldName = oldRowEntry.getKey();
Field oldField = oldRowEntry.getValue();
Field newField = newRow.get(fieldName);
if (newField == null) {
return Result.buildWithParams(false, "compare row failed, rowKey {}, fieldName {}, reason [newField is null]", key, fieldName);
}
if(isCompareField(fieldName,ignoreColumns)) {
Result<Boolean> oldEqualsNewFieldResult = isFieldEquals(oldField, newField);
if (!oldEqualsNewFieldResult.getResult()) {
return oldEqualsNewFieldResult;
}
}
}
}
return Result.ok();
}

/**
* Row list to map map.
Expand Down
10 changes: 7 additions & 3 deletions rm-datasource/src/main/java/io/seata/rm/datasource/undo/AbstractUndoExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,11 @@ public abstract class AbstractUndoExecutor {
*/
public static final boolean IS_UNDO_DATA_VALIDATION_ENABLE = ConfigurationFactory.getInstance()
.getBoolean(ConfigurationKeys.TRANSACTION_UNDO_DATA_VALIDATION, DEFAULT_TRANSACTION_UNDO_DATA_VALIDATION);

/**
* undo data validation ignore columns compare
*/
public static final String DATA_VALIDATION_IGNORE_COLUMNS = ConfigurationFactory.getInstance()
.getConfig(ConfigurationKeys.TRANSACTION_UNDO_DATA_VALIDATION_IGNORE_COLUMNS,"");
/**
* The Sql undo log.
*/
Expand Down Expand Up @@ -246,12 +250,12 @@ protected boolean dataValidationAndGoOn(Connection conn) throws SQLException {
// Validate if data is dirty.
TableRecords currentRecords = queryCurrentRecords(conn);
// compare with current data and after image.
Result<Boolean> afterEqualsCurrentResult = DataCompareUtils.isRecordsEquals(afterRecords, currentRecords);
Result<Boolean> afterEqualsCurrentResult = DataCompareUtils.isRecordsEquals(afterRecords, currentRecords,DATA_VALIDATION_IGNORE_COLUMNS);
if (!afterEqualsCurrentResult.getResult()) {

// If current data is not equivalent to the after data, then compare the current data with the before
// data, too. No need continue to undo if current data is equivalent to the before data snapshot
Result<Boolean> beforeEqualsCurrentResult = DataCompareUtils.isRecordsEquals(beforeRecords, currentRecords);
Result<Boolean> beforeEqualsCurrentResult = DataCompareUtils.isRecordsEquals(beforeRecords, currentRecords,DATA_VALIDATION_IGNORE_COLUMNS);
if (beforeEqualsCurrentResult.getResult()) {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Stop rollback because there is no data change " +
Expand Down
71 changes: 71 additions & 0 deletions rm-datasource/src/test/java/io/seata/rm/datasource/DataCompareUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,76 @@ public void isRecordsEquals() {
afterImage.setRows(new ArrayList<>());
Assertions.assertTrue(DataCompareUtils.isRecordsEquals(beforeImage, afterImage).getResult());
}
@Test
public void isRecordsEquals2() {
String ignoreColumn = "updated_time,inserted_time";
TableMeta tableMeta = Mockito.mock(TableMeta.class);
Mockito.when(tableMeta.getPrimaryKeyOnlyName()).thenReturn(Arrays.asList(new String[]{"pk"}));
Mockito.when(tableMeta.getTableName()).thenReturn("table_name");

TableRecords beforeImage = new TableRecords();
beforeImage.setTableName("table_name");
beforeImage.setTableMeta(tableMeta);

List<Row> rows = new ArrayList<>();
Row row = new Row();
Field field01 = addField(row,"pk", 1, "12345");
Field field02 = addField(row,"age", 1, "18");
Field field03 = addField(row,"updated_time", 1, "2022-01-18 14:26");
rows.add(row);
beforeImage.setRows(rows);

Assertions.assertFalse(DataCompareUtils.isRecordsEquals(beforeImage, null,ignoreColumn).getResult());
Assertions.assertFalse(DataCompareUtils.isRecordsEquals(null, beforeImage,ignoreColumn).getResult());

TableRecords afterImage = new TableRecords();
afterImage.setTableName("table_name1"); // wrong table name
afterImage.setTableMeta(tableMeta);

Assertions.assertFalse(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());
afterImage.setTableName("table_name");

Assertions.assertFalse(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());

List<Row> rows2 = new ArrayList<>();
Row row2 = new Row();
Field field11 = addField(row2,"pk", 1, "12345");
Field field12 = addField(row2,"age", 1, "18");
Field field13 = addField(row2,"updated_time", 1, "2022-01-18 14:26");
rows2.add(row2);
afterImage.setRows(rows2);
//set diff datetime ,return true
field13.setValue("2022-01-19 14:26");
Assertions.assertTrue(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());

Assertions.assertTrue(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());

field11.setValue("23456");
Assertions.assertFalse(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());
field11.setValue("12345");

field12.setName("sex");
Assertions.assertFalse(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());
field12.setName("age");

field12.setValue("19");
Assertions.assertFalse(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());
field12.setName("18");




Field field3 = new Field("pk", 1, "12346");
Row row3 = new Row();
row3.add(field3);
rows2.add(row3);
Assertions.assertFalse(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());


beforeImage.setRows(new ArrayList<>());
afterImage.setRows(new ArrayList<>());
Assertions.assertTrue(DataCompareUtils.isRecordsEquals(beforeImage, afterImage,ignoreColumn).getResult());
}

private Field addField(Row row, String name, int type, Object value){
Field field = new Field(name, type, value);
Expand Down Expand Up @@ -159,6 +229,7 @@ public void isRowsEquals() {
Assertions.assertFalse(DataCompareUtils.isRowsEquals(tableMeta, rows, rows2).getResult());
}


@Test
public void testRowListToMapWithSinglePk(){
List<String> primaryKeyList = new ArrayList<>();
Expand Down
1 change: 1 addition & 0 deletions script/client/spring/application.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ seata.client.tm.degrade-check-allow-times=10
seata.client.tm.degrade-check-period=2000
seata.client.tm.interceptor-order=-2147482648 #Ordered.HIGHEST_PRECEDENCE + 1000
seata.client.undo.data-validation=true
seata.client.undo.data-validation-ignore-columns=updated_time,inserted_time
seata.client.undo.log-serialization=jackson
seata.client.undo.only-care-update-columns=true
seata.client.undo.log-table=undo_log
Expand Down