Skip to content

Commit f7748a4

Browse files
linliu-codelinliu-code
committed
Standardize autoupgrade usage
1 parent 4ebc254 commit f7748a4

File tree

3 files changed

+131
-4
lines changed

3 files changed

+131
-4
lines changed

‎hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.apache.hudi.common.util.collection.Triple;
6161
import org.apache.hudi.exception.HoodieException;
6262
import org.apache.hudi.exception.HoodieIOException;
63+
import org.apache.hudi.exception.HoodieNotSupportedException;
6364
import org.apache.hudi.exception.TableNotFoundException;
6465
import org.apache.hudi.keygen.constant.KeyGeneratorType;
6566
import org.apache.hudi.metadata.HoodieTableMetadata;
@@ -90,6 +91,7 @@
9091
import static org.apache.hudi.common.table.HoodieTableConfig.RECORD_MERGE_STRATEGY_ID;
9192
import static org.apache.hudi.common.table.HoodieTableConfig.TIMELINE_PATH;
9293
import static org.apache.hudi.common.table.HoodieTableConfig.VERSION;
94+
import static org.apache.hudi.common.table.HoodieTableConfig.getTableVersion;
9395
import static org.apache.hudi.common.util.ConfigUtils.containsConfigProperty;
9496
import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys;
9597
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
@@ -1059,6 +1061,7 @@ public static class TableBuilder {
10591061
private String tableName;
10601062
private String tableCreateSchema;
10611063
private HoodieTableVersion tableVersion;
1064+
private boolean autoUpgrade = true;
10621065
private String recordKeyFields;
10631066
private String secondaryKeyFields;
10641067
private String archiveLogFolder;
@@ -1127,6 +1130,11 @@ public TableBuilder setTableVersion(HoodieTableVersion tableVersion) {
11271130
return this;
11281131
}
11291132

1133+
public TableBuilder setAutoUpgrade(boolean autoUpgrade) {
1134+
this.autoUpgrade = autoUpgrade;
1135+
return this;
1136+
}
1137+
11301138
public TableBuilder setTimelinePath(String timelinePath) {
11311139
this.timelinePath = timelinePath;
11321140
return this;
@@ -1473,10 +1481,7 @@ public Properties build() {
14731481
tableConfig.setValue(HoodieTableConfig.NAME, tableName);
14741482
tableConfig.setValue(HoodieTableConfig.TYPE, tableType.name());
14751483

1476-
if (null == tableVersion) {
1477-
tableVersion = HoodieTableVersion.current();
1478-
}
1479-
1484+
tableVersion = getTableVersionProperly();
14801485
tableConfig.setTableVersion(tableVersion);
14811486
tableConfig.setInitialVersion(tableVersion);
14821487

@@ -1591,6 +1596,25 @@ public Properties build() {
15911596
return tableConfig.getProps();
15921597
}
15931598

1599+
HoodieTableVersion getTableVersionProperly() {
1600+
if (tableVersion == null
1601+
|| tableVersion.greaterThanOrEquals(HoodieTableVersion.current())) {
1602+
return HoodieTableVersion.current();
1603+
} else {
1604+
if (autoUpgrade && (tableVersion.versionCode() == 6 || tableVersion.versionCode() == 8)) {
1605+
String errorMessage = String.format(
1606+
"Table version \"%d\" is set for table creation, which is lower than "
1607+
+ "the highest available version \"%d\" the current Hudi binary supports. Meanwhile, "
1608+
+ "the auto upgrade configuration \"hoodie.write.auto.upgrade\" is enabled. A Potential risk is that "
1609+
+ "future commits might upgrade this table without any awareness. "
1610+
+ "To prevent this, please either set \"hoodie.write.auto.upgrade=false\" or unset the table version.",
1611+
tableVersion.versionCode(), HoodieTableVersion.current().versionCode());
1612+
throw new HoodieNotSupportedException(errorMessage);
1613+
}
1614+
return tableVersion;
1615+
}
1616+
}
1617+
15941618
public HoodieTableMetaClient initTable(StorageConfiguration<?> configuration, String basePath) throws IOException {
15951619
return initTable(configuration, new StoragePath(basePath));
15961620
}

‎hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.hudi.common.testutils.HoodieTestUtils;
3030
import org.apache.hudi.common.util.FileIOUtils;
3131
import org.apache.hudi.common.util.Option;
32+
import org.apache.hudi.exception.HoodieNotSupportedException;
3233
import org.apache.hudi.metadata.HoodieIndexVersion;
3334
import org.apache.hudi.metadata.MetadataPartitionType;
3435
import org.apache.hudi.storage.StoragePath;
@@ -149,6 +150,7 @@ void testTableVersion() throws IOException {
149150
.setTableType(HoodieTableType.MERGE_ON_READ.name())
150151
.setTableName("table-version-test")
151152
.setTableVersion(HoodieTableVersion.SIX.versionCode())
153+
.setAutoUpgrade(false)
152154
.initTable(this.metaClient.getStorageConf(), basePath);
153155
assertEquals(HoodieTableVersion.SIX, metaClient1.getTableConfig().getTableVersion());
154156

@@ -168,10 +170,12 @@ void testGenerateFromAnotherMetaClient() throws IOException {
168170
.setTableType(HoodieTableType.MERGE_ON_READ.name())
169171
.setTableName("table-version-test")
170172
.setTableVersion(HoodieTableVersion.SIX.versionCode())
173+
.setAutoUpgrade(false)
171174
.initTable(this.metaClient.getStorageConf(), basePath1);
172175

173176
HoodieTableMetaClient metaClient2 = HoodieTableMetaClient.newTableBuilder()
174177
.fromMetaClient(metaClient1)
178+
.setAutoUpgrade(false)
175179
.initTable(this.metaClient.getStorageConf(), basePath2);
176180

177181
assertEquals(metaClient1.getTableConfig().getTableType(), metaClient2.getTableConfig().getTableType());
@@ -283,4 +287,100 @@ void testDeleteDefinition() throws IOException {
283287
new String(FileIOUtils.readDataFromPath(metaClient.getStorage(), new StoragePath(metaClient.getIndexDefinitionPath())).get()));
284288
assertTrue(indexMetadata.getIndexDefinitions().isEmpty());
285289
}
290+
291+
@Test
292+
void testGetTableVersionProperlyWithNullTableVersion() {
293+
HoodieTableMetaClient.TableBuilder tableBuilder = HoodieTableMetaClient.newTableBuilder()
294+
.setTableName("test_table")
295+
.setTableType(HoodieTableType.COPY_ON_WRITE)
296+
.setAutoUpgrade(false);
297+
298+
// tableVersion is null by default
299+
HoodieTableVersion result = tableBuilder.getTableVersionProperly();
300+
assertEquals(HoodieTableVersion.current(), result);
301+
}
302+
303+
@Test
304+
void testGetTableVersionProperlyWithCurrentVersion() {
305+
HoodieTableMetaClient.TableBuilder tableBuilder = HoodieTableMetaClient.newTableBuilder()
306+
.setTableName("test_table")
307+
.setTableType(HoodieTableType.COPY_ON_WRITE)
308+
.setTableVersion(HoodieTableVersion.current())
309+
.setAutoUpgrade(false);
310+
HoodieTableVersion result = tableBuilder.getTableVersionProperly();
311+
assertEquals(HoodieTableVersion.current(), result);
312+
}
313+
314+
@Test
315+
void testGetTableVersionProperlyWithHigherVersion() {
316+
HoodieTableMetaClient.TableBuilder tableBuilder = HoodieTableMetaClient.newTableBuilder()
317+
.setTableName("test_table")
318+
.setTableType(HoodieTableType.COPY_ON_WRITE)
319+
.setTableVersion(HoodieTableVersion.NINE)
320+
.setAutoUpgrade(false);
321+
HoodieTableVersion result = tableBuilder.getTableVersionProperly();
322+
assertEquals(HoodieTableVersion.current(), result);
323+
}
324+
325+
@Test
326+
void testGetTableVersionProperlyWithVariousLowerVersions() {
327+
// Test with different lower versions
328+
HoodieTableVersion[] lowerVersions = {
329+
HoodieTableVersion.ZERO,
330+
HoodieTableVersion.ONE,
331+
HoodieTableVersion.TWO,
332+
HoodieTableVersion.THREE,
333+
HoodieTableVersion.FOUR,
334+
HoodieTableVersion.FIVE,
335+
HoodieTableVersion.SIX,
336+
HoodieTableVersion.SEVEN,
337+
HoodieTableVersion.EIGHT
338+
};
339+
340+
for (HoodieTableVersion version : lowerVersions) {
341+
HoodieTableMetaClient.TableBuilder tableBuilder = HoodieTableMetaClient.newTableBuilder()
342+
.setTableName("test_table")
343+
.setTableType(HoodieTableType.COPY_ON_WRITE)
344+
.setTableVersion(version)
345+
.setAutoUpgrade(false);
346+
347+
HoodieTableVersion result = tableBuilder.getTableVersionProperly();
348+
assertEquals(version, result);
349+
}
350+
}
351+
352+
@Test
353+
void testGetTableVersionProperlyWithLowerVersionsAndAutoUpgradeTrue() {
354+
// Test with different lower versions and autoUpgrade=true
355+
HoodieTableVersion[] lowerVersions = {
356+
HoodieTableVersion.ZERO,
357+
HoodieTableVersion.ONE,
358+
HoodieTableVersion.TWO,
359+
HoodieTableVersion.THREE,
360+
HoodieTableVersion.FOUR,
361+
HoodieTableVersion.FIVE,
362+
HoodieTableVersion.SIX,
363+
HoodieTableVersion.SEVEN,
364+
HoodieTableVersion.EIGHT
365+
};
366+
367+
for (HoodieTableVersion version : lowerVersions) {
368+
HoodieTableMetaClient.TableBuilder tableBuilder = HoodieTableMetaClient.newTableBuilder()
369+
.setTableName("test_table")
370+
.setTableType(HoodieTableType.COPY_ON_WRITE)
371+
.setTableVersion(version)
372+
.setAutoUpgrade(true);
373+
374+
if (version.versionCode() == 6 || version.versionCode() == 8) {
375+
HoodieNotSupportedException exception = assertThrows(HoodieNotSupportedException.class,
376+
() -> tableBuilder.getTableVersionProperly());
377+
// Verify the error message contains the expected version information
378+
assertTrue(exception.getMessage().contains(
379+
String.format("Table version \"%d\"", version.versionCode())));
380+
} else {
381+
HoodieTableVersion result = tableBuilder.getTableVersionProperly();
382+
assertEquals(version, result);
383+
}
384+
}
385+
}
286386
}

‎hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,9 +300,12 @@ class HoodieSparkSqlWriterInternal {
300300
hoodieConfig.getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME)
301301
else KeyGeneratorType.getKeyGeneratorClassName(hoodieConfig)
302302
val tableFormat = hoodieConfig.getStringOrDefault(HoodieTableConfig.TABLE_FORMAT)
303+
val autoUpgrade = hoodieConfig.getBooleanOrDefault(
304+
HoodieWriteConfig.AUTO_UPGRADE_VERSION, HoodieWriteConfig.AUTO_UPGRADE_VERSION.defaultValue())
303305
HoodieTableMetaClient.newTableBuilder()
304306
.setTableType(tableType)
305307
.setTableVersion(tableVersion)
308+
.setAutoUpgrade(autoUpgrade)
306309
.setTableFormat(tableFormat)
307310
.setDatabaseName(databaseName)
308311
.setTableName(tblName)

0 commit comments

Comments
 (0)