IGNITE-22410 Implement rebalance triggers for zone based partitions

buildscripts/generate-version-file.gradle

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def createVersionFile = tasks.register('createVersionFile') {
+    onlyIf {
+        project.hasProperty("lastCommitHash")
+    }
+
+    def versionFileName = "ignite.version.full"
+    def versionFile = file("$buildDir/$versionFileName")
+
+    if (versionFile.exists()) {
+        versionFile.delete()
+    }
+
+    doLast {
+        def versionProp = project.property("lastCommitHash")
+
+        versionFile << project.version + "-sha1:" + versionProp
+    }
+
+    outputs.file versionFile
+}
+
+// make  processResources depends on createExtraFilesIndex task
+processResources {
+    from createVersionFile
+}

buildscripts/java-core.gradle

@@ -26,6 +26,7 @@ java {
     sourceCompatibility = JavaVersion.VERSION_11
     targetCompatibility = JavaVersion.VERSION_11
     withSourcesJar()
+    withJavadocJar()
 }
 
 test {

buildscripts/sql-parser-generator.gradle

@@ -86,6 +86,7 @@ extractParserTemplate.dependsOn copyFmppResources
 generateFmppSources.dependsOn extractParserTemplate
 javaccTemplate.dependsOn generateFmppSources
 compileJava.dependsOn javaccTemplate
+javadocJar.dependsOn compileJava
 
 idea.module {
     sourceDirs += file("${buildDir}/generated-sources/javacc")

docs/_docs/administrators-guide/config/storage/rocksdb.adoc

@@ -33,7 +33,6 @@ Below is the list of properties for link:https://rocksdb.org/[RocksDB,window=_bl
 |size| `256 * 1024 * 1024` | Size of the offheap cache.
 |writeBufferSize | `64 * 1024 * 1024` | Size of the write buffer.
 |cache| `lru` | The type of the cache to use. Currently only lru is supported. Using `clock` cache is not recommended.
-|numShardBits| `-1` | The number of parts the cache is sharded to.
 |flushDelayMillis |100| The delay before handling RocksDB flush events.
 |===
 

docs/_docs/ignite-cli-tool.adoc

@@ -63,7 +63,7 @@ These commands let you manage your cluster.
 [cols="1,1,3",opts="header", stripes=none]
 |===
 | Command| Parameters | Description
-| `cluster init` | `[--cmg-node] [--meta-storage-node] <clusterName> or < --url> <cluster-config> or < --config-file>`| Initializes a cluster.
+| `cluster init` | `[--cmg-node] [--meta-storage-node] <clusterName> or < --url> <cluster-config> or < --config-files>`| Initializes a cluster.
 | `cluster status` | `[--url]` | Displays the detailed status of the specified cluster.
 | `cluster topology physical` | `[--plain] [--url]` | Shows physical topology of the specified cluster.
 | `cluster topology logical` | `[--plain] [--url]` | Shows logical topology of the specified cluster.

examples/src/integrationTest/java/org/apache/ignite/example/AbstractExamplesTest.java

@@ -23,12 +23,9 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.List;
 import java.util.concurrent.CompletableFuture;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgnitionManager;
+import org.apache.ignite.IgniteServer;
 import org.apache.ignite.InitParameters;
-import org.apache.ignite.internal.app.IgniteImpl;
 import org.apache.ignite.internal.testframework.IgniteAbstractTest;
 import org.apache.ignite.internal.testframework.WorkDirectoryExtension;
 import org.junit.jupiter.api.AfterEach;
@@ -45,40 +42,35 @@ public abstract class AbstractExamplesTest extends IgniteAbstractTest {
     /** Empty argument to invoke an example. */
     protected static final String[] EMPTY_ARGS = new String[0];
 
-    /** Started ignite instance. */
-    protected IgniteImpl ignite;
+    /** Started embedded node. */
+    private IgniteServer node;
 
     /**
      * Starts a node.
      */
     @BeforeEach
     public void startNode() throws Exception {
-        CompletableFuture<Ignite> igniteFuture = IgnitionManager.start(
+        node = IgniteServer.start(
                 TEST_NODE_NAME,
                 configFile(),
                 workDir
         );
 
         InitParameters initParameters = InitParameters.builder()
-                .destinationNodeName(TEST_NODE_NAME)
-                .metaStorageNodeNames(List.of(TEST_NODE_NAME))
+                .metaStorageNodeNames(TEST_NODE_NAME)
                 .clusterName("cluster")
                 .build();
 
-        IgnitionManager.init(initParameters);
-
-        assertThat(igniteFuture, willCompleteSuccessfully());
-
-        // We can call without a timeout, since the future is guaranteed to be completed above.
-        ignite = (IgniteImpl) igniteFuture.join();
+        CompletableFuture<Void> initFuture = node.initClusterAsync(initParameters);
+        assertThat(initFuture, willCompleteSuccessfully());
     }
 
     /**
      * Stops the node.
      */
     @AfterEach
     public void stopNode() {
-        IgnitionManager.stop(TEST_NODE_NAME);
+        node.shutdown();
     }
 
     /**

examples/src/integrationTest/java/org/apache/ignite/example/sql/ItSqlExamplesTest.java

@@ -21,7 +21,6 @@
 
 import org.apache.ignite.example.AbstractExamplesTest;
 import org.apache.ignite.example.sql.jdbc.SqlJdbcExample;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -58,7 +57,6 @@ public void testSqlJdbcExample() throws Exception {
      *
      * @throws Exception If failed.
      */
-    @Disabled("https://issues.apache.org/jira/browse/IGNITE-17059")
     @Test
     public void testSqlApiExample() throws Exception {
         assertConsoleOutputContains(SqlApiExample::main, EMPTY_ARGS,

examples/src/main/java/org/apache/ignite/example/sql/SqlApiExample.java

@@ -27,6 +27,8 @@
 import org.apache.ignite.sql.Statement;
 import org.apache.ignite.sql.async.AsyncResultSet;
 import org.apache.ignite.table.mapper.Mapper;
+import org.apache.ignite.tx.Transaction;
+import org.apache.ignite.tx.TransactionOptions;
 
 /**
  * Examples of using SQL API.
@@ -57,64 +59,86 @@ public static void main(String[] args) throws Exception {
             //
             //--------------------------------------------------------------------------------------
 
-            client.sql().execute(
-                    null,
+            client.sql().executeScript(
                     "CREATE TABLE CITIES ("
                             + "ID   INT PRIMARY KEY,"
-                            + "NAME VARCHAR)"
-            ).close(); // Ignore result.
+                            + "NAME VARCHAR);"
 
-            client.sql().execute(
-                    null,
-                    "CREATE TABLE ACCOUNTS ("
+                            + "CREATE TABLE ACCOUNTS ("
                             + "    ACCOUNT_ID INT PRIMARY KEY,"
                             + "    CITY_ID    INT,"
                             + "    FIRST_NAME VARCHAR,"
                             + "    LAST_NAME  VARCHAR,"
                             + "    BALANCE    DOUBLE)"
-            ).close();
+            );
 
             //--------------------------------------------------------------------------------------
             //
-            // Populating 'CITIES' table.
+            // Starting transaction.
             //
             //--------------------------------------------------------------------------------------
 
-            System.out.println("\nPopulating 'CITIES' table...");
-
-            try (Statement stmt = client.sql().createStatement("INSERT INTO CITIES (ID, NAME) VALUES (?, ?)")) {
-                long rowsAdded = 0;
-
-                try (ResultSet rs = client.sql().execute(null, stmt, 1, "Forest Hill")) {
-                    rowsAdded += rs.affectedRows();
-                }
-                try (ResultSet rs = client.sql().execute(null, stmt, 2, "Denver")) {
-                    rowsAdded += rs.affectedRows();
-                }
-                try (ResultSet rs = client.sql().execute(null, stmt, 3, "St. Petersburg")) {
-                    rowsAdded += rs.affectedRows();
-                }
+            System.out.println("\nStarting read-write transaction...");
 
-                System.out.println("\nAdded cities: " + rowsAdded);
-            }
+            Transaction tx = client.transactions().begin(new TransactionOptions().readOnly(false));
 
             //--------------------------------------------------------------------------------------
             //
-            // Populating 'ACCOUNTS' table.
+            // Populating 'CITIES' table.
             //
             //--------------------------------------------------------------------------------------
 
-            System.out.println("\nPopulating 'ACCOUNTS' table...");
+            try {
+                System.out.println("\nPopulating 'CITIES' table...");
 
-            long rowsAdded = Arrays.stream(client.sql().executeBatch(null,
-                            "INSERT INTO ACCOUNTS (ACCOUNT_ID, CITY_ID, FIRST_NAME, LAST_NAME, BALANCE) values (?, ?, ?, ?, ?)",
-                            BatchedArguments.of(1, 1, "John", "Doe", 1000.0d)
-                                    .add(2, 1, "Jane", "Roe", 2000.0d)
-                                    .add(3, 1, "Mary", "Major", 1500.0d)
-                                    .add(4, 1, "Richard", "Miles", 1450.0d)))
-                    .sum();
+                try (Statement stmt = client.sql().createStatement("INSERT INTO CITIES (ID, NAME) VALUES (?, ?)")) {
+                    long rowsAdded = 0;
 
-            System.out.println("\nAdded accounts: " + rowsAdded);
+                    try (ResultSet<?> rs = client.sql().execute(tx, stmt, 1, "Forest Hill")) {
+                        rowsAdded += rs.affectedRows();
+                    }
+                    try (ResultSet<?> rs = client.sql().execute(tx, stmt, 2, "Denver")) {
+                        rowsAdded += rs.affectedRows();
+                    }
+                    try (ResultSet<?> rs = client.sql().execute(tx, stmt, 3, "St. Petersburg")) {
+                        rowsAdded += rs.affectedRows();
+                    }
+
+                    System.out.println("\nAdded cities: " + rowsAdded);
+                }
+
+                //--------------------------------------------------------------------------------------
+                //
+                // Populating 'ACCOUNTS' table.
+                //
+                //--------------------------------------------------------------------------------------
+
+                System.out.println("\nPopulating 'ACCOUNTS' table...");
+
+                long rowsAdded = Arrays.stream(client.sql().executeBatch(tx,
+                                "INSERT INTO ACCOUNTS (ACCOUNT_ID, CITY_ID, FIRST_NAME, LAST_NAME, BALANCE) values (?, ?, ?, ?, ?)",
+                                BatchedArguments.of(1, 1, "John", "Doe", 1000.0d)
+                                        .add(2, 1, "Jane", "Roe", 2000.0d)
+                                        .add(3, 2, "Mary", "Major", 1500.0d)
+                                        .add(4, 3, "Richard", "Miles", 1450.0d)))
+                        .sum();
+
+                System.out.println("\nAdded accounts: " + rowsAdded);
+
+                //--------------------------------------------------------------------------------------
+                //
+                // Committing transaction.
+                //
+                //--------------------------------------------------------------------------------------
+
+                System.out.println("\nCommitting transaction...");
+
+                tx.commit();
+            } catch (Exception e) {
+                tx.rollback();
+
+                throw e;
+            }
 
             //--------------------------------------------------------------------------------------
             //
@@ -131,9 +155,9 @@ public static void main(String[] args) throws Exception {
                     SqlRow row = rs.next();
 
                     System.out.println("    "
+                            + row.stringValue(0) + ", "
                             + row.stringValue(1) + ", "
-                            + row.stringValue(2) + ", "
-                            + row.stringValue(3));
+                            + row.stringValue(2));
                 }
             }
 
@@ -199,8 +223,10 @@ public static void main(String[] args) throws Exception {
 
             System.out.println("\nDropping the tables...");
 
-            client.sql().execute(null, "DROP TABLE ACCOUNTS").close();
-            client.sql().execute(null, "DROP TABLE CITIES").close();
+            client.sql().executeScript(
+                    "DROP TABLE ACCOUNTS;"
+                    + "DROP TABLE CITIES"
+            );
         }
     }
 
@@ -216,9 +242,9 @@ private static CompletionStage<Void> fetchAllRowsInto(AsyncResultSet<SqlRow> res
         //
         for (var row : resultSet.currentPage()) {
             System.out.println("    "
+                    + row.stringValue(0) + ", "
                     + row.stringValue(1) + ", "
-                    + row.stringValue(2) + ", "
-                    + row.stringValue(3));
+                    + row.stringValue(2));
         }
 
         //

gradle/libs.versions.toml

@@ -94,13 +94,13 @@ openapiGenerator = "org.openapi.generator:7.6.0"
 javacc = "com.intershop.gradle.javacc:4.1.3"
 shadow = "io.github.goooler.shadow:8.1.7"
 cmake = "net.freudasoft.gradle-cmake-plugin:0.0.4"
-nebula = "com.netflix.nebula.ospackage:11.9.0"
+nebula = "com.netflix.nebula.ospackage:11.9.1"
 docker = "com.palantir.docker:0.36.0"
 checksum = "org.gradle.crypto.checksum:1.4.0"
 setupbuilder = "de.inetsoftware.setupbuilder:7.2.21"
 aggregateJavadoc = "io.freefair.aggregate-javadoc:6.6.3"
 ideaext = "org.jetbrains.gradle.plugin.idea-ext:1.1.8"
-spotbugs = "com.github.spotbugs:6.0.17"
+spotbugs = "com.github.spotbugs:6.0.18"
 
 
 [libraries]

modules/affinity/src/main/java/org/apache/ignite/internal/affinity/Assignments.java

@@ -153,6 +153,26 @@ public String toString() {
         return S.toString(this);
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        Assignments that = (Assignments) o;
+        return force == that.force && nodes.equals(that.nodes);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = nodes.hashCode();
+        result = 31 * result + Boolean.hashCode(force);
+        return result;
+    }
+
     /**
      * Creates a string representation of the given assignments list for logging usage purpose mostly.
      *

modules/affinity/src/main/java/org/apache/ignite/internal/affinity/TokenizedAssignments.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.affinity;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Set of nodes along with associated token that is guaranteed to be changed if the set was changed.
+ */
+public interface TokenizedAssignments extends Serializable {
+    /** Returns a set of nodes, represented by this assignments instance. */
+    Set<Assignment> nodes();
+
+    /** Returns a token associated with given assignments that is guaranteed to be changed if assignments were changed. */
+    long token();
+}