Adds test of Scylla Manager Alternator restore

This patch adds `test_alternator_backup_feature` which checks Scylla Manager's feature for backing up and restoring Alternator data.
It reuses workflow of previous CQL backup test `test_restore_backup_with_task`, but:
- it pre-creates Alternator tables,
- it fills tables and runs post-restore check using YSCB.

Besides plain table+data restore, it is ready to check additional features:
- table names including special characters - requires changing table names in `test-cases/manager/manager-regression-alternator-singleDC-set-distro.yaml` and `sdcm/utils/alternator/consts.py`
- restoring data with secondary indexes - requires:
    - updating YCSB to support index in scan command
    - setting `lsi = True, gsi = True` in ClusterTester::pre_create_alternator_backuped_tables
    - uncommenting additional stress command in `test-cases/manager/manager-regression-alternator-singleDC-set-distro.yaml` to include `gsi` and `lsi` tables.

When defined in configuration, the tests passes to Scylla Manager additional command line parameters `--alternator-access-key-id` `--alternator-secret-access-key`.

Fixes #11696

Author: m-szymon

jenkins-pipelines/manager/ubuntu22-manager-backup-alternator.jenkinsfile

@@ -0,0 +1,15 @@
+#!groovy
+
+// trick from https://github.com/jenkinsci/workflow-cps-global-lib-plugin/pull/43
+def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)
+
+managerPipeline(
+    backend: 'aws',
+    region: 'us-east-1',
+    test_name: 'mgmt_cli_test.ManagerBackupTests.test_backup_alternator_feature',
+    test_config: 'test-cases/manager/manager-regression-alternator-singleDC-set-distro.yaml',
+
+    post_behavior_db_nodes: 'destroy',
+    post_behavior_loader_nodes: 'destroy',
+    post_behavior_monitor_nodes: 'destroy',
+)

mgmt_cli_test.py

@@ -678,8 +678,10 @@ def verify_backup_success(self, mgr_cluster, backup_task, ks_names: list = None,
         if ks_names is None:
             ks_names = ['keyspace1']
         if tables_names is None:
-            tables_names = ['standard1']
-        ks_tables_map = {keyspace: tables_names for keyspace in ks_names}
+            ks_tables_map = {keyspace: [self.alternator.get_table_name(
+                keyspace) or 'standard1'] for keyspace in ks_names}
+        else:
+            ks_tables_map = {keyspace: tables_names for keyspace in ks_names}
         if truncate:
             for ks, tables in ks_tables_map.items():
                 for table_name in tables:
@@ -829,9 +831,9 @@ def test_restore_multiple_backup_snapshots(self):  # noqa: PLR0914
             read_thread = self.run_stress_thread(stress_cmd=stress, round_robin=False)
             self.verify_stress_thread(read_thread)
 
-    def test_restore_backup_with_task(self, ks_names: list = None):
+    def test_restore_backup_with_task(self, ks_names: list = None, cluster_extra_params: dict = None):
         self.log.info('starting test_restore_backup_with_task')
-        mgr_cluster = self.db_cluster.get_cluster_manager()
+        mgr_cluster = self.db_cluster.get_cluster_manager(**(cluster_extra_params or {}))
         if not ks_names:
             ks_names = ['keyspace1']
         backup_task = mgr_cluster.create_backup_task(location_list=self.locations, keyspace_list=ks_names)
@@ -1005,6 +1007,13 @@ def test_backup_feature(self):
         with self.subTest('Test Restore end of space'):
             self.test_enospc_before_restore()
 
+    def test_alternator_backup_feature(self):
+        ks_names, alternator_credentials = self.pre_create_alternator_backuped_tables()
+        self.generate_load_and_wait_for_results(keyspace_name=ks_names)
+        with self.subTest('Test restore a backup with restore task'):
+            self.test_restore_backup_with_task(ks_names=ks_names, cluster_extra_params={
+                                               "alternator_credentials": alternator_credentials})
+
     def test_no_delta_backup_at_disabled_compaction(self):
         """The purpose of test is to check that delta backup (no changes to DB between backups) takes time -> 0.
 

sdcm/mgmt/cli.py

@@ -1043,7 +1043,7 @@ def get_cluster_hosts_with_ips(db_cluster):
         return [[n, n.ip_address] for n in db_cluster.nodes]
 
     def add_cluster(self, name, host=None, db_cluster=None, client_encrypt=None, disable_automatic_repair=True,
-                    auth_token=None, credentials=None, force_non_ssl_session_port=False):
+                    auth_token=None, credentials=None, force_non_ssl_session_port=False, alternator_credentials=None):
         """
         :param name: cluster name
         :param host: cluster node IP
@@ -1054,6 +1054,7 @@ def add_cluster(self, name, host=None, db_cluster=None, client_encrypt=None, dis
         :param auth_token: a token used to authenticate requests to the Agent
         :param credentials: a tuple of the username and password that are used to access the cluster.
         :param force_non_ssl_session_port: force SM to always use the non-SSL port for TLS-enabled cluster CQL sessions.
+        :param alternator_credentials: a tuple of the alternator access key and secret key that are used to access the Alternator API.
         :return: ManagerCluster
 
         Add a cluster to manager
@@ -1091,6 +1092,10 @@ def add_cluster(self, name, host=None, db_cluster=None, client_encrypt=None, dis
             username, password = credentials
             cmd += f" --username {username} --password {password}"
 
+        if alternator_credentials:
+            access_key_id, secret_access_key = alternator_credentials
+            cmd += f" --alternator-access-key-id {access_key_id} --alternator-secret-access-key {secret_access_key}"
+
         res_cluster_add = self.sctool.run(cmd, parse_table_res=False)
         if not res_cluster_add or 'Cluster added' not in res_cluster_add.stderr:
             raise ScyllaManagerError("Encountered an error on 'sctool cluster add' command response: {}".format(

sdcm/tester.py

@@ -1184,13 +1184,9 @@ def pre_create_alternator_tables(self):
         node = self.db_cluster.nodes[0]
         if self.params.get('alternator_port'):
             self.log.info("Going to create alternator tables")
-            if self.params.get('alternator_enforce_authorization'):
-                with self.db_cluster.cql_connection_patient(self.db_cluster.nodes[0]) as session:
-                    session.execute("CREATE ROLE %s WITH PASSWORD = %s AND login = true AND superuser = true",
-                                    (self.params.get('alternator_access_key_id'),
-                                     self.params.get('alternator_secret_access_key')))
+            self.alternator.set_credentials(node=node)
 
-            tablets_enabled = is_tablets_feature_enabled(self.db_cluster.nodes[0])
+            tablets_enabled = is_tablets_feature_enabled(node)
             prepare_cmd = self.params.get('prepare_write_cmd')
             stress_cmd = self.params.get('stress_cmd')
             is_ttl_in_workload = any('dynamodb.ttlKey' in str(cmd) for cmd in [prepare_cmd, stress_cmd])
@@ -1213,6 +1209,35 @@ def pre_create_alternator_tables(self):
                 self.alternator.update_table_ttl(node=node, table_name=alternator.consts.TABLE_NAME)
                 self.alternator.update_table_ttl(node=node, table_name=alternator.consts.NO_LWT_TABLE_NAME)
 
+    def pre_create_alternator_backuped_tables(self):
+        node = self.db_cluster.nodes[0]
+        if self.params.get('alternator_port'):
+            self.log.info("Going to create alternator tables")
+            self.alternator.set_credentials(node=node)
+
+            tablets_enabled = is_tablets_feature_enabled(node)
+            # prepare_cmd = self.params.get('stress_read_cmd')
+            # stress_cmd = self.params.get('stress_cmd')
+            # is_ttl_in_workload = any('dynamodb.ttlKey' in str(cmd) for cmd in [prepare_cmd, stress_cmd])
+
+            # Enable tablets with Alternator TTL requires (#16567).
+            # tablets_support_ttl = not SkipPerIssues(issues="scylladb/scylladb#16567", params=self.params)
+
+            # Enable tablets for Alternator with LWT requires issue: "[Epic] tablets: support LWT #18068"
+            tablets_support_lwt = not SkipPerIssues(issues="scylladb/scylladb#18068", params=self.params)
+
+            tablets_supported_alternator_workload = True  # not is_ttl_in_workload or tablets_support_ttl
+            schema = self.params.get("dynamodb_primarykey_type")
+            self.alternator.create_table(node=node, schema=schema,
+                                         table_name=alternator.consts.BACKUPED_TABLE_NAME,
+                                         tablets_enabled=tablets_enabled and tablets_support_lwt and tablets_supported_alternator_workload,
+                                         lsi=False, gsi=False)
+
+            # if is_ttl_in_workload:
+            #    self.alternator.update_table_ttl(node=node, table_name=alternator.consts.BACKUPED_TABLE_NAME)
+            return [self.alternator.get_keyspace_name(alternator.consts.BACKUPED_TABLE_NAME)], self.alternator.get_credentials(node=node)
+        return None, None
+
     def get_nemesis_class(self):
         """
         Get a Nemesis class from parameters.

sdcm/utils/alternator/api.py

@@ -81,6 +81,21 @@ def get_dynamodb_api(self, node) -> AlternatorApi:
                 self.alternator_apis[endpoint_url] = AlternatorApi(resource=resource, client=client)
         return self.alternator_apis[endpoint_url]
 
+    def set_credentials(self, node):
+        if self.params.get('alternator_enforce_authorization'):
+            with node.parent_cluster.cql_connection_patient(node) as session:
+                session.execute("CREATE ROLE %s WITH PASSWORD = %s AND login = true AND superuser = true",
+                                (self.params.get('alternator_access_key_id'),
+                                    self.params.get('alternator_secret_access_key')))
+
+    def get_credentials(self, node):
+        access_key_id = self.params.get('alternator_access_key_id')
+        if self.params.get('alternator_enforce_authorization'):
+            return (access_key_id, self.get_salted_hash(node=node, username=access_key_id))
+        else:
+            access_key = self.params.get('alternator_access_key')
+            return (access_key_id, access_key) if access_key_id and access_key else None
+
     def set_write_isolation(self, node, isolation, table_name=consts.TABLE_NAME):
         dynamodb_api = self.get_dynamodb_api(node=node)
         isolation = isolation if not isinstance(isolation, enums.WriteIsolation) else isolation.value
@@ -96,10 +111,27 @@ def set_write_isolation(self, node, isolation, table_name=consts.TABLE_NAME):
 
     def create_table(self, node,
                      schema=enums.YCSBSchemaTypes.HASH_AND_RANGE, isolation=None, table_name=consts.TABLE_NAME,
-                     wait_until_table_exists=True, tablets_enabled: bool = False, **kwargs) -> Table:
+                     wait_until_table_exists=True, tablets_enabled: bool = False, lsi=False, gsi=False, **kwargs) -> Table:
         if isinstance(schema, enums.YCSBSchemaTypes):
             schema = schema.value
         schema = schemas.ALTERNATOR_SCHEMAS[schema]
+        if lsi:
+            schema['LocalSecondaryIndexes'] = [
+                {
+                    'IndexName': "lsi",
+                    'KeySchema': schema['KeySchema'],
+                    'Projection': {'ProjectionType': 'ALL'}
+                }
+            ]
+        if gsi:
+            schema['GlobalSecondaryIndexes'] = [
+                {
+                    'IndexName': "gsi",
+                    'KeySchema': schema['KeySchema'],
+                    'Projection': {'ProjectionType': 'ALL'}
+                }
+            ]
+
         dynamodb_api = self.get_dynamodb_api(node=node)
         # Tablets feature is currently supported by Alternator, but disabled by default (since LWT is not supported).
         # It should be explicitly requested by the specified tag.
@@ -203,3 +235,13 @@ def delete_table(self, node, table_name: consts.TABLE_NAME, wait_until_table_rem
             LOGGER.info("The '{}' table successfully removed".format(table_name))
         else:
             LOGGER.info("Send request to removed '{}' table".format(table_name))
+
+    @staticmethod
+    def get_keyspace_name(table_name: consts.TABLE_NAME) -> str:
+        return consts.KEYSPACE_PREFIX + table_name
+
+    @staticmethod
+    def get_table_name(keyspace_name: consts.KEYSPACE_PREFIX+consts.TABLE_NAME) -> str:
+        if keyspace_name.startswith(consts.KEYSPACE_PREFIX):
+            return keyspace_name[len(consts.KEYSPACE_PREFIX):]
+        return None

sdcm/utils/alternator/consts.py

@@ -2,3 +2,5 @@
 RANGE_KEY_NAME = "c"
 TABLE_NAME = "usertable"
 NO_LWT_TABLE_NAME = "usertable_no_lwt"
+BACKUPED_TABLE_NAME = "usertable_backup"  # Use [\.\-A-Z] when works
+KEYSPACE_PREFIX = "alternator_"

test-cases/manager/manager-regression-alternator-singleDC-set-distro.yaml

@@ -0,0 +1,42 @@
+test_duration: 240
+
+round_robin: true
+
+stress_cmd:
+    bin/ycsb -jvm-args='-Dorg.slf4j.simpleLogger.defaultLogLevel=OFF' load dynamodb -P workloads/workloadc -threads 200 -p table=usertable_backup
+    -p requestdistribution=sequential -p insertorder=ordered -p insertstart=0
+
+stress_read_cmd: [
+    "bin/ycsb -jvm-args='-Dorg.slf4j.simpleLogger.defaultLogLevel=OFF' run dynamodb -P workloads/workloadc -threads 200 -p table=usertable_backup
+    -p requestdistribution=sequential -p insertorder=ordered -p insertstart=0 -p dataintegrity=true",
+#    "bin/ycsb -jvm-args='-Dorg.slf4j.simpleLogger.defaultLogLevel=OFF' run dynamodb -P workloads/workloadc -threads 200 -p table=usertable_backup:gsi
+#    -p requestdistribution=sequential -p insertorder=ordered -p insertstart=0 -p readproportion=0 -p scanproportion=1",
+#    "bin/ycsb -jvm-args='-Dorg.slf4j.simpleLogger.defaultLogLevel=OFF' run dynamodb -P workloads/workloadc -threads 200 -p table=usertable_backup:lsi
+#    -p requestdistribution=sequential -p insertorder=ordered -p insertstart=0 -p readproportion=0 -p scanproportion=1"
+]
+
+alternator_write_isolation: 'forbid'
+alternator_port: '8080'
+dynamodb_primarykey_type: HASH_AND_RANGE
+
+#alternator_enforce_authorization: true
+#alternator_access_key_id: 'alternator'
+#alternator_secret_access_key: 'password'
+
+instance_type_db: 'i4i.large'
+instance_type_loader: 'c6i.large'
+
+region_name: 'us-east-1'
+n_db_nodes: '3'
+n_loaders: 1
+
+client_encrypt: true
+
+post_behavior_db_nodes: "destroy"
+post_behavior_loader_nodes: "destroy"
+post_behavior_monitor_nodes: "destroy"
+
+user_prefix: manager-regression
+space_node_threshold: 6442
+
+aws_instance_profile_name_db: 'qa-scylla-manager-backup-instance-profile'