Long running setup (#126)

* add scripts for setting up long running io test

* improve io test logging

---------

Co-authored-by: Ravi Nagarjun Akella <raakella1@$HOSTNAME>

Author: raakella1

.jenkins/Dockerfile

@@ -0,0 +1,24 @@
+# ##########   #######   ############
+FROM hub.tess.io/sds/sds_develop:7.x-latest
+LABEL description="Automated HomeBlocks compilation"
+
+WORKDIR /output
+
+RUN set -eux; \
+    sed -i 's/master/latest/g' /etc/apt/sources.list; \
+    apt-get update; \
+    apt-get upgrade -y; \
+    apt-get install iputils-ping; \
+    apt-get install net-tools; \
+    rm -rf /var/lib/apt/lists/*;
+
+
+# executables
+COPY test_volume_io /usr/local/bin/test_volume_io
+# scripts
+COPY io_test.py /usr/local/bin/scripts/io_test.py
+COPY long_running.py /usr/local/bin/scripts/long_running.py
+
+
+EXPOSE 5000
+# ##########   #######   ############

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeBlocksConan(ConanFile):
     name = "homeblocks"
-    version = "2.1.2"
+    version = "2.1.3"
     homepage = "https://github.com/eBay/HomeBlocks"
     description = "Block Store built on HomeStore"
     topics = ("ebay")
@@ -45,7 +45,7 @@ def build_requirements(self):
         self.test_requires("gtest/1.14.0")
 
     def requirements(self):
-        self.requires("homestore/[~6.18]@oss/master", transitive_headers=True)
+        self.requires("homestore/[^6.20]@oss/master", transitive_headers=True)
         self.requires("iomgr/[^11.3]@oss/master", transitive_headers=True)
         self.requires("sisl/[^12.2]@oss/master", transitive_headers=True)
         self.requires("lz4/1.9.4", override=True)

src/lib/volume/index_fixed_table.hpp

@@ -29,7 +29,6 @@ class VolumeIndexTable {
                                                             BtreeValue const& value) {
             auto lba = r_cast< const VolumeIndexKey& >(key).lba();
             auto& existing_value_vol_idx = r_cast< const VolumeIndexValue& >(existing_value);
-            LOGINFO("Filter callback for lba {}, existing value: {}", lba, existing_value_vol_idx.blkid().to_string());
             blocks_info[lba].old_blkid = existing_value_vol_idx.blkid();
             blocks_info[lba].old_checksum = existing_value_vol_idx.checksum();
             return homestore::put_filter_decision::replace;

src/lib/volume/tests/test_common.hpp

@@ -52,6 +52,8 @@ SISL_OPTION_GROUP(
      "true or false"),
     (enable_crash, "", "enable_crash", "enable crash", ::cxxopts::value< bool >()->default_value("0"), ""),
     (app_mem_size, "", "app_mem_size", "app memory size", ::cxxopts::value< uint64_t >()->default_value("20"),
+     "number"),
+    (hs_chunk_size_mb, "", "hs_chunk_size_mb", "hs_chunk_size_mb", ::cxxopts::value< uint64_t >()->default_value("128"),
      "number"));
 
 using namespace homeblocks;
@@ -175,7 +177,7 @@ class HBTestHelper {
         init_dev_list(true /*init_device*/);
 
         LOGINFO("Starting HomeBlocks");
-        homeblocks::HomeBlocksImpl::_hs_chunk_size = 128 * Mi;
+        homeblocks::HomeBlocksImpl::_hs_chunk_size = SISL_OPTIONS["hs_chunk_size_mb"].as< uint64_t >() * Mi;
         // set_min_chunk_size(4 * Mi);
         app_ = std::make_shared< HBTestApplication >(*this);
         hb_ = init_homeblocks(std::weak_ptr< HBTestApplication >(app_));
@@ -309,6 +311,7 @@ class HBTestHelper {
     }
 
     void remove_files() {
+        if (SISL_OPTIONS.count("device_list")) { return; }
         for (const auto& fpath : dev_list_) {
             if (std::filesystem::exists(fpath)) {
                 LOGINFO("Removing file {}", fpath);

src/lib/volume/tests/test_scripts/CMakeLists.txt

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+## @file index_test.py
+import subprocess
+import argparse
+
+
+class TestFailedError(Exception):
+    pass
+
+
+def run_test(options):
+    cmd_opts = f"--gtest_filter=VolumeIOTest.LongRunningRandomIO --gtest_break_on_failure {options['log_mods']} --run_time={options['run_time']} --vol_size_gb={options['vol_size_gb']} --num_vols={options['num_vols']} --device_list {options['device_list']} --hs_chunk_size_mb={options['hs_chunk_size_mb']}"
+    test_cmd = f"{options['dirpath']}/test_volume_io {cmd_opts}"
+    print(f"Running command: {test_cmd}")
+    try:
+        subprocess.check_call(test_cmd, stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Test failed: {e}")
+        raise TestFailedError(f"Test failed!")
+    print("Test completed")
+
+
+def parse_arguments():
+    # Create the parser
+    parser = argparse.ArgumentParser(description='Parse command line options.')
+
+    # Add arguments with default values
+    parser.add_argument('--dirpath', help='Directory path', default='.')
+    parser.add_argument('--log_mods', help='Log modules', default='')
+    parser.add_argument('--cp_timer_ms', help='cp_timer_ms', type=int, default=60000)
+    parser.add_argument('--run_time', help='Run time in seconds', type=int, default=10)
+    parser.add_argument('--vol_size_gb', help='vol_size_gb', type=int, default=500)
+    parser.add_argument('--num_vols', help='num_vols', type=int, default=5)
+    parser.add_argument('--device_list', help='Device list', default='')
+    parser.add_argument('--init_device', help='Initialize device', type=bool, default=True)
+    parser.add_argument('--use_file', help='Initialize device', action="store_true")
+    parser.add_argument('--hs_chunk_size_mb', help='hs_chunk_size', type=int, default=2048)
+
+    # Parse the known arguments and ignore any unknown arguments
+    args, unknown = parser.parse_known_args()
+
+    options = vars(args)
+
+    return options
+
+
+def long_runnig_io(options):
+    print("Long running test started")
+    if options['device_list'] == '':
+        if not options['use_file']:
+            print("No device list provided, use --use_file flag to use a file as a device")
+            exit(1)
+        # fallocate a file to use as a device
+        options['device_list'] = f"{options['dirpath']}/io_long_running_device"
+        dev_size = options['vol_size_gb'] * 1.5  # Ensure the device is larger than the volume size
+        subprocess.check_call(f"fallocate -l {dev_size}G {options['device_list']}", shell=True)
+        print(f"Created device: {options['device_list']} of size {dev_size} GB")
+    print(f"options: {options}")
+    run_test(options)
+    # remove the device file after the test
+    if "io_long_running_device" in options['device_list']:
+        subprocess.check_call(f"rm -f {options['device_list']}", shell=True)
+        print(f"Removed device: {options['device_list']}")
+    print("Long running test completed")
+
+
+def long_running(*args):
+    options = parse_arguments()
+    long_runnig_io(options)

src/lib/volume/tests/test_scripts/io_test.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+## @file long_running.py
+#
+
+import sys
+import argparse
+import io_test
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Run long-running tests.')
+    parser.add_argument('--test_suit', help='Test suits to run', default='io_long_running')
+    args, service_args = parser.parse_known_args()
+    return args, service_args
+
+test_functions = {
+    'io_long_running': io_test.long_running
+}
+
+def main():
+    args, service_args = parse_arguments()
+
+    # Check if the test_suits argument is provided and is valid
+    if args.test_suit in test_functions:
+        test_functions[args.test_suit](service_args)
+    else:
+        print(f"Unknown test suite: {args.test_suit}")
+        print(f"Available tests: {', '.join(test_functions.keys())}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

src/lib/volume/tests/test_scripts/long_running.py

@@ -554,8 +554,12 @@ TEST_F(VolumeIOTest, LongRunningRandomIO) {
         total_writes += get_total_writes();
         std::chrono::duration< double > elapsed = std::chrono::high_resolution_clock::now() - start_time;
         auto elapsed_seconds = static_cast< uint64_t >(elapsed.count());
-        LOGINFO("total_read={} total_write={} elapsed={}", total_reads, total_writes, elapsed_seconds);
-
+        static uint64_t log_pct = 0;
+        if (auto done_pct = (run_time > 0) ? (elapsed_seconds * 100) / run_time : 100; done_pct > log_pct) {
+            LOGINFO("total_read={} total_write={} elapsed={}, done pct={}", total_reads, total_writes, elapsed_seconds, done_pct);
+            log_pct += 5;
+        }
+        
         if (elapsed_seconds >= run_time) { break; }
     } while (true);
 }
@@ -585,9 +589,10 @@ TEST_F(VolumeIOTest, LongRunningSequentialIO) {
         total_writes += get_total_writes();
         std::chrono::duration< double > elapsed = std::chrono::high_resolution_clock::now() - start_time;
         auto elapsed_seconds = static_cast< uint64_t >(elapsed.count());
-        if (elapsed_seconds % 2 == 0) {
-            LOGINFO("total_read={} total_write={} lba={} elapsed={}", total_reads, total_writes, cur_lba,
-                    elapsed_seconds);
+        static uint64_t log_pct = 0;
+        if (auto done_pct = (run_time > 0) ? (elapsed_seconds * 100) / run_time : 100; done_pct > log_pct) {
+            LOGINFO("total_read={} total_write={} elapsed={}, done pct={}", total_reads, total_writes, elapsed_seconds, done_pct);
+            log_pct += 5;
         }
 
         if (((cur_lba + nblks) * g_page_size) >= volume_size) {

src/lib/volume/tests/test_volume_io.cpp

@@ -141,9 +141,9 @@ homestore::cshared< Chunk > VolumeChunkSelector::select_chunk(homestore::blk_cou
             if (chunk && chunk->available_blks() > 0) { return chunk->get_internal_chunk(); }
         }
 
-        LOGI("Waiting to allocate more chunks active={} total={}", volc->num_active_chunks.load(),
+        LOGT("Waiting to allocate more chunks active={} total={}", volc->num_active_chunks.load(),
              volc->max_num_chunks);
-        dump_chunks();
+        LOGT("{}", dump_chunks());
         std::this_thread::sleep_for(std::chrono::milliseconds(100));
     } while (true);
 
@@ -184,7 +184,7 @@ void VolumeChunkSelector::resize_volume_num_chunks(homestore::blk_count_t nblks,
     }
 
     // Spawn background task to create new chunks.
-    LOGI("Initiating op to resize num chunks for volume={} available={} total={}", volc->ordinal, available_blks,
+    LOGD("Initiating op to resize num chunks for volume={} available={} total={}", volc->ordinal, available_blks,
          total_blks);
     iomanager.run_on_forget(iomgr::reactor_regex::random_worker, [volc, this]() mutable {
         std::string str;
@@ -213,7 +213,7 @@ void VolumeChunkSelector::resize_volume_num_chunks(homestore::blk_count_t nblks,
         // Update the number of active chunks and compelete the resize operation.
         volc->num_active_chunks = indx;
         resize_op.store(ResizeOp::Idle);
-        LOGI("Resize op done. Allocated more chunks for volume={} total={} new={} new_chunks={}", volc->ordinal,
+        LOGD("Resize op done. Allocated more chunks for volume={} total={} new={} new_chunks={}", volc->ordinal,
              volc->num_active_chunks.load(), chunks.size(), str);
     });
 }
@@ -359,18 +359,20 @@ void VolumeChunkSelector::dump_per_pdev_chunks() const {
     }
 }
 
-void VolumeChunkSelector::dump_chunks() const {
+std::string VolumeChunkSelector::dump_chunks() const {
     std::lock_guard lock(m_chunk_sel_mutex);
+    std::string str;
     for (uint32_t i = 0; i < m_volume_chunks.size(); i++) {
         if (!m_volume_chunks[i]) { continue; }
-        std::string str;
+        fmt::format_to(std::back_inserter(str), "volume={} num_chunks={} chunks=", i, m_volume_chunks[i]->m_chunks.size());
         for (const auto& chunk : m_volume_chunks[i]->m_chunks) {
             if (!chunk) { continue; }
             fmt::format_to(std::back_inserter(str), "{}({}/{}) ", chunk->get_chunk_id(), chunk->available_blks(),
                            chunk->get_total_blks());
         }
-        LOGI("volume={} num_chunks={} chunks={}", i, m_volume_chunks[i]->m_chunks.size(), str);
+        fmt::format_to(std::back_inserter(str), "\n");
     }
+    return str;
 }
 
 } // namespace homeblocks

src/lib/volume/volume_chunk_selector.cpp

@@ -88,7 +88,7 @@ class VolumeChunkSelector : public homestore::ChunkSelector {
     std::vector< shared< HBChunk > > allocate_resize_chunks_from_pdev(uint32_t pdev, uint64_t num_chunks);
     void resize_volume_num_chunks(homestore::blk_count_t nblks, shared< VolumeChunksInfo > volc);
     void dump_per_pdev_chunks() const;
-    void dump_chunks() const;
+    std::string dump_chunks() const;
 
 private:
     enum class ResizeOp {