feat: add ls command to LittDB CLI

litt/cli/litt_cli.go

@@ -34,7 +34,7 @@ func buildCLIParser(logger logging.Logger) *cli.App {
 						Required: true,
 					},
 				},
-				Action: nil, // lsCommand, // TODO this will be added in a follow up PR
+				Action: lsCommand,
 			},
 			{
 				Name: "table-info",

litt/cli/ls.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/Layr-Labs/eigenda/common"
+	"github.com/Layr-Labs/eigenda/litt/disktable/segment"
+	"github.com/Layr-Labs/eigenda/litt/util"
+	"github.com/Layr-Labs/eigensdk-go/logging"
+	"github.com/urfave/cli/v2"
+)
+
+func lsCommand(ctx *cli.Context) error {
+	logger, err := common.NewLogger(common.DefaultConsoleLoggerConfig())
+	if err != nil {
+		return fmt.Errorf("failed to create logger: %w", err)
+	}
+
+	sources := ctx.StringSlice("src")
+	if len(sources) == 0 {
+		return fmt.Errorf("no sources provided")
+	}
+	for i, src := range sources {
+		var err error
+		sources[i], err = util.SanitizePath(src)
+		if err != nil {
+			return fmt.Errorf("invalid source path: %s", src)
+		}
+	}
+
+	tables, err := lsPaths(logger, sources, true, true)
+	if err != nil {
+		return fmt.Errorf("failed to list tables in paths %v: %w", sources, err)
+	}
+
+	sb := &strings.Builder{}
+	for _, table := range tables {
+		sb.WriteString(table)
+		sb.WriteString("\n")
+	}
+
+	logger.Infof("Tables found:\n%s", sb.String())
+
+	return nil
+}
+
+// Similar to ls, but searches for tables in multiple paths.
+func lsPaths(logger logging.Logger, rootPaths []string, lock bool, fsync bool) ([]string, error) {
+	tableSet := make(map[string]struct{})
+
+	for _, rootPath := range rootPaths {
+		tables, err := ls(logger, rootPath, lock, fsync)
+		if err != nil {
+			return nil, fmt.Errorf("error finding tables: %w", err)
+		}
+		for _, table := range tables {
+			tableSet[table] = struct{}{}
+		}
+	}
+
+	tableNames := make([]string, 0, len(tableSet))
+	for tableName := range tableSet {
+		tableNames = append(tableNames, tableName)
+	}
+
+	sort.Strings(tableNames)
+
+	return tableNames, nil
+}
+
+// Returns a list of LittDB tables at the specified LittDB path. Tables are alphabetically sorted by their names.
+// Returns an error if the path does not exist or if no tables are found.
+func ls(logger logging.Logger, rootPath string, lock bool, fsync bool) ([]string, error) {
+
+	if lock {
+		// Forbid touching tables in active use.
+		lockPath := path.Join(rootPath, util.LockfileName)
+		fLock, err := util.NewFileLock(logger, lockPath, fsync)
+		if err != nil {
+			return nil, fmt.Errorf("failed to acquire lock on %s: %w", rootPath, err)
+		}
+		defer fLock.Release()
+	}
+
+	// LittDB has one directory under the root directory per table, with the name
+	// of the table being the name of the directory.
+	possibleTables, err := os.ReadDir(rootPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read dir %s: %w", rootPath, err)
+	}
+
+	// Each table directory will contain a "segments" directory. Infer that any directory containing this directory
+	// is a table. If we are looking at a real LittDB instance, there shouldn't be any other directories, but
+	// there is no need to enforce that here.
+	tables := make([]string, 0, len(possibleTables))
+	for _, entry := range possibleTables {
+		if !entry.IsDir() {
+			continue
+		}
+
+		segmentPath := filepath.Join(rootPath, entry.Name(), segment.SegmentDirectory)
+		isDirectory, err := util.IsDirectory(segmentPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to check if segment path %s is a directory: %w", segmentPath, err)
+		}
+		if isDirectory {
+			tables = append(tables, entry.Name())
+		}
+	}
+
+	// Alphabetically sort the tables.
+	sort.Strings(tables)
+
+	return tables, nil
+}

litt/cli/ls_test.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/Layr-Labs/eigenda/common"
+	"github.com/Layr-Labs/eigenda/common/testutils/random"
+	"github.com/Layr-Labs/eigenda/litt"
+	"github.com/Layr-Labs/eigenda/litt/littbuilder"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLs(t *testing.T) {
+	t.Parallel()
+
+	logger, err := common.NewLogger(common.DefaultConsoleLoggerConfig())
+	require.NoError(t, err)
+
+	rand := random.NewTestRandom()
+	directory := t.TempDir()
+
+	// Spread data across several root directories.
+	rootCount := rand.Uint32Range(2, 5)
+	roots := make([]string, 0, rootCount)
+	for i := 0; i < int(rootCount); i++ {
+		roots = append(roots, fmt.Sprintf("%s/root-%d", directory, i))
+	}
+
+	config, err := litt.DefaultConfig(roots...)
+	require.NoError(t, err)
+
+	// Make it so that we have at least as many shards as roots.
+	config.ShardingFactor = rootCount * rand.Uint32Range(1, 4)
+
+	// Settings that should be enabled for LittDB unit tests.
+	config.DoubleWriteProtection = true
+	config.Fsync = false
+
+	// Use small segments to ensure that we create a few segments per table.
+	config.TargetSegmentFileSize = 100
+
+	// Enable snapshotting.
+	snapshotDir := t.TempDir()
+	config.SnapshotDirectory = snapshotDir
+
+	// Build the DB and a handful of tables.
+	db, err := littbuilder.NewDB(config)
+	require.NoError(t, err)
+
+	tableCount := rand.Uint32Range(2, 5)
+	tables := make([]litt.Table, 0, tableCount)
+	expectedData := make(map[string]map[string][]byte)
+	tableNames := make([]string, 0, tableCount)
+	for i := 0; i < int(tableCount); i++ {
+		tableName := fmt.Sprintf("table-%d-%s", i, rand.PrintableBytes(8))
+		table, err := db.GetTable(tableName)
+		require.NoError(t, err)
+		tables = append(tables, table)
+		expectedData[table.Name()] = make(map[string][]byte)
+		tableNames = append(tableNames, tableName)
+	}
+
+	// Alphabetize table names. ls should always return tables in this order.
+	sort.Strings(tableNames)
+
+	// Insert some data into the tables.
+	for _, table := range tables {
+		for i := 0; i < 100; i++ {
+			key := rand.PrintableBytes(32)
+			value := rand.PrintableVariableBytes(10, 200)
+			expectedData[table.Name()][string(key)] = value
+			err = table.Put(key, value)
+			require.NoError(t, err, "Failed to put key-value pair in table %s", table.Name())
+		}
+		err = table.Flush()
+		require.NoError(t, err, "Failed to flush table %s", table.Name())
+	}
+
+	// Verify that the data is correctly stored in the tables.
+	for _, table := range tables {
+		for key, expectedValue := range expectedData[table.Name()] {
+			value, ok, err := table.Get([]byte(key))
+			require.NoError(t, err, "Failed to get value for key %s in table %s", key, table.Name())
+			require.True(t, ok, "Key %s not found in table %s", key, table.Name())
+			require.Equal(t, expectedValue, value,
+				"Value mismatch for key %s in table %s", key, table.Name())
+		}
+	}
+
+	// We should not be able to call ls on the core directories while the table holds a lock.
+	for _, root := range roots {
+		_, err = ls(logger, root, true, false)
+		require.Error(t, err)
+	}
+	_, err = lsPaths(logger, roots, true, false)
+	require.Error(t, err)
+
+	// Even when the DB is running, it should always be possible to ls the snapshot directory.
+	lsResult, err := ls(logger, snapshotDir, true, false)
+	require.NoError(t, err)
+	require.Equal(t, tableNames, lsResult)
+
+	lsResult, err = lsPaths(logger, []string{snapshotDir}, true, false)
+	require.NoError(t, err)
+	require.Equal(t, tableNames, lsResult)
+
+	err = db.Close()
+	require.NoError(t, err)
+
+	// Now that the DB is closed, we should be able to ls it. We should find all tables defined regardless of which
+	// root directory we peer into.
+	for _, root := range roots {
+		lsResult, err = ls(logger, root, true, false)
+		require.NoError(t, err)
+		require.Equal(t, tableNames, lsResult)
+	}
+
+	lsResult, err = lsPaths(logger, roots, true, true)
+	require.NoError(t, err)
+	require.Equal(t, tableNames, lsResult)
+
+	// Data should still be present in the snapshot directory.
+	lsResult, err = ls(logger, snapshotDir, true, false)
+	require.NoError(t, err)
+	require.Equal(t, tableNames, lsResult)
+}

litt/util/file_utils.go

@@ -37,6 +37,19 @@ func ErrIfSymlink(path string) error {
 	return nil
 }
 
+// IsDirectory checks if the given path is a directory. Returns false if the path is not a directory or does not exist.
+func IsDirectory(path string) (bool, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Path does not exist, so it can't be a directory
+			return false, nil
+		}
+		return false, fmt.Errorf("failed to stat path %s: %w", path, err)
+	}
+	return info.IsDir(), nil
+}
+
 // SanitizePath returns a sanitized version of the given path, doing things like expanding
 // "~" to the user's home directory, converting to absolute path, normalizing slashes, etc.
 func SanitizePath(path string) (string, error) {

litt/util/file_utils_test.go

@@ -1413,3 +1413,29 @@ func TestDeepDelete(t *testing.T) {
 	require.NoError(t, err, "Failed to check if original non-empty directory exists after deleting symlink")
 	require.True(t, exists, "Original non-empty directory should still exist after failed deletion")
 }
+
+func TestIsDirectory(t *testing.T) {
+	testDir := t.TempDir()
+
+	// non-existent path
+	nonExistentPath := filepath.Join(testDir, "non-existent-dir")
+	isDir, err := IsDirectory(nonExistentPath)
+	require.NoError(t, err, "IsDirectory should not return an error for non-existent path")
+	require.False(t, isDir, "Non-existent path should not be a directory")
+
+	// path is a file
+	filePath := filepath.Join(testDir, "file.txt")
+	err = os.WriteFile(filePath, []byte("test content"), 0644)
+	require.NoError(t, err, "Failed to create test file")
+	isDir, err = IsDirectory(filePath)
+	require.NoError(t, err, "IsDirectory should not return an error for file path")
+	require.False(t, isDir, "File path should not be a directory")
+
+	// path is a directory
+	dirPath := filepath.Join(testDir, "test-dir")
+	err = os.Mkdir(dirPath, 0755)
+	require.NoError(t, err, "Failed to create test directory")
+	isDir, err = IsDirectory(dirPath)
+	require.NoError(t, err, "IsDirectory should not return an error for directory path")
+	require.True(t, isDir, "Directory path should be recognized as a directory")
+}