feat(core): Add Data Store Backend API (no-changelog)

packages/@n8n/api-types/src/dto/data-store/add-data-store-column.dto.ts

@@ -0,0 +1,5 @@
+import { Z } from 'zod-class';
+
+import { dataStoreCreateColumnSchema } from '../../schemas/data-store.schema';
+
+export class AddDataStoreColumnDto extends Z.class(dataStoreCreateColumnSchema.shape) {}

packages/@n8n/api-types/src/dto/data-store/add-data-store-records.dto.ts

@@ -0,0 +1,6 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+export class AddDatastoreRecordsDto extends Z.class({
+	records: z.array(z.record(z.string(), z.any())),
+}) {}

packages/@n8n/api-types/src/dto/data-store/create-data-store-column.dto.ts

@@ -0,0 +1,11 @@
+import { Z } from 'zod-class';
+
+import {
+	dataStoreColumnNameSchema,
+	dataStoreColumnTypeSchema,
+} from '../../schemas/data-store.schema';
+
+export class CreateDataStoreColumnDto extends Z.class({
+	name: dataStoreColumnNameSchema,
+	type: dataStoreColumnTypeSchema,
+}) {}

packages/@n8n/api-types/src/dto/data-store/create-data-store.dto.ts

@@ -0,0 +1,10 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { CreateDataStoreColumnDto } from './create-data-store-column.dto';
+import { dataStoreNameSchema } from '../../schemas/data-store.schema';
+
+export class CreateDataStoreDto extends Z.class({
+	name: dataStoreNameSchema,
+	columns: z.array(CreateDataStoreColumnDto),
+}) {}

packages/@n8n/api-types/src/dto/data-store/delete-data-store-column.dto.ts

@@ -0,0 +1,7 @@
+import { Z } from 'zod-class';
+
+import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
+
+export class DeleteDataStoreColumnDto extends Z.class({
+	columnName: dataStoreColumnNameSchema,
+}) {}

packages/@n8n/api-types/src/dto/data-store/delete-data-store.dto.ts

@@ -0,0 +1,7 @@
+import { Z } from 'zod-class';
+
+import { dataStoreIdSchema } from '../../schemas/data-store.schema';
+
+export class DeleteDataStoreDto extends Z.class({
+	id: dataStoreIdSchema,
+}) {}

packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts

@@ -0,0 +1,76 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
+import { paginationSchema } from '../pagination/pagination.dto';
+
+const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]);
+export type ListDataStoreContentFilterConditionType = z.infer<typeof FilterConditionSchema>;
+
+const filterRecord = z.object({
+	columnName: dataStoreColumnNameSchema,
+	condition: FilterConditionSchema.default('eq'),
+	value: z.union([z.string(), z.number(), z.boolean(), z.date()]),
+});
+
+const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]);
+
+export type ListDataStoreContentFilter = z.infer<typeof filterValidator>;
+
+// ---------------------
+// Parameter Validators
+// ---------------------
+
+// Filter parameter validation
+const filterValidator = z.object({
+	type: chainedFilterSchema.default('and'),
+	filters: z.array(filterRecord).default([]),
+});
+
+// SortBy parameter validation
+const sortByValidator = z
+	.string()
+	.optional()
+	.transform((val, ctx) => {
+		if (val === undefined) return val;
+
+		if (!val.includes(':')) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort format, expected <columnName>:<asc/desc>',
+				path: ['sort'],
+			});
+			return z.NEVER;
+		}
+
+		let [column, direction] = val.split(':');
+
+		try {
+			column = dataStoreColumnNameSchema.parse(column);
+		} catch {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort columnName',
+				path: ['sort'],
+			});
+			return z.NEVER;
+		}
+
+		direction = direction?.toUpperCase();
+		if (direction !== 'ASC' && direction !== 'DESC') {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort direction',
+				path: ['sort'],
+			});
+
+			return z.NEVER;
+		}
+		return [column, direction] as const;
+	});
+
+export class ListDataStoreContentQueryDto extends Z.class({
+	...paginationSchema,
+	filter: filterValidator,
+	sortBy: sortByValidator,
+}) {}

packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts

@@ -0,0 +1,69 @@
+import { jsonParse } from 'n8n-workflow';
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { paginationSchema } from '../pagination/pagination.dto';
+
+const VALID_SORT_OPTIONS = [
+	'name:asc',
+	'name:desc',
+	'createdAt:asc',
+	'createdAt:desc',
+	'updatedAt:asc',
+	'updatedAt:desc',
+	'sizeBytes:asc',
+	'sizeBytes:desc',
+] as const;
+
+export type ListDataStoreQuerySortOptions = (typeof VALID_SORT_OPTIONS)[number];
+
+const FILTER_OPTIONS = {
+	id: z.union([z.string(), z.array(z.string())]).optional(),
+	name: z.union([z.string(), z.array(z.string())]).optional(),
+	// todo: can probably include others here as well?
+};
+
+// Filter schema - only allow specific properties
+const filterSchema = z.object(FILTER_OPTIONS).strict();
+// ---------------------
+// Parameter Validators
+// ---------------------
+
+// Filter parameter validation
+const filterValidator = z
+	.string()
+	.optional()
+	.transform((val, ctx) => {
+		if (!val) return undefined;
+		try {
+			const parsed: unknown = jsonParse(val);
+			try {
+				return filterSchema.parse(parsed);
+			} catch (e) {
+				ctx.addIssue({
+					code: z.ZodIssueCode.custom,
+					message: 'Invalid filter fields',
+					path: ['filter'],
+				});
+				return z.NEVER;
+			}
+		} catch (e) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid filter format',
+				path: ['filter'],
+			});
+			return z.NEVER;
+		}
+	});
+
+// SortBy parameter validation
+const sortByValidator = z
+	.enum(VALID_SORT_OPTIONS, { message: `sortBy must be one of: ${VALID_SORT_OPTIONS.join(', ')}` })
+	.optional();
+
+export class ListDataStoreQueryDto extends Z.class({
+	...paginationSchema,
+	filter: filterValidator,
+	sortBy: sortByValidator,
+}) {}

packages/@n8n/api-types/src/dto/data-store/rename-data-store.dto.ts

@@ -0,0 +1,7 @@
+import { Z } from 'zod-class';
+
+import { dataStoreNameSchema } from '../../schemas/data-store.schema';
+
+export class RenameDataStoreDto extends Z.class({
+	name: dataStoreNameSchema,
+}) {}

packages/@n8n/api-types/src/dto/index.ts

@@ -79,3 +79,12 @@ export {
 } from './user/users-list-filter.dto';
 
 export { OidcConfigDto } from './oidc/config.dto';
+
+export { CreateDataStoreDto } from './data-store/create-data-store.dto';
+export { RenameDataStoreDto } from './data-store/rename-data-store.dto';
+export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto';
+export { ListDataStoreContentQueryDto } from './data-store/list-data-store-content-query.dto';
+export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto';
+export { AddDatastoreRecordsDto } from './data-store/add-data-store-records.dto';
+export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto';
+export { DeleteDataStoreColumnDto } from './data-store/delete-data-store-column.dto';

packages/@n8n/api-types/src/index.ts

@@ -45,3 +45,5 @@ export {
 	type UsersList,
 	usersListSchema,
 } from './schemas/user.schema';
+
+export type { DataStore, DataStoreColumn } from './schemas/data-store.schema';

packages/@n8n/api-types/src/schemas/data-store.schema.ts

@@ -0,0 +1,40 @@
+import { z } from 'zod';
+
+export const dataStoreNameSchema = z.string().trim().min(1).max(128);
+export const dataStoreIdSchema = z.string().max(36);
+
+export const DATA_STORE_COLUMN_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9-]*$/;
+
+export const dataStoreColumnNameSchema = z
+	.string()
+	.trim()
+	.min(1)
+	.max(128)
+	.regex(
+		DATA_STORE_COLUMN_REGEX,
+		'Only alphanumeric characters and non-leading dashes are allowed for column names',
+	);
+export const dataStoreColumnTypeSchema = z.enum(['string', 'number', 'boolean', 'date']);
+
+export const dataStoreCreateColumnSchema = z.object({
+	name: dataStoreColumnNameSchema,
+	type: dataStoreColumnTypeSchema,
+	columnIndex: z.number().optional(),
+});
+export type DataStoreCreateColumnSchema = z.infer<typeof dataStoreCreateColumnSchema>;
+
+export const dataStoreColumnSchema = dataStoreCreateColumnSchema.extend({
+	createdAt: z.string().datetime(),
+	updatedAt: z.string().datetime(),
+	dataStoreId: dataStoreIdSchema,
+});
+
+export const dataStoreSchema = z.object({
+	id: dataStoreIdSchema,
+	name: dataStoreNameSchema,
+	columns: z.array(dataStoreColumnSchema),
+	createdAt: z.string().datetime(),
+	updatedAt: z.string().datetime(),
+});
+export type DataStore = z.infer<typeof dataStoreSchema>;
+export type DataStoreColumn = z.infer<typeof dataStoreColumnSchema>;

packages/@n8n/backend-common/src/modules/module-registry.ts

@@ -23,7 +23,7 @@ export class ModuleRegistry {
 		private readonly modulesConfig: ModulesConfig,
 	) {}
 
-	private readonly defaultModules: ModuleName[] = ['insights', 'external-secrets'];
+	private readonly defaultModules: ModuleName[] = ['insights', 'external-secrets', 'data-store'];
 
 	private readonly activeModules: string[] = [];
 

packages/@n8n/backend-common/src/modules/modules.config.ts

@@ -2,7 +2,7 @@ import { CommaSeparatedStringArray, Config, Env } from '@n8n/config';
 
 import { UnknownModuleError } from './errors/unknown-module.error';
 
-export const MODULE_NAMES = ['insights', 'external-secrets'] as const;
+export const MODULE_NAMES = ['insights', 'external-secrets', 'data-store'] as const;
 
 export type ModuleName = (typeof MODULE_NAMES)[number];
 

packages/@n8n/backend-test-utils/src/test-db.ts

@@ -66,7 +66,13 @@ export async function terminate() {
 	dbConnection.connectionState.connected = false;
 }
 
-type EntityName = keyof typeof entities | 'InsightsRaw' | 'InsightsByPeriod' | 'InsightsMetadata';
+type EntityName =
+	| keyof typeof entities
+	| 'InsightsRaw'
+	| 'InsightsByPeriod'
+	| 'InsightsMetadata'
+	| 'DataStoreEntity'
+	| 'DataStoreColumnEntity';
 
 /**
  * Truncate specific DB tables in a test DB.

packages/@n8n/config/src/configs/logging.config.ts

@@ -19,6 +19,7 @@ export const LOG_SCOPES = [
 	'insights',
 	'workflow-activation',
 	'ssh-client',
+	'data-store',
 ] as const;
 
 export type LogScope = (typeof LOG_SCOPES)[number];

packages/@n8n/db/src/migrations/common/1747814180618-CreateDataStoreTables.ts

@@ -0,0 +1,44 @@
+import type { MigrationContext, ReversibleMigration } from '../migration-types';
+
+const dataStoreTableName = 'data_store_entity';
+const dataStoreColumnTableName = 'data_store_column_entity';
+
+export class CreateDataStoreTables1747814180618 implements ReversibleMigration {
+	async up({ schemaBuilder: { createTable, column } }: MigrationContext) {
+		await createTable(dataStoreTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('name').varchar(128).notNull,
+				column('projectId').varchar(36).notNull,
+				column('sizeBytes').int.notNull,
+			)
+			.withForeignKey('projectId', {
+				tableName: 'project', // @Review: this hardcodes a dependency on the project table, is this cool?
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			})
+			.withIndexOn(['projectId', 'name'], true).withTimestamps; // @Review: The intention here is to ensure each name can only be used once per project, so [projectId, name] needs to be unique, right?...
+
+		await createTable(dataStoreColumnTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('name').varchar(128).notNull,
+				column('type').varchar(32).notNull,
+				column('columnIndex').int.notNull,
+				column('dataStoreId').varchar(36).notNull,
+			)
+			.withForeignKey('dataStoreId', {
+				tableName: dataStoreTableName,
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			})
+			.withIndexOn(['dataStoreId', 'name'], true).withTimestamps;
+	}
+
+	async down({ schemaBuilder: { dropTable } }: MigrationContext) {
+		await dropTable(dataStoreTableName);
+		await dropTable(dataStoreColumnTableName);
+
+		// @Review: Should we also drop all created user tables here?
+	}
+}

packages/@n8n/db/src/migrations/sqlite/index.ts

@@ -83,6 +83,7 @@ import { ClearEvaluation1745322634000 } from '../common/1745322634000-CleanEvalu
 import { AddWorkflowStatisticsRootCount1745587087521 } from '../common/1745587087521-AddWorkflowStatisticsRootCount';
 import { AddWorkflowArchivedColumn1745934666076 } from '../common/1745934666076-AddWorkflowArchivedColumn';
 import { DropRoleTable1745934666077 } from '../common/1745934666077-DropRoleTable';
+import { CreateDataStoreTables1747814180618 } from '../common/1747814180618-CreateDataStoreTables';
 import { AddProjectDescriptionColumn1747824239000 } from '../common/1747824239000-AddProjectDescriptionColumn';
 import { AddLastActiveAtColumnToUser1750252139166 } from '../common/1750252139166-AddLastActiveAtColumnToUser';
 import type { Migration } from '../migration-types';
@@ -172,6 +173,7 @@ const sqliteMigrations: Migration[] = [
 	AddWorkflowArchivedColumn1745934666076,
 	DropRoleTable1745934666077,
 	ClearEvaluation1745322634000,
+	CreateDataStoreTables1747814180618,
 	AddProjectDescriptionColumn1747824239000,
 	AddLastActiveAtColumnToUser1750252139166,
 ];

packages/@n8n/permissions/src/constants.ee.ts

@@ -26,6 +26,7 @@ export const RESOURCES = {
 	folder: [...DEFAULT_OPERATIONS, 'move'] as const,
 	insights: ['list'] as const,
 	oidc: ['manage'] as const,
+	dataStore: [...DEFAULT_OPERATIONS, 'readRow', 'writeRow'] as const,
 } as const;
 
 export const API_KEY_RESOURCES = {

packages/@n8n/permissions/src/roles/scopes/global-scopes.ee.ts

@@ -79,6 +79,13 @@ export const GLOBAL_OWNER_SCOPES: Scope[] = [
 	'insights:list',
 	'folder:move',
 	'oidc:manage',
+	'dataStore:create',
+	'dataStore:delete',
+	'dataStore:read',
+	'dataStore:update',
+	'dataStore:list',
+	'dataStore:readRow',
+	'dataStore:writeRow',
 ];
 
 export const GLOBAL_ADMIN_SCOPES = GLOBAL_OWNER_SCOPES.concat();
@@ -98,4 +105,7 @@ export const GLOBAL_MEMBER_SCOPES: Scope[] = [
 	'user:list',
 	'variable:list',
 	'variable:read',
+	'dataStore:read',
+	'dataStore:list',
+	'dataStore:readRow',
 ];