feat(core): Add Data Store Backend API (no-changelog)

packages/@n8n/api-types/src/dto/data-store/add-data-store-column.dto.ts

@@ -0,0 +1,5 @@
+import { Z } from 'zod-class';
+
+import { dataStoreCreateColumnSchema } from '../../schemas/data-store.schema';
+
+export class AddDataStoreColumnDto extends Z.class(dataStoreCreateColumnSchema.shape) {}

packages/@n8n/api-types/src/dto/data-store/add-data-store-rows.dto.ts

@@ -0,0 +1,8 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
+
+export class AddDataStoreRowsDto extends Z.class({
+	data: z.array(z.record(dataStoreColumnNameSchema, z.any())),
+}) {}

packages/@n8n/api-types/src/dto/data-store/create-data-store-column.dto.ts

@@ -0,0 +1,11 @@
+import { Z } from 'zod-class';
+
+import {
+	dataStoreColumnNameSchema,
+	dataStoreColumnTypeSchema,
+} from '../../schemas/data-store.schema';
+
+export class CreateDataStoreColumnDto extends Z.class({
+	name: dataStoreColumnNameSchema,
+	type: dataStoreColumnTypeSchema,
+}) {}

packages/@n8n/api-types/src/dto/data-store/create-data-store.dto.ts

@@ -0,0 +1,10 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { CreateDataStoreColumnDto } from './create-data-store-column.dto';
+import { dataStoreNameSchema } from '../../schemas/data-store.schema';
+
+export class CreateDataStoreDto extends Z.class({
+	name: dataStoreNameSchema,
+	columns: z.array(CreateDataStoreColumnDto),
+}) {}

packages/@n8n/api-types/src/dto/data-store/delete-data-store-column.dto.ts

@@ -0,0 +1,6 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+export class DeleteDataStoreColumnDto extends Z.class({
+	columnId: z.string(),
+}) {}

packages/@n8n/api-types/src/dto/data-store/list-data-store-content-query.dto.ts

@@ -0,0 +1,76 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
+import { paginationSchema } from '../pagination/pagination.dto';
+
+const FilterConditionSchema = z.union([z.literal('eq'), z.literal('neq')]);
+export type ListDataStoreContentFilterConditionType = z.infer<typeof FilterConditionSchema>;
+
+const filterRecord = z.object({
+	columnName: dataStoreColumnNameSchema,
+	condition: FilterConditionSchema.default('eq'),
+	value: z.union([z.string(), z.number(), z.boolean(), z.date()]),
+});
+
+const chainedFilterSchema = z.union([z.literal('and'), z.literal('or')]);
+
+export type ListDataStoreContentFilter = z.infer<typeof filterValidator>;
+
+// ---------------------
+// Parameter Validators
+// ---------------------
+
+// Filter parameter validation
+const filterValidator = z.object({
+	type: chainedFilterSchema.default('and'),
+	filters: z.array(filterRecord).default([]),
+});
+
+// SortBy parameter validation
+const sortByValidator = z
+	.string()
+	.optional()
+	.transform((val, ctx) => {
+		if (val === undefined) return val;
+
+		if (!val.includes(':')) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort format, expected <columnName>:<asc/desc>',
+				path: ['sort'],
+			});
+			return z.NEVER;
+		}
+
+		let [column, direction] = val.split(':');
+
+		try {
+			column = dataStoreColumnNameSchema.parse(column);
+		} catch {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort columnName',
+				path: ['sort'],
+			});
+			return z.NEVER;
+		}
+
+		direction = direction?.toUpperCase();
+		if (direction !== 'ASC' && direction !== 'DESC') {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid sort direction',
+				path: ['sort'],
+			});
+
+			return z.NEVER;
+		}
+		return [column, direction] as const;
+	});
+
+export class ListDataStoreContentQueryDto extends Z.class({
+	...paginationSchema,
+	filter: filterValidator,
+	sortBy: sortByValidator,
+}) {}

packages/@n8n/api-types/src/dto/data-store/list-data-store-query.dto.ts

@@ -0,0 +1,70 @@
+import { jsonParse } from 'n8n-workflow';
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { paginationSchema } from '../pagination/pagination.dto';
+
+const VALID_SORT_OPTIONS = [
+	'name:asc',
+	'name:desc',
+	'createdAt:asc',
+	'createdAt:desc',
+	'updatedAt:asc',
+	'updatedAt:desc',
+	'sizeBytes:asc',
+	'sizeBytes:desc',
+] as const;
+
+export type ListDataStoreQuerySortOptions = (typeof VALID_SORT_OPTIONS)[number];
+
+const FILTER_OPTIONS = {
+	id: z.union([z.string(), z.array(z.string())]).optional(),
+	name: z.union([z.string(), z.array(z.string())]).optional(),
+	projectId: z.union([z.string(), z.array(z.string())]).optional(),
+	// todo: can probably include others here as well?
+};
+
+// Filter schema - only allow specific properties
+const filterSchema = z.object(FILTER_OPTIONS).strict();
+// ---------------------
+// Parameter Validators
+// ---------------------
+
+// Filter parameter validation
+const filterValidator = z
+	.string()
+	.optional()
+	.transform((val, ctx) => {
+		if (!val) return undefined;
+		try {
+			const parsed: unknown = jsonParse(val);
+			try {
+				return filterSchema.parse(parsed);
+			} catch (e) {
+				ctx.addIssue({
+					code: z.ZodIssueCode.custom,
+					message: 'Invalid filter fields',
+					path: ['filter'],
+				});
+				return z.NEVER;
+			}
+		} catch (e) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				message: 'Invalid filter format',
+				path: ['filter'],
+			});
+			return z.NEVER;
+		}
+	});
+
+// SortBy parameter validation
+const sortByValidator = z
+	.enum(VALID_SORT_OPTIONS, { message: `sortBy must be one of: ${VALID_SORT_OPTIONS.join(', ')}` })
+	.optional();
+
+export class ListDataStoreQueryDto extends Z.class({
+	...paginationSchema,
+	filter: filterValidator,
+	sortBy: sortByValidator,
+}) {}

packages/@n8n/api-types/src/dto/data-store/move-data-store-column.dto.ts

@@ -0,0 +1,6 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+export class MoveDataStoreColumnDto extends Z.class({
+	targetIndex: z.number(),
+}) {}

packages/@n8n/api-types/src/dto/data-store/update-data-store.dto.ts

@@ -0,0 +1,7 @@
+import { Z } from 'zod-class';
+
+import { dataStoreNameSchema } from '../../schemas/data-store.schema';
+
+export class UpdateDataStoreDto extends Z.class({
+	name: dataStoreNameSchema,
+}) {}

packages/@n8n/api-types/src/dto/data-store/upsert-data-store-rows.dto.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod';
+import { Z } from 'zod-class';
+
+import { dataStoreColumnNameSchema } from '../../schemas/data-store.schema';
+
+const dataStoreValueSchema = z.union([z.string(), z.number(), z.boolean(), z.date(), z.null()]);
+
+const upsertDataStoreRowsShape = {
+	rows: z.array(z.record(dataStoreValueSchema)),
+	matchFields: z.array(dataStoreColumnNameSchema).min(1),
+};
+
+export class UpsertDataStoreRowsDto extends Z.class(upsertDataStoreRowsShape) {}

packages/@n8n/api-types/src/dto/index.ts

@@ -79,3 +79,14 @@ export {
 } from './user/users-list-filter.dto';
 
 export { OidcConfigDto } from './oidc/config.dto';
+
+export { CreateDataStoreDto } from './data-store/create-data-store.dto';
+export { UpdateDataStoreDto } from './data-store/update-data-store.dto';
+export { UpsertDataStoreRowsDto } from './data-store/upsert-data-store-rows.dto';
+export { ListDataStoreQueryDto } from './data-store/list-data-store-query.dto';
+export { ListDataStoreContentQueryDto } from './data-store/list-data-store-content-query.dto';
+export { CreateDataStoreColumnDto } from './data-store/create-data-store-column.dto';
+export { AddDataStoreRowsDto } from './data-store/add-data-store-rows.dto';
+export { AddDataStoreColumnDto } from './data-store/add-data-store-column.dto';
+export { DeleteDataStoreColumnDto } from './data-store/delete-data-store-column.dto';
+export { MoveDataStoreColumnDto } from './data-store/move-data-store-column.dto';

packages/@n8n/api-types/src/index.ts

@@ -45,3 +45,15 @@ export {
 	type UsersList,
 	usersListSchema,
 } from './schemas/user.schema';
+
+export {
+	DATA_STORE_COLUMN_REGEX,
+	type DataStore,
+	type DataStoreColumn,
+	type DataStoreCreateColumnSchema,
+	type DataStoreColumnJsType,
+	type DataStoreListFilter,
+	type DataStoreRows,
+	type DataStoreListOptions,
+	type DataStoreUserTableName,
+} from './schemas/data-store.schema';

packages/@n8n/api-types/src/schemas/data-store.schema.ts

@@ -0,0 +1,56 @@
+import { z } from 'zod';
+
+import type { ListDataStoreQueryDto } from '../dto';
+
+export const dataStoreNameSchema = z.string().trim().min(1).max(128);
+export const dataStoreIdSchema = z.string().max(36);
+
+export const DATA_STORE_COLUMN_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9-]*$/;
+
+export const dataStoreColumnNameSchema = z
+	.string()
+	.trim()
+	.min(1)
+	.max(128)
+	.regex(
+		DATA_STORE_COLUMN_REGEX,
+		'Only alphanumeric characters and non-leading dashes are allowed for column names',
+	);
+export const dataStoreColumnTypeSchema = z.enum(['string', 'number', 'boolean', 'date']);
+
+export const dataStoreCreateColumnSchema = z.object({
+	name: dataStoreColumnNameSchema,
+	type: dataStoreColumnTypeSchema,
+	columnIndex: z.number().optional(),
+});
+export type DataStoreCreateColumnSchema = z.infer<typeof dataStoreCreateColumnSchema>;
+
+export const dataStoreColumnSchema = dataStoreCreateColumnSchema.extend({
+	dataStoreId: dataStoreIdSchema,
+});
+
+export const dataStoreSchema = z.object({
+	id: dataStoreIdSchema,
+	name: dataStoreNameSchema,
+	columns: z.array(dataStoreColumnSchema),
+	createdAt: z.string().datetime(),
+	updatedAt: z.string().datetime(),
+});
+export type DataStore = z.infer<typeof dataStoreSchema>;
+export type DataStoreColumn = z.infer<typeof dataStoreColumnSchema>;
+
+export type DataStoreUserTableName = `data_store_user_${string}`;
+
+export type DataStoreListFilter = {
+	id?: string | string[];
+	projectId?: string | string[];
+	name?: string;
+};
+
+export type DataStoreListOptions = Partial<ListDataStoreQueryDto> & {
+	filter: { projectId: string };
+};
+
+export type DataStoreColumnJsType = string | number | boolean | Date;
+
+export type DataStoreRows = Array<Record<string, DataStoreColumnJsType | null>>;

packages/@n8n/backend-test-utils/src/test-db.ts

@@ -66,7 +66,13 @@ export async function terminate() {
 	dbConnection.connectionState.connected = false;
 }
 
-type EntityName = keyof typeof entities | 'InsightsRaw' | 'InsightsByPeriod' | 'InsightsMetadata';
+type EntityName =
+	| keyof typeof entities
+	| 'InsightsRaw'
+	| 'InsightsByPeriod'
+	| 'InsightsMetadata'
+	| 'DataStoreEntity'
+	| 'DataStoreColumnEntity';
 
 /**
  * Truncate specific DB tables in a test DB.

packages/@n8n/config/src/configs/logging.config.ts

@@ -19,6 +19,7 @@ export const LOG_SCOPES = [
 	'insights',
 	'workflow-activation',
 	'ssh-client',
+	'data-store',
 	'cron',
 	'community-nodes',
 	'legacy-sqlite-execution-recovery',

packages/@n8n/db/src/index.ts

@@ -23,6 +23,8 @@ export { NoUrl } from './utils/validators/no-url.validator';
 export * from './repositories';
 export * from './subscribers';
 
+export { Column as DslColumn } from './migrations/dsl/column';
+export { CreateTable } from './migrations/dsl/table';
 export { sqliteMigrations } from './migrations/sqlite';
 export { mysqlMigrations } from './migrations/mysqldb';
 export { postgresMigrations } from './migrations/postgresdb';

packages/@n8n/db/src/migrations/common/1747814180618-CreateDataStoreTables.ts

@@ -0,0 +1,44 @@
+import type { MigrationContext, ReversibleMigration } from '../migration-types';
+
+const dataStoreTableName = 'data_store_entity';
+const dataStoreColumnTableName = 'data_store_column_entity';
+
+export class CreateDataStoreTables1747814180618 implements ReversibleMigration {
+	async up({ schemaBuilder: { createTable, column } }: MigrationContext) {
+		await createTable(dataStoreTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('name').varchar(128).notNull,
+				column('projectId').varchar(36).notNull,
+				column('sizeBytes').int.default(0).notNull,
+			)
+			.withForeignKey('projectId', {
+				tableName: 'project',
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			})
+			.withIndexOn(['projectId', 'name'], true).withTimestamps;
+
+		await createTable(dataStoreColumnTableName)
+			.withColumns(
+				column('id').varchar(36).primary.notNull,
+				column('name').varchar(128).notNull,
+				column('type').varchar(32).notNull,
+				column('columnIndex').int.notNull,
+				column('dataStoreId').varchar(36).notNull,
+			)
+			.withForeignKey('dataStoreId', {
+				tableName: dataStoreTableName,
+				columnName: 'id',
+				onDelete: 'CASCADE',
+			})
+			.withIndexOn(['dataStoreId', 'name'], true).withTimestamps;
+	}
+
+	async down({ schemaBuilder: { dropTable } }: MigrationContext) {
+		await dropTable(dataStoreTableName);
+		await dropTable(dataStoreColumnTableName);
+
+		// @Review: Should we also drop all created user tables here?
+	}
+}