[PM-14360] Import Batching

jslib/angular/src/services/jslib-services.module.ts

@@ -26,6 +26,12 @@ import { StateService } from "@/jslib/common/src/services/state.service";
 import { StateMigrationService } from "@/jslib/common/src/services/stateMigration.service";
 import { TokenService } from "@/jslib/common/src/services/token.service";
 
+import { DirectoryFactoryService } from "@/src/abstractions/directory-factory.service";
+import { StateService as StateServiceExtended } from "@/src/abstractions/state.service";
+import { DefaultBatchRequestBuilder } from "@/src/services/default-batch-request-builder";
+import { DefaultSingleRequestBuilder } from "@/src/services/default-single-request-builder";
+import { DefaultDirectoryFactoryService } from "@/src/services/directory-factory.service";
+
 import {
   SafeInjectionToken,
   SECURE_STORAGE,
@@ -138,6 +144,19 @@ import { ValidationService } from "./validation.service";
         ),
       deps: [StorageServiceAbstraction, SECURE_STORAGE],
     }),
+    safeProvider({
+      provide: DefaultBatchRequestBuilder,
+      useAngularDecorators: true,
+    }),
+    safeProvider({
+      provide: DefaultSingleRequestBuilder,
+      useAngularDecorators: true,
+    }),
+    safeProvider({
+      provide: DirectoryFactoryService,
+      useClass: DefaultDirectoryFactoryService,
+      deps: [LogService, I18nServiceAbstraction, StateServiceExtended],
+    }),
   ] satisfies SafeProvider[],
 })
 export class JslibServicesModule {}

jslib/common/src/abstractions/api.service.ts

@@ -2,9 +2,9 @@ import { ApiTokenRequest } from "../models/request/identityToken/apiTokenRequest
 import { PasswordTokenRequest } from "../models/request/identityToken/passwordTokenRequest";
 import { SsoTokenRequest } from "../models/request/identityToken/ssoTokenRequest";
 import { OrganizationImportRequest } from "../models/request/organizationImportRequest";
-import { IdentityCaptchaResponse } from '../models/response/identityCaptchaResponse';
-import { IdentityTokenResponse } from '../models/response/identityTokenResponse';
-import { IdentityTwoFactorResponse } from '../models/response/identityTwoFactorResponse';
+import { IdentityCaptchaResponse } from "../models/response/identityCaptchaResponse";
+import { IdentityTokenResponse } from "../models/response/identityTokenResponse";
+import { IdentityTwoFactorResponse } from "../models/response/identityTwoFactorResponse";
 
 export abstract class ApiService {
   postIdentityToken: (

src/abstractions/directory-factory.service.ts

@@ -0,0 +1,6 @@
+import { DirectoryType } from "@/src/enums/directoryType";
+import { IDirectoryService } from "@/src/services/directory.service";
+
+export abstract class DirectoryFactoryService {
+  abstract createService(type: DirectoryType): IDirectoryService;
+}

src/abstractions/request-builder.service.ts

@@ -0,0 +1,13 @@
+import { OrganizationImportRequest } from "@/jslib/common/src/models/request/organizationImportRequest";
+
+import { GroupEntry } from "@/src/models/groupEntry";
+import { UserEntry } from "@/src/models/userEntry";
+
+export abstract class RequestBuilderAbstratction {
+  buildRequest: (
+    groups: GroupEntry[],
+    users: UserEntry[],
+    removeDisabled: boolean,
+    overwriteExisting: boolean,
+  ) => OrganizationImportRequest[];
+}

src/app/services/services.module.ts

@@ -25,6 +25,10 @@
 import { NodeApiService } from "@/jslib/node/src/services/nodeApi.service";
 import { NodeCryptoFunctionService } from "@/jslib/node/src/services/nodeCryptoFunction.service";
 
+import { DirectoryFactoryService } from "@/src/abstractions/directory-factory.service";
+import { DefaultBatchRequestBuilder } from "@/src/services/default-batch-request-builder";
+import { DefaultSingleRequestBuilder } from "@/src/services/default-single-request-builder";
+
 import { AuthService as AuthServiceAbstraction } from "../../abstractions/auth.service";
 import { StateService as StateServiceAbstraction } from "../../abstractions/state.service";
 import { Account } from "../../models/account";
@@ -168,13 +172,15 @@
       provide: SyncService,
       useClass: SyncService,
       deps: [
-        LogServiceAbstraction,
         CryptoFunctionServiceAbstraction,
         ApiServiceAbstraction,
         MessagingServiceAbstraction,
         I18nServiceAbstraction,
         EnvironmentServiceAbstraction,
         StateServiceAbstraction,
+        DefaultBatchRequestBuilder,
+        DefaultSingleRequestBuilder,
+        DirectoryFactoryService,
       ],
     }),
     safeProvider(AuthGuardService),

src/bwdc.ts

@@ -17,9 +17,13 @@
 import { NodeApiService } from "@/jslib/node/src/services/nodeApi.service";
 import { NodeCryptoFunctionService } from "@/jslib/node/src/services/nodeCryptoFunction.service";
 
+import { DirectoryFactoryService } from "./abstractions/directory-factory.service";
 import { Account } from "./models/account";
 import { Program } from "./program";
 import { AuthService } from "./services/auth.service";
+import { DefaultBatchRequestBuilder } from "./services/default-batch-request-builder";
+import { DefaultSingleRequestBuilder } from "./services/default-single-request-builder";
+import { DefaultDirectoryFactoryService } from "./services/directory-factory.service";
 import { I18nService } from "./services/i18n.service";
 import { KeytarSecureStorageService } from "./services/keytarSecureStorage.service";
 import { LowdbStorageService } from "./services/lowdbStorage.service";
@@ -51,6 +55,9 @@
   syncService: SyncService;
   stateService: StateService;
   stateMigrationService: StateMigrationService;
+  directoryFactoryService: DirectoryFactoryService;
+  batchRequestBuilder: DefaultBatchRequestBuilder;
+  singleRequestBuilder: DefaultSingleRequestBuilder;
 
   constructor() {
     const applicationName = "Bitwarden Directory Connector";
@@ -146,14 +153,25 @@
       this.stateService,
     );
 
-    this.syncService = new SyncService(
+    this.directoryFactoryService = new DefaultDirectoryFactoryService(
       this.logService,
+      this.i18nService,
+      this.stateService,
+    );
+
+    this.batchRequestBuilder = new DefaultBatchRequestBuilder();
+    this.singleRequestBuilder = new DefaultSingleRequestBuilder();
+
+    this.syncService = new SyncService(
       this.cryptoFunctionService,
       this.apiService,
       this.messagingService,
       this.i18nService,
       this.environmentService,
       this.stateService,
+      this.batchRequestBuilder,
+      this.singleRequestBuilder,
+      this.directoryFactoryService,
     );
 
     this.program = new Program(this);

src/services/default-batch-request-builder.ts

@@ -0,0 +1,70 @@
+import { OrganizationImportRequest } from "@/jslib/common/src/models/request/organizationImportRequest";
+
+import { GroupEntry } from "@/src/models/groupEntry";
+import { UserEntry } from "@/src/models/userEntry";
+
+import { RequestBuilderAbstratction } from "../abstractions/request-builder.service";
+
+import { batchSize } from "./sync.service";
+
+/**
+ * This class is responsible for batching large sync requests (>=2k users) into multiple smaller
+ * requests to the /import REST endpoint. This is done to ensure we are under the default
+ * maximum packet size for NGINX web servers to avoid the request potentially timing out
+ * */
+export class DefaultBatchRequestBuilder implements RequestBuilderAbstratction {
+  buildRequest(
+    groups: GroupEntry[],
+    users: UserEntry[],
+    removeDisabled: boolean,
+    overwriteExisting: boolean,
+  ): OrganizationImportRequest[] {
+    const requests: OrganizationImportRequest[] = [];
+
+    if (users.length > 0) {
+      const usersRequest = users.map((u) => {
+        return {
+          email: u.email,
+          externalId: u.externalId,
+          deleted: u.deleted || (removeDisabled && u.disabled),
+        };
+      });
+
+      // Partition users
+      for (let i = 0; i < usersRequest.length; i += batchSize) {
+        const u = usersRequest.slice(i, i + batchSize);
+        const req = new OrganizationImportRequest({
+          groups: [],
+          users: u,
+          largeImport: true,
+          overwriteExisting,
+        });
+        requests.push(req);
+      }
+    }
+
+    if (groups.length > 0) {
+      const groupRequest = groups.map((g) => {
+        return {
+          name: g.name,
+          externalId: g.externalId,
+          memberExternalIds: Array.from(g.userMemberExternalIds),
+        };
+      });
+
+      // Partition groups
+      for (let i = 0; i < groupRequest.length; i += batchSize) {
+        const g = groupRequest.slice(i, i + batchSize);
+        const req = new OrganizationImportRequest({
+          groups: g,
+          users: [],
+          largeImport: true,
+          overwriteExisting,
+        });
+        requests.push(req);
+      }
+    }
+
+    return requests;
+  }
+}

src/services/default-batch-requests-builder.spec.ts

@@ -0,0 +1,47 @@
+import { GroupEntry } from "@/src/models/groupEntry";
+import { UserEntry } from "@/src/models/userEntry";
+
+import { DefaultBatchRequestBuilder } from "./default-batch-request-builder";
+import { DefaultSingleRequestBuilder } from "./default-single-request-builder";
+
+describe("BatchingService", () => {
+  let batchRequestBuilder: DefaultBatchRequestBuilder;
+  let singleRequestBuilder: DefaultSingleRequestBuilder;
+
+  function userSimulator(userCount: number) {
+    return Array(userCount).fill(new UserEntry());
+  }
+
+  function groupSimulator(groupCount: number) {
+    return Array(groupCount).fill(new GroupEntry());
+  }
+
+  beforeEach(async () => {
+    batchRequestBuilder = new DefaultBatchRequestBuilder();
+    singleRequestBuilder = new DefaultSingleRequestBuilder();
+  });
+
+  it("BatchRequestBuilder batches requests for > 2000 users", () => {
+    const mockGroups = groupSimulator(11000);
+    const mockUsers = userSimulator(11000);
+
+    const requests = batchRequestBuilder.buildRequest(mockGroups, mockUsers, true, true);
+
+    expect(requests.length).toEqual(12);
+  });
+
+  it("SingleRequestBuilder returns single request for 200 users", () => {
+    const mockGroups = groupSimulator(200);
+    const mockUsers = userSimulator(200);
+
+    const requests = singleRequestBuilder.buildRequest(mockGroups, mockUsers, true, true);
+
+    expect(requests.length).toEqual(1);
+  });
+
+  it("BatchRequestBuilder retuns an empty array when there are no users or groups", () => {
+    const requests = batchRequestBuilder.buildRequest([], [], true, true);
+
+    expect(requests).toEqual([]);
+  });
+});

src/services/default-single-request-builder.ts

@@ -0,0 +1,42 @@
+import { OrganizationImportRequest } from "@/jslib/common/src/models/request/organizationImportRequest";
+
+import { GroupEntry } from "@/src/models/groupEntry";
+import { UserEntry } from "@/src/models/userEntry";
+
+import { RequestBuilderAbstratction } from "../abstractions/request-builder.service";
+
+/**
+ * This class is responsible for building small (<2k users) syncs as a single
+ * request to the /import REST endpoint. This is done to be backwards compatiblle with
+ * existing functionality for sync requests that are sufficiently small enough to not
+ * exceed default maximum packet size limits on NGINX web servers.
+ * */
+export class DefaultSingleRequestBuilder implements RequestBuilderAbstratction {
+  buildRequest(
+    groups: GroupEntry[],
+    users: UserEntry[],
+    removeDisabled: boolean,
+    overwriteExisting: boolean,
+  ): OrganizationImportRequest[] {
+    return [
+      new OrganizationImportRequest({
+        groups: (groups ?? []).map((g) => {
+          return {
+            name: g.name,
+            externalId: g.externalId,
+            memberExternalIds: Array.from(g.userMemberExternalIds),
+          };
+        }),
+        users: (users ?? []).map((u) => {
+          return {
+            email: u.email,
+            externalId: u.externalId,
+            deleted: u.deleted || (removeDisabled && u.disabled),
+          };
+        }),
+        overwriteExisting: overwriteExisting,
+        largeImport: false,
+      }),
+    ];
+  }
+}

src/services/directory-factory.service.ts

@@ -0,0 +1,37 @@
+import { I18nService } from "@/jslib/common/src/abstractions/i18n.service";
+import { LogService } from "@/jslib/common/src/abstractions/log.service";
+
+import { DirectoryFactoryService } from "../abstractions/directory-factory.service";
+import { StateService } from "../abstractions/state.service";
+import { DirectoryType } from "../enums/directoryType";
+
+import { AzureDirectoryService } from "./azure-directory.service";
+import { GSuiteDirectoryService } from "./gsuite-directory.service";
+import { LdapDirectoryService } from "./ldap-directory.service";
+import { OktaDirectoryService } from "./okta-directory.service";
+import { OneLoginDirectoryService } from "./onelogin-directory.service";
+
+export class DefaultDirectoryFactoryService implements DirectoryFactoryService {
+  constructor(
+    private logService: LogService,
+    private i18nService: I18nService,
+    private stateService: StateService,
+  ) {}
+
+  createService(directoryType: DirectoryType) {
+    switch (directoryType) {
+      case DirectoryType.GSuite:
+        return new GSuiteDirectoryService(this.logService, this.i18nService, this.stateService);
+      case DirectoryType.AzureActiveDirectory:
+        return new AzureDirectoryService(this.logService, this.i18nService, this.stateService);
+      case DirectoryType.Ldap:
+        return new LdapDirectoryService(this.logService, this.i18nService, this.stateService);
+      case DirectoryType.Okta:
+        return new OktaDirectoryService(this.logService, this.i18nService, this.stateService);
+      case DirectoryType.OneLogin:
+        return new OneLoginDirectoryService(this.logService, this.i18nService, this.stateService);
+      default:
+        throw new Error("Invalid Directory Type");
+    }
+  }
+}