feat(@angular-devkit/build-angular): support incremental TypeScript semantic diagnostics in esbuild builder

clydin · clydin · commit d8930facc075 · 2023-05-12T11:33:51.000-04:00
When using the esbuild-based browser application builder with CLI caching enabled, TypeScript's `incremental`
option will also be enabled by default. A TypeScript build information file will be written after each build and
an attempt to load and use the file will be made during compilation setup. Caching is enabled by default within
the CLI and can be controlled via the `ng cache` command. This is the first use of persistent caching for the
esbuild-based builder. If the TypeScript `incremental` option is manually set to `false`, the build system will
not alter the value. This can be used to disable the behavior, if preferred, by setting the option to `false` in
the application's configured `tsconfig` file.
NOTE: The build information only contains information regarding the TypeScript compilation itself and does not
contain information about the Angular AOT compilation. TypeScript does not have knowledge of the AOT compiler
and it therefore cannot include that information in its build information file. Angular AOT analysis is still
performed for each build.
diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/angular/aot-compilation.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/angular/aot-compilation.ts
@@ -24,6 +24,7 @@ const { mergeTransformers, replaceBootstrap } = require('@ngtools/webpack/src/iv
 class AngularCompilationState {
   constructor(
     public readonly angularProgram: ng.NgtscProgram,
+    public readonly compilerHost: ng.CompilerHost,
     public readonly typeScriptProgram: ts.EmitAndSemanticDiagnosticsBuilderProgram,
     public readonly affectedFiles: ReadonlySet<ts.SourceFile>,
     public readonly templateDiagnosticsOptimization: ng.OptimizeFor,
@@ -67,20 +68,28 @@ export class AotCompilation extends AngularCompilation {
     const angularTypeScriptProgram = angularProgram.getTsProgram();
     ensureSourceFileVersions(angularTypeScriptProgram);
 
+    let oldProgram = this.#state?.typeScriptProgram;
+    let usingBuildInfo = false;
+    if (!oldProgram) {
+      oldProgram = ts.readBuilderProgram(compilerOptions, host);
+      usingBuildInfo = true;
+    }
+
     const typeScriptProgram = ts.createEmitAndSemanticDiagnosticsBuilderProgram(
       angularTypeScriptProgram,
       host,
-      this.#state?.typeScriptProgram,
+      oldProgram,
       configurationDiagnostics,
     );
 
     await profileAsync('NG_ANALYZE_PROGRAM', () => angularCompiler.analyzeAsync());
     const affectedFiles = profileSync('NG_FIND_AFFECTED', () =>
-      findAffectedFiles(typeScriptProgram, angularCompiler),
+      findAffectedFiles(typeScriptProgram, angularCompiler, usingBuildInfo),
     );
 
     this.#state = new AngularCompilationState(
       angularProgram,
+      host,
       typeScriptProgram,
       affectedFiles,
       affectedFiles.size === 1 ? OptimizeFor.SingleFile : OptimizeFor.WholeProgram,
@@ -151,14 +160,16 @@ export class AotCompilation extends AngularCompilation {
 
   emitAffectedFiles(): Iterable<EmitFileResult> {
     assert(this.#state, 'Angular compilation must be initialized prior to emitting files.');
-    const { angularCompiler, typeScriptProgram } = this.#state;
+    const { angularCompiler, compilerHost, typeScriptProgram } = this.#state;
     const buildInfoFilename =
       typeScriptProgram.getCompilerOptions().tsBuildInfoFile ?? '.tsbuildinfo';
 
     const emittedFiles = new Map<ts.SourceFile, EmitFileResult>();
     const writeFileCallback: ts.WriteFileCallback = (filename, contents, _a, _b, sourceFiles) => {
-      if (sourceFiles?.length === 0 && filename.endsWith(buildInfoFilename)) {
-        // TODO: Store incremental build info
+      if (!sourceFiles?.length && filename.endsWith(buildInfoFilename)) {
+        // Save builder info contents to specified location
+        compilerHost.writeFile(filename, contents, false);
+
         return;
       }
 
@@ -168,6 +179,7 @@ export class AotCompilation extends AngularCompilation {
         return;
       }
 
+      angularCompiler.incrementalCompilation.recordSuccessfulEmit(sourceFile);
       emittedFiles.set(sourceFile, { filename: sourceFile.fileName, contents });
     };
     const transformers = mergeTransformers(angularCompiler.prepareEmit().transformers, {
@@ -187,6 +199,10 @@ export class AotCompilation extends AngularCompilation {
         continue;
       }
 
+      if (sourceFile.isDeclarationFile) {
+        continue;
+      }
+
       if (angularCompiler.incrementalCompilation.safeToSkipEmit(sourceFile)) {
         continue;
       }
@@ -200,7 +216,8 @@ export class AotCompilation extends AngularCompilation {
 
 function findAffectedFiles(
   builder: ts.EmitAndSemanticDiagnosticsBuilderProgram,
-  { ignoreForDiagnostics, ignoreForEmit, incrementalCompilation }: ng.NgtscProgram['compiler'],
+  { ignoreForDiagnostics }: ng.NgtscProgram['compiler'],
+  includeTTC: boolean,
 ): Set<ts.SourceFile> {
   const affectedFiles = new Set<ts.SourceFile>();
 
@@ -235,13 +252,22 @@ function findAffectedFiles(
     affectedFiles.add(result.affected as ts.SourceFile);
   }
 
-  // A file is also affected if the Angular compiler requires it to be emitted
-  for (const sourceFile of builder.getSourceFiles()) {
-    if (ignoreForEmit.has(sourceFile) || incrementalCompilation.safeToSkipEmit(sourceFile)) {
-      continue;
+  // Add all files with associated template type checking files.
+  // Stored TS build info does not have knowledge of the AOT compiler or the typechecking state of the templates.
+  // To ensure that errors are reported correctly, all AOT component diagnostics need to be analyzed even if build
+  // info is present.
+  if (includeTTC) {
+    for (const sourceFile of builder.getSourceFiles()) {
+      if (ignoreForDiagnostics.has(sourceFile) && sourceFile.fileName.endsWith('.ngtypecheck.ts')) {
+        // This file name conversion relies on internal compiler logic and should be converted
+        // to an official method when available. 15 is length of `.ngtypecheck.ts`
+        const originalFilename = sourceFile.fileName.slice(0, -15) + '.ts';
+        const originalSourceFile = builder.getSourceFile(originalFilename);
+        if (originalSourceFile) {
+          affectedFiles.add(originalSourceFile);
+        }
+      }
     }
-
-    affectedFiles.add(sourceFile);
   }
 
   return affectedFiles;
diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/angular/compiler-plugin.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/angular/compiler-plugin.ts
@@ -45,6 +45,10 @@ export class SourceFileCache extends Map<string, ts.SourceFile> {
   readonly typeScriptFileCache = new Map<string, string | Uint8Array>();
   readonly loadResultCache = new MemoryLoadResultCache();
 
+  constructor(readonly persistentCachePath?: string) {
+    super();
+  }
+
   invalidate(files: Iterable<string>): void {
     this.modifiedFiles.clear();
     for (let file of files) {
@@ -208,6 +212,18 @@ export function createCompilerPlugin(
             });
           }
 
+          // Enable incremental compilation by default if caching is enabled
+          if (pluginOptions.sourceFileCache?.persistentCachePath) {
+            compilerOptions.incremental ??= true;
+            // Set the build info file location to the configured cache directory
+            compilerOptions.tsBuildInfoFile = path.join(
+              pluginOptions.sourceFileCache?.persistentCachePath,
+              '.tsbuildinfo',
+            );
+          } else {
+            compilerOptions.incremental = false;
+          }
+
           return {
             ...compilerOptions,
             noEmitOnError: false,
@@ -232,70 +248,62 @@ export function createCompilerPlugin(
         });
 
         // Update TypeScript file output cache for all affected files
-        for (const { filename, contents } of compilation.emitAffectedFiles()) {
-          typeScriptFileCache.set(pathToFileURL(filename).href, contents);
-        }
+        profileSync('NG_EMIT_TS', () => {
+          for (const { filename, contents } of compilation.emitAffectedFiles()) {
+            typeScriptFileCache.set(pathToFileURL(filename).href, contents);
+          }
+        });
 
         // Reset the setup warnings so that they are only shown during the first build.
         setupWarnings = undefined;
 
         return result;
       });
 
-      build.onLoad({ filter: /\.[cm]?[jt]sx?$/ }, (args) =>
-        profileAsync(
-          'NG_EMIT_TS*',
-          async () => {
-            const request = pluginOptions.fileReplacements?.[args.path] ?? args.path;
+      build.onLoad({ filter: /\.[cm]?[jt]sx?$/ }, async (args) => {
+        const request = pluginOptions.fileReplacements?.[args.path] ?? args.path;
 
-            // Skip TS load attempt if JS TypeScript compilation not enabled and file is JS
-            if (shouldTsIgnoreJs && /\.[cm]?js$/.test(request)) {
-              return undefined;
-            }
+        // Skip TS load attempt if JS TypeScript compilation not enabled and file is JS
+        if (shouldTsIgnoreJs && /\.[cm]?js$/.test(request)) {
+          return undefined;
+        }
 
-            // The filename is currently used as a cache key. Since the cache is memory only,
-            // the options cannot change and do not need to be represented in the key. If the
-            // cache is later stored to disk, then the options that affect transform output
-            // would need to be added to the key as well as a check for any change of content.
-            let contents = typeScriptFileCache.get(pathToFileURL(request).href);
+        // The filename is currently used as a cache key. Since the cache is memory only,
+        // the options cannot change and do not need to be represented in the key. If the
+        // cache is later stored to disk, then the options that affect transform output
+        // would need to be added to the key as well as a check for any change of content.
+        let contents = typeScriptFileCache.get(pathToFileURL(request).href);
+
+        if (contents === undefined) {
+          // No TS result indicates the file is not part of the TypeScript program.
+          // If allowJs is enabled and the file is JS then defer to the next load hook.
+          if (!shouldTsIgnoreJs && /\.[cm]?js$/.test(request)) {
+            return undefined;
+          }
 
-            if (contents === undefined) {
-              // No TS result indicates the file is not part of the TypeScript program.
-              // If allowJs is enabled and the file is JS then defer to the next load hook.
-              if (!shouldTsIgnoreJs && /\.[cm]?js$/.test(request)) {
-                return undefined;
-              }
-
-              // Otherwise return an error
-              return {
-                errors: [
-                  createMissingFileError(
-                    request,
-                    args.path,
-                    build.initialOptions.absWorkingDir ?? '',
-                  ),
-                ],
-              };
-            } else if (typeof contents === 'string') {
-              // A string indicates untransformed output from the TS/NG compiler
-              contents = await javascriptTransformer.transformData(
-                request,
-                contents,
-                true /* skipLinker */,
-              );
-
-              // Store as the returned Uint8Array to allow caching the fully transformed code
-              typeScriptFileCache.set(pathToFileURL(request).href, contents);
-            }
+          // Otherwise return an error
+          return {
+            errors: [
+              createMissingFileError(request, args.path, build.initialOptions.absWorkingDir ?? ''),
+            ],
+          };
+        } else if (typeof contents === 'string') {
+          // A string indicates untransformed output from the TS/NG compiler
+          contents = await javascriptTransformer.transformData(
+            request,
+            contents,
+            true /* skipLinker */,
+          );
+
+          // Store as the returned Uint8Array to allow caching the fully transformed code
+          typeScriptFileCache.set(pathToFileURL(request).href, contents);
+        }
 
-            return {
-              contents,
-              loader: 'js',
-            };
-          },
-          true,
-        ),
-      );
+        return {
+          contents,
+          loader: 'js',
+        };
+      });
 
       build.onLoad({ filter: /\.[cm]?js$/ }, (args) =>
         profileAsync(
diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/angular/jit-compilation.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/angular/jit-compilation.ts
@@ -16,6 +16,7 @@ import { createJitResourceTransformer } from './jit-resource-transformer';
 
 class JitCompilationState {
   constructor(
+    public readonly compilerHost: ng.CompilerHost,
     public readonly typeScriptProgram: ts.EmitAndSemanticDiagnosticsBuilderProgram,
     public readonly constructorParametersDownlevelTransform: ts.TransformerFactory<ts.SourceFile>,
     public readonly replaceResourcesTransform: ts.TransformerFactory<ts.SourceFile>,
@@ -51,7 +52,7 @@ export class JitCompilation extends AngularCompilation {
         rootNames,
         compilerOptions,
         host,
-        this.#state?.typeScriptProgram,
+        this.#state?.typeScriptProgram ?? ts.readBuilderProgram(compilerOptions, host),
         configurationDiagnostics,
       ),
     );
@@ -61,6 +62,7 @@ export class JitCompilation extends AngularCompilation {
     );
 
     this.#state = new JitCompilationState(
+      host,
       typeScriptProgram,
       constructorParametersDownlevelTransform(typeScriptProgram.getProgram()),
       createJitResourceTransformer(() => typeScriptProgram.getProgram().getTypeChecker()),
@@ -86,6 +88,7 @@ export class JitCompilation extends AngularCompilation {
   emitAffectedFiles(): Iterable<EmitFileResult> {
     assert(this.#state, 'Compilation must be initialized prior to emitting files.');
     const {
+      compilerHost,
       typeScriptProgram,
       constructorParametersDownlevelTransform,
       replaceResourcesTransform,
@@ -95,8 +98,10 @@ export class JitCompilation extends AngularCompilation {
 
     const emittedFiles: EmitFileResult[] = [];
     const writeFileCallback: ts.WriteFileCallback = (filename, contents, _a, _b, sourceFiles) => {
-      if (sourceFiles?.length === 0 && filename.endsWith(buildInfoFilename)) {
-        // TODO: Store incremental build info
+      if (!sourceFiles?.length && filename.endsWith(buildInfoFilename)) {
+        // Save builder info contents to specified location
+        compilerHost.writeFile(filename, contents, false);
+
         return;
       }
 
diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/index.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/index.ts
@@ -98,16 +98,17 @@ async function execute(
     assets,
     serviceWorkerOptions,
     indexHtmlOptions,
+    cacheOptions,
   } = options;
 
   const browsers = getSupportedBrowsers(projectRoot, context.logger);
   const target = transformSupportedBrowsersToTargets(browsers);
 
   // Reuse rebuild state or create new bundle contexts for code and global stylesheets
   let bundlerContexts = rebuildState?.rebuildContexts;
-  const codeBundleCache = options.watch
-    ? rebuildState?.codeBundleCache ?? new SourceFileCache()
-    : undefined;
+  const codeBundleCache =
+    rebuildState?.codeBundleCache ??
+    new SourceFileCache(cacheOptions.enabled ? cacheOptions.path : undefined);
   if (bundlerContexts === undefined) {
     bundlerContexts = [];
 
diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/options.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/options.ts
@@ -7,7 +7,6 @@
  */
 
 import { BuilderContext } from '@angular-devkit/architect';
-import fs from 'node:fs';
 import { createRequire } from 'node:module';
 import path from 'node:path';
 import { normalizeAssetPatterns, normalizeOptimization, normalizeSourceMaps } from '../../utils';
@@ -64,7 +63,9 @@ export async function normalizeOptions(
     path.join(workspaceRoot, (projectMetadata.sourceRoot as string | undefined) ?? 'src'),
   );
 
+  // Gather persistent caching option and provide a project specific cache location
   const cacheOptions = normalizeCacheOptions(projectMetadata, workspaceRoot);
+  cacheOptions.path = path.join(cacheOptions.path, projectName);
 
   const entryPoints = normalizeEntryPoints(workspaceRoot, options.main, options.entryPoints);
   const tsconfig = path.join(workspaceRoot, options.tsConfig);
