Merge pull request #32103 from storybookjs/yann/automigration-failure-telemetry

Telemetry: Add automigration errors

Author: valentinpalkovic

code/lib/cli-storybook/src/automigrate/multi-project.ts

@@ -1,5 +1,6 @@
 import type { JsPackageManager } from 'storybook/internal/common';
 import { CLI_COLORS, type TaskLogInstance, logger, prompt } from 'storybook/internal/node-logger';
+import { sanitizeError } from 'storybook/internal/telemetry';
 import type { StorybookConfigRaw } from 'storybook/internal/types';
 
 import type { UpgradeOptions } from '../upgrade';
@@ -246,20 +247,24 @@ export async function promptForAutomigrations(
   return automigrations.filter((am) => selectedIds.includes(am.fix.id));
 }
 
+// Group automigrations by project
+type ConfigDir = string;
+type ErrorMessage = string;
+export type AutomigrationResult = {
+  automigrationStatuses: Record<FixId, FixStatus>;
+  automigrationErrors: Record<FixId, ErrorMessage>;
+};
 /** Runs selected automigrations for each project */
 export async function runAutomigrationsForProjects(
   selectedAutomigrations: AutomigrationCheckResult[],
   options: MultiProjectRunAutomigrationOptions
-): Promise<Record<string, Record<FixId, FixStatus>>> {
+): Promise<Record<ConfigDir, AutomigrationResult>> {
   const { dryRun, skipInstall, automigrations } = options;
-  const projectResults: Record<ConfigDir, Record<FixId, FixStatus>> = {};
+  const projectResults: Record<ConfigDir, AutomigrationResult> = {};
 
   const applicableAutomigrations = selectedAutomigrations.filter((am) =>
     am.reports.every((rep) => rep.status !== 'not_applicable')
   );
-
-  // Group automigrations by project
-  type ConfigDir = string;
   const projectAutomigrationResults = new Map<
     ConfigDir,
     {
@@ -327,6 +332,7 @@ export async function runAutomigrationsForProjects(
             },
           };
     const fixResults: Record<FixId, FixStatus> = {};
+    const fixFailures: Record<FixId, ErrorMessage> = {};
 
     for (const automigration of projectAutomigration) {
       const { fix, result, project, status } = automigration;
@@ -374,9 +380,12 @@ export async function runAutomigrationsForProjects(
           taskLog.message(CLI_COLORS.success(`${logger.SYMBOLS.success} ${fix.id}`));
         }
       } catch (error) {
+        const errorMessage =
+          (error instanceof Error ? error.stack : String(error)) ?? 'Unknown error';
         fixResults[fix.id] = FixStatus.FAILED;
+        fixFailures[fix.id] = sanitizeError(error as Error);
         taskLog.message(CLI_COLORS.error(`${logger.SYMBOLS.error} ${automigration.fix.id}`));
-        logger.debug(`${error instanceof Error ? error.stack : String(error)}`);
+        logger.debug(errorMessage);
       }
     }
 
@@ -391,7 +400,10 @@ export async function runAutomigrationsForProjects(
       taskLog.success(`${countPrefix}Completed automigrations for ${projectName}`);
     }
 
-    projectResults[configDir] = fixResults;
+    projectResults[configDir] = {
+      automigrationStatuses: fixResults,
+      automigrationErrors: fixFailures,
+    };
   }
 
   return projectResults;
@@ -402,7 +414,7 @@ export async function runAutomigrations(
   options: UpgradeOptions
 ): Promise<{
   detectedAutomigrations: AutomigrationCheckResult[];
-  automigrationResults: Record<string, Record<FixId, FixStatus>>;
+  automigrationResults: Record<string, AutomigrationResult>;
 }> {
   // Prepare project data for automigrations
   const projectAutomigrationData: ProjectAutomigrationData[] = projects.map((project) => ({
@@ -453,13 +465,13 @@ export async function runAutomigrations(
 
   // Special case handling for rnstorybook-config which renames the config dir
   // TODO: Remove this as soon as the rn-storybook-config automigration is removed
-  Object.entries(automigrationResults).forEach(([configDir, fixResults]) => {
-    if (fixResults[rnstorybookConfig.id] === FixStatus.SUCCEEDED) {
+  Object.entries(automigrationResults).forEach(([configDir, resultData]) => {
+    if (resultData.automigrationStatuses[rnstorybookConfig.id] === FixStatus.SUCCEEDED) {
       const project = projects.find((p) => p.configDir === configDir);
       if (project) {
         const oldConfigDir = project.configDir;
         project.configDir = project.configDir.replace('.storybook', '.rnstorybook');
-        automigrationResults[project.configDir] = fixResults;
+        automigrationResults[project.configDir] = resultData;
         delete automigrationResults[oldConfigDir];
       }
     }

code/lib/cli-storybook/src/upgrade.ts

@@ -21,8 +21,11 @@ import semver, { clean, lt } from 'semver';
 import { dedent } from 'ts-dedent';
 
 import { processAutoblockerResults } from './autoblock/utils';
-import { type AutomigrationCheckResult, runAutomigrations } from './automigrate/multi-project';
-import type { FixId } from './automigrate/types';
+import {
+  type AutomigrationCheckResult,
+  type AutomigrationResult,
+  runAutomigrations,
+} from './automigrate/multi-project';
 import { FixStatus } from './automigrate/types';
 import { displayDoctorResults, runMultiProjectDoctor } from './doctor';
 import type { ProjectDoctorData, ProjectDoctorResults } from './doctor/types';
@@ -127,30 +130,32 @@ export type UpgradeOptions = {
 };
 
 function getUpgradeResults(
-  projectResults: Record<string, Record<FixId, FixStatus>>,
+  projectResults: Record<string, AutomigrationResult>,
   doctorResults: Record<string, ProjectDoctorResults>
 ) {
   const successfulProjects: string[] = [];
   const failedProjects: string[] = [];
   const projectsWithNoFixes: string[] = [];
 
-  const allProjects = Object.entries(projectResults).map(([configDir, fixResults]) => {
-    const automigrationResults = Object.entries(fixResults).map(([fixId, status]) => {
-      const succeeded = status === FixStatus.SUCCEEDED || status === FixStatus.MANUAL_SUCCEEDED;
-      return {
-        fixId,
-        status,
-        succeeded,
-      };
-    });
+  const allProjects = Object.entries(projectResults).map(([configDir, resultData]) => {
+    const automigrationResults = Object.entries(resultData.automigrationStatuses).map(
+      ([fixId, status]) => {
+        const succeeded = status === FixStatus.SUCCEEDED || status === FixStatus.MANUAL_SUCCEEDED;
+        return {
+          fixId,
+          status,
+          succeeded,
+        };
+      }
+    );
 
     const hasFailures = automigrationResults.some(
       (fix) => fix.status === FixStatus.FAILED || fix.status === FixStatus.CHECK_FAILED
     );
     const hasSuccessfulFixes = automigrationResults.some(
       (fix) => fix.status === FixStatus.SUCCEEDED || fix.status === FixStatus.MANUAL_SUCCEEDED
     );
-    const noFixesNeeded = Object.keys(fixResults).length === 0;
+    const noFixesNeeded = Object.keys(resultData.automigrationStatuses).length === 0;
 
     // Determine if migration was successful (has successful fixes and no failures)
     const migratedSuccessfully = hasSuccessfulFixes && !hasFailures;
@@ -196,7 +201,7 @@ function getUpgradeResults(
 
 /** Logs the results of the upgrade process, including project categorization and diagnostic messages */
 function logUpgradeResults(
-  projectResults: Record<string, Record<FixId, FixStatus>>,
+  projectResults: Record<string, AutomigrationResult>,
   detectedAutomigrations: AutomigrationCheckResult[],
   doctorResults: Record<string, ProjectDoctorResults>
 ) {
@@ -242,10 +247,10 @@ function logUpgradeResults(
   const automigrationLinks = detectedAutomigrations
     .filter((am) =>
       Object.entries(projectResults).some(
-        ([_, fixResults]) =>
-          fixResults[am.fix.id] === FixStatus.FAILED ||
-          fixResults[am.fix.id] === FixStatus.SUCCEEDED ||
-          fixResults[am.fix.id] === FixStatus.CHECK_FAILED
+        ([_, resultData]) =>
+          resultData.automigrationStatuses[am.fix.id] === FixStatus.FAILED ||
+          resultData.automigrationStatuses[am.fix.id] === FixStatus.SUCCEEDED ||
+          resultData.automigrationStatuses[am.fix.id] === FixStatus.CHECK_FAILED
       )
     )
     .map((am) => `• ${createHyperlink(am.fix.id, am.fix.link!)}`);
@@ -267,7 +272,7 @@ function logUpgradeResults(
 interface MultiUpgradeTelemetryOptions {
   allProjects: CollectProjectsSuccessResult[];
   selectedProjects: CollectProjectsSuccessResult[];
-  projectResults: Record<string, Record<FixId, FixStatus>>;
+  projectResults: Record<string, AutomigrationResult>;
   doctorResults: Record<string, ProjectDoctorResults>;
   hasUserInterrupted?: boolean;
 }
@@ -339,7 +344,7 @@ export async function upgrade(options: UpgradeOptions): Promise<void> {
         );
       }
 
-      const automigrationResults: Record<string, Record<FixId, FixStatus>> = {};
+      const automigrationResults: Record<string, AutomigrationResult> = {};
       let doctorResults: Record<string, ProjectDoctorResults> = {};
 
       // Set up signal handling for interruptions
@@ -481,7 +486,10 @@ export async function upgrade(options: UpgradeOptions): Promise<void> {
         // TELEMETRY
         if (!options.disableTelemetry) {
           for (const project of storybookProjects) {
-            const fixResults = automigrationResults[project.configDir] || {};
+            const resultData = automigrationResults[project.configDir] || {
+              automigrationStatuses: {},
+              automigrationErrors: {},
+            };
             let doctorFailureCount = 0;
             let doctorErrorCount = 0;
             Object.values(doctorResults[project.configDir]?.diagnostics || {}).forEach((status) => {
@@ -493,9 +501,7 @@ export async function upgrade(options: UpgradeOptions): Promise<void> {
                 doctorErrorCount++;
               }
             });
-            const automigrationFailureCount = Object.values(fixResults).filter(
-              (status) => status === 'failed'
-            ).length;
+            const automigrationFailureCount = Object.keys(resultData.automigrationErrors).length;
             const automigrationPreCheckFailure =
               project.autoblockerCheckResults && project.autoblockerCheckResults.length > 0
                 ? project.autoblockerCheckResults
@@ -510,7 +516,8 @@ export async function upgrade(options: UpgradeOptions): Promise<void> {
             await telemetry('upgrade', {
               beforeVersion: project.beforeVersion,
               afterVersion: project.currentCLIVersion,
-              automigrationResults: fixResults,
+              automigrationResults: resultData.automigrationStatuses,
+              automigrationErrors: resultData.automigrationErrors,
               automigrationFailureCount,
               automigrationPreCheckFailure,
               doctorResults: doctorResults[project.configDir]?.diagnostics || {},