Skip to content

Commit d8b205a

Browse files
authored
test(a11y): e2e baseline tests for a11y screen reader descriptions (#2684)
* feat(a11y): add accessibility e2e test suite with baseline tests for all chart types * fix(a11y): update e2e test expectations to match current accessibility output format * feat(e2e): add accessibility test suites for all chart types * test(a11y): fix accessibility test assertions to include chart types * test(a11y): add comprehensive chart type coverage to accessibility tests * test(a11y): update accessibility test expectations to match current output format * refactor(e2e): remove unused accessibility helper file * refactor(e2e): remove unused a11y helper and add edge case test comment * refactor(e2e): consolidate accessibility test helper and unify test patterns * refactor(e2e): consolidate start scripts with --a11y flag * refactor(e2e): consolidate test scripts by adding --a11y flag to test.sh * feat(ci): add accessibility e2e tests to Buildkite pipeline * feat(ci): add accessibility e2e tests to build pipeline * refactor(e2e): move a11y tests to tests_a11y directory and update config * fix(e2e): update a11y test imports and config for tests_a11y directory * fix(ci): use correct parallel key for a11y test job status reporting * feat(ci): add A11Y report deployment and PR comment integration * refactor(ci): improve code formatting in e2e reports script * set parallelism to 1 for a11y e2e tests * skip flame chart a11y tests * more distinct naming related to VRT/A11Y e2e tests * add vrt postfix to playwright references * add more vrt references * fix e2e->vrt * fix firebase deploy * fix: review feedback
1 parent fade989 commit d8b205a

39 files changed

+1220
-44
lines changed

.buildkite/pipelines/pull_request/pipeline.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ import {
1818
e2eServerStep,
1919
eslintStep,
2020
jestStep,
21-
playwrightStep,
21+
playwrightVrtStep,
22+
playwrightA11yStep,
2223
prettierStep,
2324
docsStep,
2425
storybookStep,
@@ -60,7 +61,8 @@ void (async () => {
6061
e2eServerStep(),
6162
firebasePreDeployStep(),
6263
ghpDeployStep(),
63-
playwrightStep(),
64+
playwrightVrtStep(),
65+
playwrightA11yStep(),
6466
firebaseDeployStep(),
6567
].map((step) => step(changeCtx));
6668

.buildkite/scripts/pre_exit.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import { yarnInstall } from './../utils/exec';
1010
import { bkEnv, buildkiteGQLQuery, codeCheckIsCompleted, getJobMetadata, updateCheckStatus } from '../utils';
1111

12-
const skipChecks = new Set(['playwright']);
12+
const skipChecks = new Set(['playwright_vrt', 'playwright_a11y']);
1313

1414
void (async function () {
1515
const { checkId, jobId, jobUrl } = bkEnv;

.buildkite/scripts/steps/e2e_reports.ts

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ async function setGroupStatus() {
3838
return;
3939
}
4040

41-
const e2eJobs = await getBuildJobs('playwright__parallel-step');
41+
const parallelKey =
42+
checkId === 'playwright_a11y' ? 'playwright_a11y__parallel-step' : 'playwright_vrt__parallel-step';
43+
const e2eJobs = await getBuildJobs(parallelKey);
4244
const jobStateMap = new Map<string, number>();
4345
jobStateMap.set('Success', 0);
4446
jobStateMap.set('Failed', 0);
@@ -174,9 +176,17 @@ void (async () => {
174176

175177
await setGroupStatus();
176178

177-
await downloadArtifacts('.buildkite/artifacts/e2e_reports/*');
179+
const { checkId } = bkEnv;
180+
const isA11y = checkId === 'playwright_a11y';
181+
const artifactPath = isA11y ? '.buildkite/artifacts/a11y_reports/*' : '.buildkite/artifacts/vrt_reports/*';
182+
const reportDir = isA11y ? '.buildkite/artifacts/a11y_reports' : '.buildkite/artifacts/vrt_reports';
183+
const outputDir = isA11y ? 'merged_a11y_html_report' : 'merged_vrt_html_report';
184+
const outputArtifact = isA11y
185+
? '.buildkite/artifacts/merged_a11y_html_report.gz'
186+
: '.buildkite/artifacts/merged_vrt_html_report.gz';
187+
188+
await downloadArtifacts(artifactPath);
178189

179-
const reportDir = '.buildkite/artifacts/e2e_reports';
180190
const files = fs.readdirSync(reportDir);
181191
await Promise.all<void>(
182192
files
@@ -189,18 +199,18 @@ void (async () => {
189199
),
190200
);
191201

192-
startGroup('Merging e2e reports');
202+
startGroup(`Merging ${isA11y ? 'a11y' : 'e2e'} reports`);
193203

194204
await exec('yarn merge:reports', {
195205
cwd: 'e2e',
196206
env: {
197-
HTML_REPORT_DIR: 'merged_html_report',
207+
HTML_REPORT_DIR: outputDir,
198208
},
199209
});
200210

201211
await compress({
202-
src: 'e2e/merged_html_report',
203-
dest: '.buildkite/artifacts/merged_html_report.gz',
212+
src: path.join('e2e', outputDir),
213+
dest: outputArtifact,
204214
});
205215

206216
if (bkEnv.steps.playwright.updateScreenshots) {

.buildkite/scripts/steps/firebase_deploy.ts

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,24 +40,33 @@ void (async () => {
4040
dest: path.join(outDir, 'e2e'),
4141
});
4242

43-
const e2eReportSrc = '.buildkite/artifacts/merged_html_report.gz';
44-
await downloadArtifacts(e2eReportSrc, 'playwright_merge_and_status');
43+
const vrtReportSrc = '.buildkite/artifacts/merged_vrt_html_report.gz';
44+
await downloadArtifacts(vrtReportSrc, 'playwright_vrt_merge_and_status');
4545
await decompress({
46-
src: e2eReportSrc,
47-
dest: path.join(outDir, 'e2e-report'),
46+
src: vrtReportSrc,
47+
dest: path.join(outDir, 'vrt-report'),
48+
});
49+
50+
const a11yReportSrc = '.buildkite/artifacts/merged_a11y_html_report.gz';
51+
await downloadArtifacts(a11yReportSrc, 'playwright_a11y_merge_and_status');
52+
await decompress({
53+
src: a11yReportSrc,
54+
dest: path.join(outDir, 'a11y-report'),
4855
});
4956

5057
startGroup('Check deployment files');
5158

5259
const hasDocsIndex = fs.existsSync(path.join(outDir, 'index.html'));
5360
const hasStorybookIndex = fs.existsSync(path.join(outDir, 'storybook/index.html'));
5461
const hasE2EIndex = fs.existsSync(path.join(outDir, 'e2e/index.html'));
55-
const hasE2EReportIndex = fs.existsSync(path.join(outDir, 'e2e-report/index.html'));
62+
const hasVrtReportIndex = fs.existsSync(path.join(outDir, 'vrt-report/index.html'));
63+
const hasA11yReportIndex = fs.existsSync(path.join(outDir, 'a11y-report/index.html'));
5664
const missingFiles = [
5765
['docs', hasDocsIndex],
5866
['storybook', hasStorybookIndex],
5967
['e2e server', hasE2EIndex],
60-
['e2e report', hasE2EReportIndex],
68+
['vrt report', hasVrtReportIndex],
69+
['a11y report', hasA11yReportIndex],
6170
]
6271
.filter(([, exists]) => !exists)
6372
.map<string>(([f]) => f as string);
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
import path from 'path';
10+
11+
import { getMetadata, setMetadata } from 'buildkite-agent-node';
12+
13+
import { updateCheckStatus } from './../../utils/github';
14+
import { exec, downloadArtifacts, startGroup, yarnInstall, getNumber, decompress, compress, bkEnv } from '../../utils';
15+
import { ENV_URL } from '../../utils/constants';
16+
17+
const jobIndex = getNumber(process.env.BUILDKITE_PARALLEL_JOB);
18+
const shardIndex = jobIndex ? jobIndex + 1 : 1;
19+
const jobTotal = getNumber(process.env.BUILDKITE_PARALLEL_JOB_COUNT);
20+
21+
const pwFlags = ['--project=Chrome', '--config=playwright.a11y.config.ts'];
22+
23+
if (jobIndex !== null && jobTotal !== null) {
24+
pwFlags.push(`--shard=${shardIndex}/${jobTotal}`);
25+
}
26+
27+
void (async () => {
28+
await yarnInstall('e2e');
29+
30+
const key = `${bkEnv.checkId}--activeJobs`;
31+
const value = shardIndex === jobTotal ? jobTotal - 1 : Number(await getMetadata(key));
32+
// TODO improve this status logic, not easy to communicate state of parallel steps
33+
const activeJobs = Math.min((Number.isNaN(value) ? 0 : value) + 1, jobTotal ?? 1);
34+
await setMetadata(key, String(activeJobs));
35+
36+
await updateCheckStatus(
37+
{
38+
status: 'in_progress',
39+
},
40+
'playwright_a11y',
41+
`${activeJobs} of ${jobTotal ?? 1} a11y jobs started`,
42+
);
43+
44+
const src = '.buildkite/artifacts/e2e_server.gz';
45+
await downloadArtifacts(src, 'build_e2e');
46+
await decompress({
47+
src,
48+
dest: 'e2e/server',
49+
});
50+
51+
startGroup('Check Architecture');
52+
await exec('arch');
53+
54+
startGroup('Running e2e a11y playwright job');
55+
const reportDir = `reports/a11y_report_${shardIndex}`;
56+
async function postCommandTasks() {
57+
await compress({
58+
src: path.join('e2e', reportDir),
59+
dest: `.buildkite/artifacts/a11y_reports/report_${shardIndex}.gz`,
60+
});
61+
}
62+
63+
const command = `yarn playwright test ${pwFlags.join(' ')}`;
64+
65+
try {
66+
await exec(command, {
67+
cwd: 'e2e',
68+
env: {
69+
[ENV_URL]: 'http://127.0.0.1:9002',
70+
PLAYWRIGHT_HTML_REPORT: reportDir,
71+
PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/a11y-json/report_${shardIndex}.json`,
72+
},
73+
});
74+
await postCommandTasks();
75+
} catch (error) {
76+
await postCommandTasks();
77+
throw error;
78+
}
79+
})();

.buildkite/scripts/steps/playwright.ts renamed to .buildkite/scripts/steps/playwright_vrt.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ void (async () => {
8080
{
8181
status: 'in_progress',
8282
},
83-
'playwright',
83+
'playwright_vrt',
8484
`${activeJobs} of ${jobTotal ?? 1} jobs started`,
8585
);
8686

@@ -98,12 +98,12 @@ void (async () => {
9898
// TODO Fix this duplicate script that allows us to skip root node install on all e2e test runners
9999
await exec('node ./e2e/scripts/extract_examples.js');
100100

101-
startGroup('Running e2e playwright job');
102-
const reportDir = `reports/report_${shardIndex}`;
101+
startGroup('Running e2e vrt playwright job');
102+
const reportDir = `reports/vrt_report_${shardIndex}`;
103103
async function postCommandTasks() {
104104
await compress({
105105
src: path.join('e2e', reportDir),
106-
dest: `.buildkite/artifacts/e2e_reports/report_${shardIndex}.gz`,
106+
dest: `.buildkite/artifacts/vrt_reports/report_${shardIndex}.gz`,
107107
});
108108

109109
if (bkEnv.steps.playwright.updateScreenshots) {
@@ -119,7 +119,7 @@ void (async () => {
119119
env: {
120120
[ENV_URL]: 'http://127.0.0.1:9002',
121121
PLAYWRIGHT_HTML_REPORT: reportDir,
122-
PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/json/report_${shardIndex}.json`,
122+
PLAYWRIGHT_JSON_OUTPUT_NAME: `reports/vrt-json/report_${shardIndex}.json`,
123123
},
124124
});
125125
await postCommandTasks();

.buildkite/steps/firebase_deploy.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,13 @@ export const firebaseDeployStep = createStep<CustomCommandStep>(() => {
1515
label: ':firebase: Deploy - firebase',
1616
key: 'deploy_fb',
1717
allow_dependency_failure: true,
18-
depends_on: ['build_docs', 'build_storybook', 'build_e2e', 'playwright_merge_and_status'],
18+
depends_on: [
19+
'build_docs',
20+
'build_storybook',
21+
'build_e2e',
22+
'playwright_vrt_merge_and_status',
23+
'playwright_a11y_merge_and_status',
24+
],
1925
commands: ['npx ts-node .buildkite/scripts/steps/firebase_deploy.ts'],
2026
env: {
2127
ECH_CHECK_ID: 'deploy_fb',

.buildkite/steps/index.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ export * from './eslint';
1111
export * from './api_check';
1212
export * from './type_check';
1313
export * from './prettier';
14-
export * from './playwright';
14+
export * from './playwright_vrt';
15+
export * from './playwright_a11y';
1516
export * from './docs';
1617
export * from './storybook';
1718
export * from './e2e_server';

.buildkite/steps/playwright_a11y.ts

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
import type { CustomGroupStep } from '../utils';
10+
import { createStep, commandStepDefaults, Plugins } from '../utils';
11+
12+
export const playwrightA11yStep = createStep<CustomGroupStep>(() => {
13+
const skip = false;
14+
const parallelKey = 'playwright_a11y__parallel-step';
15+
return {
16+
group: ':playwright: Playwright e2e A11Y',
17+
key: 'playwright_a11y',
18+
skip,
19+
steps: [
20+
{
21+
...commandStepDefaults,
22+
label: ':playwright: Playwright e2e A11Y',
23+
skip,
24+
parallelism: 1,
25+
retry: {
26+
automatic: [
27+
{
28+
// Playwright tests likely failed correctly
29+
exit_status: 1,
30+
limit: 0,
31+
},
32+
{
33+
// Something went wrong with step command setup, retry once
34+
exit_status: '*',
35+
limit: 1,
36+
},
37+
],
38+
},
39+
timeout_in_minutes: 10, // Shorter timeout for a11y tests
40+
key: parallelKey,
41+
depends_on: ['build_e2e'],
42+
plugins: [Plugins.docker.playwright()],
43+
artifact_paths: ['.buildkite/artifacts/a11y_reports/*', 'e2e/reports/a11y-json/*'],
44+
commands: ['npx ts-node .buildkite/scripts/steps/playwright_a11y.ts'],
45+
},
46+
{
47+
...commandStepDefaults,
48+
key: 'playwright_a11y_merge_and_status',
49+
label: ':playwright: Set a11y group status and merge reports',
50+
skip,
51+
allow_dependency_failure: true,
52+
depends_on: [{ step: parallelKey, allow_failure: true }],
53+
commands: ['npx ts-node .buildkite/scripts/steps/e2e_reports.ts'],
54+
env: {
55+
ECH_CHECK_ID: 'playwright_a11y',
56+
},
57+
},
58+
],
59+
};
60+
});

.buildkite/steps/playwright.ts renamed to .buildkite/steps/playwright_vrt.ts

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,17 @@
99
import type { CustomGroupStep } from '../utils';
1010
import { createStep, commandStepDefaults, Plugins } from '../utils';
1111

12-
export const playwrightStep = createStep<CustomGroupStep>(() => {
12+
export const playwrightVrtStep = createStep<CustomGroupStep>(() => {
1313
const skip = false;
14-
const parallelKey = 'playwright__parallel-step';
14+
const parallelKey = 'playwright_vrt__parallel-step';
1515
return {
16-
group: ':playwright: Playwright e2e',
17-
key: 'playwright',
16+
group: ':playwright: Playwright e2e VRT',
17+
key: 'playwright_vrt',
1818
skip,
1919
steps: [
2020
{
2121
...commandStepDefaults,
22-
label: ':playwright: Playwright e2e',
22+
label: ':playwright: Playwright e2e VRT',
2323
skip,
2424
parallelism: 10,
2525
retry: {
@@ -41,23 +41,23 @@ export const playwrightStep = createStep<CustomGroupStep>(() => {
4141
depends_on: ['build_e2e'],
4242
plugins: [Plugins.docker.playwright()],
4343
artifact_paths: [
44-
'.buildkite/artifacts/e2e_reports/*',
44+
'.buildkite/artifacts/vrt_reports/*',
4545
'.buildkite/artifacts/screenshots/*',
4646
'.buildkite/artifacts/screenshot_meta/*',
4747
'e2e/reports/json/*',
4848
],
49-
commands: ['npx ts-node .buildkite/scripts/steps/playwright.ts'],
49+
commands: ['npx ts-node .buildkite/scripts/steps/playwright_vrt.ts'],
5050
},
5151
{
5252
...commandStepDefaults,
53-
key: 'playwright_merge_and_status',
54-
label: ':playwright: Set group status and merge reports',
53+
key: 'playwright_vrt_merge_and_status',
54+
label: ':playwright: Set vrt group status and merge reports',
5555
skip,
5656
allow_dependency_failure: true,
5757
depends_on: [{ step: parallelKey, allow_failure: true }],
5858
commands: ['npx ts-node .buildkite/scripts/steps/e2e_reports.ts'],
5959
env: {
60-
ECH_CHECK_ID: 'playwright',
60+
ECH_CHECK_ID: 'playwright_vrt',
6161
},
6262
},
6363
],

0 commit comments

Comments
 (0)