Skip to content

fix(e2e): increase timeout values to reduce test flakiness #5501

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 10, 2025
Merged

fix(e2e): increase timeout values to reduce test flakiness #5501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions e2e-tests/e2eTestUtils/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ export const ENV_URLS = {
local: process.env.SITE_ORIGIN,
};

export const TIMEOUTS = {
SHORT: 2000,
MEDIUM: 5000,
LONG: 10000,
};

export const getVerificationCode = async (
testEmail: string,
page: Page,
Expand All @@ -33,7 +39,7 @@ export const getVerificationCode = async (
return verificationCode;
}

await page.waitForTimeout(2000);
await page.waitForTimeout(TIMEOUTS.SHORT);
return getVerificationCode(testEmail, page, attempts - 1);
};

Expand All @@ -59,43 +65,45 @@ const setYourPassword = async (page: Page) => {
await page
.locator('button:has-text("Create account")')
.click({ force: true });
await page.waitForTimeout(2000);
await page.waitForTimeout(TIMEOUTS.SHORT);
await checkAuthState(page);
};

const enterConfirmationCode = async (page: Page) => {
const maybeVerificationCodeInput = "div.card input";
await page.waitForSelector(maybeVerificationCodeInput, { timeout: 2000 });
await page.waitForSelector(maybeVerificationCodeInput, {
timeout: TIMEOUTS.SHORT,
});
const confirmButton = page.locator('[type="submit"]').first();
const verificationCode = await getVerificationCode(
process.env.E2E_TEST_ACCOUNT_FREE as string,
page,
);
await page.locator(maybeVerificationCodeInput).fill(verificationCode);
await confirmButton.click({ force: true });
await page.waitForTimeout(2000);
await page.waitForTimeout(TIMEOUTS.SHORT);
await checkAuthState(page);
};

const signIn = async (page: Page) => {
const signInButton = page.getByRole("button", { name: "Sign in" });
await signInButton.waitFor({ timeout: 2000 });
await signInButton.waitFor({ timeout: TIMEOUTS.SHORT });
await page.waitForLoadState("networkidle");
await page.waitForLoadState("domcontentloaded");
await page.getByRole("button", { name: "Sign in" }).click({ force: true });
await page.waitForTimeout(2000);
await page.waitForTimeout(TIMEOUTS.SHORT);
await checkAuthState(page);
};

const enterYourEmail = async (page: Page) => {
const maybeEmailInput = 'input[name="email"]';
await page.waitForSelector(maybeEmailInput, { timeout: 2000 });
await page.waitForSelector(maybeEmailInput, { timeout: TIMEOUTS.SHORT });
const signInButton = page.locator('[type="submit"]').first();
await page
.locator(maybeEmailInput)
.fill(process.env.E2E_TEST_ACCOUNT_FREE as string);
await signInButton.click({ force: true });
await page.waitForTimeout(500);
await page.waitForTimeout(TIMEOUTS.SHORT);
await checkAuthState(page);
};

Expand All @@ -107,7 +115,7 @@ const enterYourPassword = async (page: Page) => {

// using force here due to fxa issue with playwright
await page.locator('[type="submit"]').first().click();
await page.waitForTimeout(500);
await page.waitForTimeout(TIMEOUTS.SHORT);
await checkAuthState(page);
};

Expand Down Expand Up @@ -154,7 +162,7 @@ export const checkAuthState = async (page: Page) => {
const authStateTitleString = await page
.locator("h1")
.first()
?.textContent({ timeout: 5000 });
?.textContent({ timeout: TIMEOUTS.MEDIUM });

const checkIfTitleContains = (potentialTitle: string) => {
return authStateTitleString?.includes(potentialTitle);
Expand Down
12 changes: 6 additions & 6 deletions e2e-tests/pages/authPage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Locator, Page } from "@playwright/test";
import { forceNonReactLink } from "../e2eTestUtils/helpers";
import { forceNonReactLink, TIMEOUTS } from "../e2eTestUtils/helpers";

export class AuthPage {
readonly page: Page;
Expand Down Expand Up @@ -46,7 +46,7 @@ export class AuthPage {
async enterVerificationCode(code: string) {
await this.page
.getByText("Enter confirmation code")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.LONG });
await forceNonReactLink(this.page);
await this.verifyCodeInputField.fill(code);
await this.confirmCodeButton.click();
Expand All @@ -70,24 +70,24 @@ export class AuthPage {
async login(email: string) {
await this.page
.getByText("Enter your email")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.LONG });
await this.enterEmail(email);
await this.page
.getByText("Enter your password")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.LONG });
await this.enterPassword();
}

async signUp(email: string, emailEntered: boolean = false) {
if (!emailEntered) {
await this.page
.getByText("Enter your email")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.LONG });
await this.enterEmail(email);
}
await this.page
.getByText("Set your password")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.LONG });
await this.passwordSignupInputField.fill(
process.env.E2E_TEST_ACCOUNT_PASSWORD as string,
);
Expand Down
24 changes: 13 additions & 11 deletions e2e-tests/pages/dashboardPage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Locator, Page, expect } from "@playwright/test";
import { checkAuthState } from "../e2eTestUtils/helpers";
import { checkAuthState, TIMEOUTS } from "../e2eTestUtils/helpers";

export class DashboardPage {
readonly page: Page;
Expand Down Expand Up @@ -215,7 +215,7 @@ export class DashboardPage {
async skipOnboarding() {
const onboardingElem = this.page.getByRole("button", { name: "Skip" });

if (await onboardingElem.isVisible({ timeout: 6000 })) {
if (await onboardingElem.isVisible({ timeout: TIMEOUTS.LONG })) {
await onboardingElem.click();
}
}
Expand Down Expand Up @@ -262,12 +262,12 @@ export class DashboardPage {

if (preMaskCardsCount === 0) {
// Wait for the first mask card
expect(maskCards).toBeVisible({ timeout: 3000 });
expect(maskCards).toBeVisible({ timeout: TIMEOUTS.MEDIUM });
} else {
// Wait for the mask card count to increase, or the error banner
expect(
this.bannerEmailError.or(maskCards.nth(preMaskCardsCount)),
).toBeVisible({ timeout: 3000 });
).toBeVisible({ timeout: TIMEOUTS.MEDIUM });
}

expect(
Expand All @@ -279,7 +279,7 @@ export class DashboardPage {
);

// randomize between .5-1.0 secs between each generate to deal with issue of multiple quick clicks
await this.page.waitForTimeout(Math.random() * 500 + 500);
await this.page.waitForTimeout(Math.random() * 500 + 1000);
if (await this.closeCornerUpsell.isVisible()) {
await this.closeCornerUpsell.click();
}
Expand Down Expand Up @@ -322,7 +322,9 @@ export class DashboardPage {
// if clear all, check if there's an expanded mask card
if (clearAll) {
try {
await this.page.waitForSelector(this.maskCardString, { timeout: 3000 });
await this.page.waitForSelector(this.maskCardString, {
timeout: TIMEOUTS.MEDIUM,
});
} catch (error) {
console.error("There are no masks to delete");
return;
Expand Down Expand Up @@ -353,7 +355,7 @@ export class DashboardPage {
}

// wait for 500 ms and run flow again with the next masks
await this.page.waitForTimeout(500);
await this.page.waitForTimeout(TIMEOUTS.SHORT);
await this.maybeDeleteMasks(true, numberOfMasks - 1);
}

Expand Down Expand Up @@ -406,7 +408,7 @@ export class DashboardPage {
await this.page.getByRole("button", { name: "Save" }).click();
await this.page
.getByText("Your settings have been updated")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.MEDIUM });
await this.page
.locator("//button[starts-with(@class, 'Toastify__close-button')]")
.click();
Expand All @@ -421,14 +423,14 @@ export class DashboardPage {
await this.page
.getByRole("button", { name: "let me have that cheat sheet!" })
.click();
await this.page.waitForTimeout(5000);
await this.page.waitForTimeout(TIMEOUTS.MEDIUM);
const captchaShown = await this.page.isVisible(".seva-overlay");
if (captchaShown) {
throw new Error("Unable to continue test, captcha was shown");
}
await this.page
.getByText("New to Agile? You NEED To See This!")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.MEDIUM });
await this.open();
}

Expand Down Expand Up @@ -461,7 +463,7 @@ export class DashboardPage {
return trackersCount;
}

this.page.waitForTimeout(500);
this.page.waitForTimeout(TIMEOUTS.MEDIUM);
return this.checkTrackersCount(attempts - 1);
}
}
7 changes: 2 additions & 5 deletions e2e-tests/pages/mozillaMonitorPage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
import { Page, Locator } from "@playwright/test";
import {
forceNonReactLink,
getVerificationCode,
} from "../e2eTestUtils/helpers";
import { TIMEOUTS } from "../e2eTestUtils/helpers";
import { AuthPage } from "./authPage";

export class MozillaMonitorPage {
Expand Down Expand Up @@ -32,6 +29,6 @@ export class MozillaMonitorPage {
await authPage.signUp(randomMask, true);
await this.page
.getByText("Enter confirmation code")
.waitFor({ state: "attached", timeout: 3000 });
.waitFor({ state: "attached", timeout: TIMEOUTS.MEDIUM });
}
}
7 changes: 2 additions & 5 deletions playwright.config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ const config = defineConfig({
/* Add location of specs. */
testDir: 'e2e-tests/specs',

/* Maximum time one test can run for. 3 minutes */
timeout: 60 * 1000,
/* Maximum time one test can run for. 2 minutes */
timeout: 60 * 2 * 1000,

/* Global setup */
globalSetup: require.resolve('./e2e-tests/global-setup.ts'),
Expand Down Expand Up @@ -48,9 +48,6 @@ const config = defineConfig({
],
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
actionTimeout: 0,

/* Base URL to use in actions like `await page.goto('/')`. */
baseURL: process.env.E2E_TEST_BASE_URL || 'https://stage.fxprivaterelay.nonprod.cloudops.mozgcp.net',

Expand Down