jakob48/crumbs
1
0
Fork 0
forked from mirror/crumbs-original
Code Pull Requests Activity

fix: eliminate e2e test flakiness in settings preference and API key tests

Browse Source 
Three sources of flakiness fixed:

1. **Preferences sync race** — page.goto() triggers a full reload which
   re-runs initPreferences() + syncPreferencesFromServer(). On slow CI,
   the async server sync can return stale data from parallel test workers
   that share the same user's server-side preferences. Fixed with
   gotoWithStablePrefs() which blocks the one GET /api/preferences sync
   request, so initPreferences() falls back to localStorage (always correct).

2. **localStorage not reset** — resetPreferences() only reset the server.
   After a full page reload, loadFromLocalStorage() could return stale
   values from previous test runs. Now resets both server and localStorage.

3. **API key name collision** — parallel test runs created keys with the
   same name ("Settings Test Key"), causing Playwright strict-mode
   violations when multiple elements matched. Fixed with Date.now()
   unique suffixes.

Verified: 167/167 settings tests pass (20x repeat), 173/173 full suite (5x).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Frederic Cabassut
2026-03-10 23:39:29 +01:00
co-authored by Claude Opus 4.6
parent fc391d675c
commit 370b0bc815
1 changed files with 58 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/settings.spec.ts
+58 -14
View File
@@ -1,7 +1,20 @@
import { test, expect } from './helpers/fixtures.js';
import type { Page } from '@playwright/test';
/** Reset all preferences to defaults via API so each test starts clean */
const DEFAULT_PREFS = {
defaultNoteMode: 'richtext',
defaultNoteColor: 'default',
hideFooter: false,
sidebarDefaultState: 'open'
};
/**
* Reset preferences to defaults on both server and client localStorage.
*
* Server reset ensures GET /api/preferences returns defaults.
* localStorage reset ensures initPreferences() picks up correct values
* even if the sync request is blocked or returns stale data.
*/
async function resetPreferences(page: Page) {
await page.request.put('/api/preferences', {
data: {
@@ -11,6 +24,31 @@ async function resetPreferences(page: Page) {
sidebarDefaultState: 'open'
}
});
await page.evaluate(
(prefs) => localStorage.setItem('crumbs-preferences', JSON.stringify(prefs)),
DEFAULT_PREFS
);
}
/**
* Navigate to a URL with reliable preference state.
*
* page.goto() triggers a full page reload which re-runs initPreferences():
* 1. loadFromLocalStorage() — synchronous, returns correct value
* 2. syncPreferencesFromServer() — async, can return stale data from
* parallel test workers that share the same user's server-side prefs
*
* This helper blocks the one GET /api/preferences sync request so
* initPreferences() falls back to localStorage (which is always correct
* from either resetPreferences or updatePreference on the settings page).
*/
async function gotoWithStablePrefs(page: Page, url: string) {
await page.route(
'**/api/preferences',
(route) => route.abort(),
{ times: 1 }
);
await page.goto(url);
}
test.describe('Settings — Preferences', () => {
@@ -30,13 +68,13 @@ test.describe('Settings — Preferences', () => {
await resetPreferences(page);
// Given the user sets default note mode to Markdown
await page.goto('/settings/preferences');
await gotoWithStablePrefs(page, '/settings/preferences');
const putResponse = page.waitForResponse((res) => res.url().includes('/api/preferences') && res.request().method() === 'PUT');
await page.getByTestId('pref-mode-markdown').click();
await putResponse;
// When the user creates a new note
await page.goto('/');
await gotoWithStablePrefs(page, '/');
await page.getByTestId('new-note-btn').click();
// Then the note editor opens in markdown mode (textarea visible)
@@ -49,23 +87,23 @@ test.describe('Settings — Preferences', () => {
await resetPreferences(page);
// Given the footer is visible
await page.goto('/');
await gotoWithStablePrefs(page, '/');
await expect(page.getByTestId('app-footer')).toBeVisible();
// When the user enables the hide footer preference
await page.goto('/settings/preferences');
await gotoWithStablePrefs(page, '/settings/preferences');
await page.getByTestId('pref-hide-footer').check();
// Then the footer is hidden
await page.goto('/');
await gotoWithStablePrefs(page, '/');
await expect(page.getByTestId('app-footer')).not.toBeVisible();
// When the user disables the hide footer preference
await page.goto('/settings/preferences');
await gotoWithStablePrefs(page, '/settings/preferences');
await page.getByTestId('pref-hide-footer').uncheck();
// Then the footer is visible again
await page.goto('/');
await gotoWithStablePrefs(page, '/');
await expect(page.getByTestId('app-footer')).toBeVisible();
});
@@ -75,12 +113,13 @@ test.describe('Settings — Preferences', () => {
await resetPreferences(page);
// Given the user changes default note mode to Markdown
await page.goto('/settings/preferences');
await gotoWithStablePrefs(page, '/settings/preferences');
const putResponse = page.waitForResponse((res) => res.url().includes('/api/preferences') && res.request().method() === 'PUT');
await page.getByTestId('pref-mode-markdown').click();
await putResponse;
// When the page is reloaded
// When the page is reloaded (block sync to prevent parallel-test interference)
await page.route('**/api/preferences', (route) => route.abort(), { times: 1 });
await page.reload();
// Then the Markdown button is still selected (has primary styling)
@@ -93,9 +132,12 @@ test.describe('Settings — API Key Management', () => {
test('Scenario: Created API key appears in the keys list', async ({
authenticatedPage: page
}) => {
// Use a unique name to avoid strict-mode violations from parallel test runs
const keyName = `Test Key ${Date.now()}`;
// When the user navigates to the MCP settings and creates an API key
await page.goto('/settings/mcp');
await page.getByTestId('api-key-name-input').fill('Settings Test Key');
await page.getByTestId('api-key-name-input').fill(keyName);
await page.getByTestId('create-api-key-btn').click();
// Then the key is shown once for copying
@@ -104,20 +146,22 @@ test.describe('Settings — API Key Management', () => {
expect(keyValue).toMatch(/^crumbs_/);
// And it appears in the keys list
const keyItem = page.getByTestId('api-key-item').filter({ hasText: 'Settings Test Key' });
const keyItem = page.getByTestId('api-key-item').filter({ hasText: keyName });
await expect(keyItem).toBeVisible();
});
test('Scenario: Revoked API key disappears from the keys list', async ({
authenticatedPage: page
}) => {
const keyName = `Revoke Key ${Date.now()}`;
// Given an API key exists
await page.goto('/settings/mcp');
await page.getByTestId('api-key-name-input').fill('Revoke Me');
await page.getByTestId('api-key-name-input').fill(keyName);
await page.getByTestId('create-api-key-btn').click();
await expect(page.getByTestId('created-key-display')).toBeVisible();
const keyItem = page.getByTestId('api-key-item').filter({ hasText: 'Revoke Me' });
const keyItem = page.getByTestId('api-key-item').filter({ hasText: keyName });
await expect(keyItem).toBeVisible();
// When the user revokes the key