From 18d2e8da9fe330dfd71ab1a2f6b148434dc15b2f Mon Sep 17 00:00:00 2001 From: sabril <5334504+saturninoabril@users.noreply.github.com> Date: Wed, 15 Oct 2025 12:00:01 +0800 Subject: [PATCH] MM-63700 E2E/Playwright: Add accessibility testing guidelines (#33997) * add accessibility testing guidelines * use accessibility locators * address comments --- e2e-tests/playwright/README.md | 35 +++ .../accessibility/automated_scan_testing.md | 131 ++++++++++ .../playwright/docs/accessibility/index.md | 206 ++++++++++++++++ .../docs/accessibility/interaction_testing.md | 223 ++++++++++++++++++ e2e-tests/playwright/lib/src/test_fixture.ts | 2 +- 5 files changed, 596 insertions(+), 1 deletion(-) create mode 100644 e2e-tests/playwright/docs/accessibility/automated_scan_testing.md create mode 100644 e2e-tests/playwright/docs/accessibility/index.md create mode 100644 e2e-tests/playwright/docs/accessibility/interaction_testing.md diff --git a/e2e-tests/playwright/README.md b/e2e-tests/playwright/README.md index 09d824bd33f..47a8c61b40e 100644 --- a/e2e-tests/playwright/README.md +++ b/e2e-tests/playwright/README.md @@ -180,6 +180,41 @@ export PERCY_TOKEN= npm run percy:docker ``` +## Accessibility Testing + +Accessibility tests ensure Mattermost meets WCAG 2.1 AA compliance standards. Tests are located in `specs/accessibility/` and cover keyboard navigation, screen reader support, focus management, and automated accessibility scanning. + +For comprehensive guidelines on writing accessibility tests, aria snapshots, and folder structure, see [docs/accessibility/](docs/accessibility/). + +### Accessibility Locators + +**Playwright's accessibility locators should be the preferred approach for all tests, not just accessibility tests.** These locators query elements based on how users and assistive technologies perceive them, making tests more resilient to implementation changes and ensuring better accessibility by design. + +#### Why Use Accessibility Locators? + +- **Resilient to changes**: Tests won't break when CSS classes or data-testid attributes change +- **Encourages accessibility**: Forces proper ARIA roles, labels, and semantic HTML +- **Better readability**: `page.getByRole('button', {name: 'Save'})` is clearer than `page.locator('[data-testid="save-btn"]')` +- **Aligns with user experience**: Tests what users actually perceive, not implementation details + +#### Preferred Locators (in order of preference) + +1. **Role-based**: `page.getByRole('button', {name: 'Save'})`, `page.getByRole('textbox', {name: 'Email'})` +2. **Label-based**: `page.getByLabel('Email address')` +3. **Text-based**: `page.getByText('Welcome')`, `page.getByPlaceholder('Enter email')` +4. **Test IDs**: `page.locator('[data-testid="..."]')` - Use only when accessibility locators aren't possible +5. **CSS selectors**: `page.locator('.class')` - Avoid unless absolutely necessary + +#### When Test IDs Are Acceptable + +Use `data-testid` only when: + +- Element has no semantic role (e.g., decorative divs) +- Multiple identical elements need distinction +- Component is not interactive or visible to assistive tech + +For all test examples, see [docs/accessibility/](docs/accessibility/) for comprehensive patterns and best practices. + ## Page/Component Object Model See https://playwright.dev/docs/test-pom. diff --git a/e2e-tests/playwright/docs/accessibility/automated_scan_testing.md b/e2e-tests/playwright/docs/accessibility/automated_scan_testing.md new file mode 100644 index 00000000000..bb282d66010 --- /dev/null +++ b/e2e-tests/playwright/docs/accessibility/automated_scan_testing.md @@ -0,0 +1,131 @@ +# Automated Accessibility Scan Testing Guidelines + +## Overview + +This guide focuses on **automated accessibility testing** using a **simplified, comprehensive approach**. Use axe-core's default rule set and scope testing to specific elements, disabling rules only when necessary. This maximizes coverage while minimizing maintenance overhead. + +## Testing Philosophy + +### **Core Principle: Scope and Analyze** + +Use axe-core's comprehensive default rule set and scope testing to specific elements. + +**Benefits:** + +- **Maximum Coverage** - All WCAG 2.1 AA rules by default +- **Low Maintenance** - No rule lists to maintain +- **Future-Proof** - New axe-core rules automatically included +- **Clear Intent** - Disabled rules are explicit and documented + +### **Basic Pattern** + +```typescript +test('page accessibility', async ({axe, page}) => { + const results = await axe.builder(page).analyze(); // All applicable rules + + expect(results.violations).toHaveLength(0); +}); +``` + +```typescript +test('component accessibility', async ({axe, page}) => { + const results = await axe + .builder(page) + .include('#element') // Scope to element + .analyze(); // All applicable rules + + expect(results.violations).toHaveLength(0); +}); +``` + +## What Can Be Automated + +### **Fully Automatable** + +- **HTML Structure & Semantics** - Markup validity, semantic elements, heading hierarchy +- **ARIA Implementation** - Attributes, states, relationships, roles +- **Form Accessibility** - Labels, validation, error messages +- **Interactive Elements** - Focus capability, accessible names +- **Color and Contrast** - WCAG AA/AAA contrast ratios +- **Alternative Text** - Image alt text presence +- **Table Structure** - Headers, captions, relationships +- **Navigation** - Skip links, landmarks, focus order + +### **Cannot Be Fully Automated** + +- **Cognitive Accessibility** - Content readability, comprehension +- **Contextual Appropriateness** - Alt text quality, meaningful link text +- **Real User Experience** - Actual keyboard workflows, screen reader UX +- **Content Quality** - Clear instructions, effective error messaging + +## Testing Patterns + +### **1. Component Testing** + +```typescript +test('modal accessibility', async ({axe, page}) => { + await page.getByRole('button', {name: 'Open modal'}).click(); + + const results = await axe.builder(page).include('[role="dialog"]').analyze(); + + expect(results.violations).toHaveLength(0); +}); +``` + +### **2. State Testing** + +```typescript +test('form error state accessibility', async ({axe, page}) => { + // Test normal state + let results = await axe.builder(page).include('form').analyze(); + expect(results.violations).toHaveLength(0); + + // Test error state + await page.getByRole('button', {name: 'Submit'}).click(); + results = await axe.builder(page).include('form').analyze(); + expect(results.violations).toHaveLength(0); +}); +``` + +## Rule Management + +### **When to Disable Rules** + +**Disable Rules Only When:** + +- **Known UI Framework Limitations** - Theme-related contrast issues +- **Context-Inappropriate Rules** - Page-level rules in modal dialogs +- **Temporary Workarounds** - With clear documentation and tracking + +### **Examples** + +```typescript +// Good - Documented limitation +test('dark theme modal', async ({axe, page}) => { + const results = await axe + .builder(page) + .include('[role="dialog"]') + .disableRules([ + 'color-contrast', // TODO: MM-nnn - Color contrast improvement + ]) + .analyze(); +}); + +// Good - Context on comment +test('modal dialog', async ({axe, page}) => { + const results = await axe + .builder(page) + .include('[role="dialog"]') + .disableRules([ + 'page-has-heading-one', // Not applicable to modals + 'landmark-one-main', // Not applicable to modals + ]) + .analyze(); +}); +``` + +**Don't Disable Rules For:** + +- **Convenience** - "This rule is annoying" +- **Lack of Understanding** - "I don't know what this rule does" +- **Time Pressure** - "We'll fix it later" (without tracking) diff --git a/e2e-tests/playwright/docs/accessibility/index.md b/e2e-tests/playwright/docs/accessibility/index.md new file mode 100644 index 00000000000..083d3e01dd1 --- /dev/null +++ b/e2e-tests/playwright/docs/accessibility/index.md @@ -0,0 +1,206 @@ +# Accessibility Testing Guidelines + +Welcome to Mattermost's accessibility testing documentation. This guide covers how to write comprehensive accessibility tests that ensure our application meets WCAG 2.1 AA compliance standards and provides an inclusive experience for all users. + +## Quick Start + +1. **Location**: Place accessibility tests in `specs/accessibility/` +2. **Structure**: Organize by product → page → component +3. **Tags**: Include `@accessibility` and feature-specific tags +4. **Tools**: Use `axe` fixture for scanning, `toMatchAriaSnapshot` for semantic structure + +## Documentation Structure + +- **[This Page]**: Overview and folder organization +- **[Automated Scan Testing](automated_scan_testing.md)**: Using axe-core for automated accessibility scanning +- **[Interaction Testing](interaction_testing.md)**: Keyboard navigation, focus management, and screen reader testing + +## Folder Structure + +Accessibility tests follow a hierarchical organization by product area: + +``` +specs/accessibility/ +├── common/ # Shared components across products +│ ├── login.spec.ts +│ ├── reset_password.spec.ts +│ └── signup_user_complete.spec.ts +├── channels/ # Channels product area +│ ├── settings_dialog/ # Settings dialog page/component +│ │ ├── notifications.spec.ts +│ │ ├── settings.spec.ts +│ │ └── notifications.spec.ts-snapshots-a11y/ # Aria snapshots +│ │ ├── desktop-and-mobile-section.yml +│ │ ├── email-notifications-section.yml +│ │ └── keywords-that-get-highlighted-section.yml +│ ├── account_menu_keyboard.spec.ts +│ ├── intro_channel.spec.ts +│ └── theme_settings.spec.ts +└── [future-products]/ # Boards, Playbooks, etc. + └── [page-or-component]/ + └── test.spec.ts +``` + +### Naming Conventions + +- **Products**: `channels/`, `boards/`, `playbooks/` +- **Pages**: `settings_dialog/`, `channel_header/`, `post_menu/` +- **Components**: `notifications.spec.ts`, `theme_picker.spec.ts` +- **Snapshots**: `[component-name]-section.yml`, `[feature-name]-modal.yml` + +## Test Categories + +Accessibility tests should cover these key areas: + +### 1. Keyboard Navigation + +- Tab order and focus management +- Enter/Space key activation +- Escape key handling +- Arrow key navigation for menus and lists + +### 2. Screen Reader Support + +- Proper ARIA labels and roles +- Meaningful announcements for state changes +- Alternative text for images and icons +- Semantic HTML structure + +### 3. Focus Management + +- Visible focus indicators +- Focus trapping in modals and dialogs +- Logical focus restoration +- Skip links and landmarks + +### 4. Color and Contrast + +- WCAG AA compliance (4.5:1 normal, 3:1 large text) +- High contrast mode compatibility +- Color not as sole information indicator + +### 5. Zoom and Responsive + +- 200% zoom without horizontal scrolling +- Mobile accessibility patterns +- Touch target sizes (44x44px minimum) + +## Test Structure Template + +```typescript +/** + * @objective Clear description of what accessibility aspect is being verified + * + * @precondition Special setup conditions (omit if using standard setup) + */ +test('descriptive test title', {tag: ['@accessibility', '@feature_tag']}, async ({pw, axe}) => { + // # Setup user and navigate to target + const {user} = await pw.initSetup(); + const {page, channelsPage} = await pw.testBrowser.login(user); + await channelsPage.goto(); + + // # Navigate to component under test + await page.getByRole('button', {name: 'Settings'}).click(); + + // # Perform accessibility interaction (keyboard navigation, etc.) + await page.keyboard.press('Tab'); + const targetElement = page.getByRole('dialog', {name: 'Settings'}); + await pw.toBeFocusedWithFocusVisible(targetElement); + + // * Verify accessibility compliance with automated scan + const results = await axe.builder(page).include('[role="dialog"]').analyze(); + expect(results.violations).toHaveLength(0); + + // * Verify semantic structure with aria snapshot + await expect(targetElement).toMatchAriaSnapshot({ + name: 'component-state.yml', + }); +}); +``` + +## Aria Snapshots + +Aria snapshots capture the accessibility tree structure, ensuring proper semantic markup and screen reader experience. + +### When to Use External vs Inline Snapshots + +**Use External .yml Files** (Recommended): + +- Static content that doesn't change between test runs +- Reusable components across multiple tests +- Complex structures that benefit from version control and diff tracking + +```typescript +const notificationsPanel = page.getByRole('region', {name: 'Notifications'}); +await expect(notificationsPanel).toMatchAriaSnapshot({ + name: 'notifications-panel.yml', +}); +``` + +**Use Inline Snapshots**: + +- Content with dynamic data (user IDs, timestamps, server configurations) +- Simple structures where external files add complexity +- One-off components unlikely to be reused + +```typescript +await expect(element).toMatchAriaSnapshot(` + - tabpanel "notifications": + - heading "Notifications" [level=3] + - link "Learn more": + - /url: https://example.com/notifications?uid=${user.id}&sid=${clientConfig.DiagnosticId} + - button "Edit Desktop notifications" +`); +``` + +### Snapshot Organization + +- External snapshots: `[test-file].spec.ts-snapshots-a11y/` +- Descriptive names: `desktop-notifications-section.yml`, `keywords-modal-expanded.yml` +- Group by component or feature area + +## Updating Snapshots + +When UI structure changes, update aria snapshots: + +```bash +# Update all accessibility snapshots +npm run test:a11y-update-snapshots + +# Update specific test snapshots +npm run test -- specs/accessibility/channels/settings_dialog/notifications.spec.ts --update-snapshots + +# Update all snapshots including visual snapshots +npm run test:update-snapshots +``` + +## Required Tags + +All accessibility tests must include: + +- `@accessibility` - Primary accessibility identifier +- Feature tags: `@settings`, `@notifications`, `@login`, etc. +- `@snapshots` - For tests including aria snapshots + +## Best Practices + +1. **Test Real User Workflows**: Focus on actual user journeys, not isolated interactions +2. **Cover All Input Methods**: Mouse, keyboard, touch, and assistive technology +3. **Test Error States**: Form validation, network errors, loading states +4. **Include Dynamic Content**: Test with various data states and configurations +5. **Document Exceptions**: When disabling accessibility rules, document the rationale +6. **Cross-Browser Testing**: Different browsers handle accessibility differently +7. **Incremental Development**: Add accessibility tests alongside feature development + +## Visual Verification + +For visual verification needs beyond accessibility tree structure (such as focus indicators, high contrast mode, or visual layout validation), consider adding visual screenshot tests at `specs/visual/`. Visual tests complement accessibility testing by capturing the actual rendered appearance. + +See the main [README.md](../../README.md#visual-testing) for visual testing guidelines and setup. + +## Getting Help + +- Review existing tests in `specs/accessibility/` for patterns +- Check automated scan guidelines for axe-core usage +- See interaction testing docs for keyboard navigation patterns +- Ask questions in [~e2e-testing](https://community.mattermost.com/core/channels/e2e-testing) channels at Mattermost Community. diff --git a/e2e-tests/playwright/docs/accessibility/interaction_testing.md b/e2e-tests/playwright/docs/accessibility/interaction_testing.md new file mode 100644 index 00000000000..6bc0bedba76 --- /dev/null +++ b/e2e-tests/playwright/docs/accessibility/interaction_testing.md @@ -0,0 +1,223 @@ +# Accessibility Interaction Testing Best Practices + +## Overview + +This guide covers **automated accessibility testing through user interaction simulation** using Playwright. These techniques test real user experiences with assistive technologies by automating keyboard navigation, focus management, and screen reader workflows. + +## Core Capabilities + +### **Keyboard Navigation Testing** + +- **Arrow Key Navigation** - Menu and list traversal +- **Tab Navigation** - Focus order validation +- **Enter/Space Activation** - Button and link interaction +- **Escape Key Handling** - Modal and menu dismissal +- **Letter Key Shortcuts** - Quick navigation patterns + +### **Focus Management Validation** + +- **Focus Visibility** - `:focus-visible` state verification +- **Focus Trapping** - Modal and dialog containment +- **Focus Restoration** - Return focus after interactions +- **Focus Order** - Logical tab sequence validation + +### **State and Behavior Testing** + +- **ARIA State Changes** - `aria-expanded`, `aria-selected` verification +- **Dynamic Content** - Live region announcements +- **Error Handling** - Validation message accessibility +- **Loading States** - Progress indication accessibility + +## Testing Patterns + +### **1. Keyboard Navigation Testing** + +```typescript +test('keyboard navigation through menu', async ({page}) => { + // Open menu with keyboard + await page.getByRole('button', {name: 'Menu'}).focus(); + await page.keyboard.press('Enter'); + + // Navigate through menu items + await expect(page.getByRole('menuitem').first()).toBeFocused(); + + await page.keyboard.press('ArrowDown'); + await expect(page.getByRole('menuitem').nth(1)).toBeFocused(); + + await page.keyboard.press('ArrowDown'); + await expect(page.getByRole('menuitem').nth(2)).toBeFocused(); + + // Test wrapping behavior + await page.keyboard.press('ArrowUp'); + await expect(page.getByRole('menuitem').nth(1)).toBeFocused(); + + // Close menu with Escape + await page.keyboard.press('Escape'); + await expect(page.getByRole('menuitem')).toHaveCount(0); + + // Verify focus restoration + await expect(page.getByRole('button', {name: 'Menu'})).toBeFocused(); +}); +``` + +### **2. Focus Trapping in Modals** + +```typescript +test('modal focus trapping', async ({page}) => { + // Open modal + await page.getByRole('button', {name: 'Open modal'}).click(); + const modal = page.getByRole('dialog'); + + // Verify initial focus + await expect(modal).toBeFocused(); + + // Find all focusable elements in modal + const focusableElements = modal.locator('button, [href], input, select, textarea, [tabindex]:not([tabindex="-1"])'); + const count = await focusableElements.count(); + + // Tab through all elements + for (let i = 0; i < count; i++) { + await page.keyboard.press('Tab'); + await expect(focusableElements.nth(i)).toBeFocused(); + } + + // Verify focus wraps back to first element + await page.keyboard.press('Tab'); + await expect(modal).toBeFocused(); // Or first focusable element + + // Test reverse tabbing + await page.keyboard.press('Shift+Tab'); + await expect(focusableElements.last()).toBeFocused(); +}); +``` + +### **3. ARIA State Management Testing** + +```typescript +test('expandable section ARIA states', async ({page}) => { + const expandButton = page.getByRole('button', {name: 'Expand section', expanded: false}); + const expandableContent = page.getByRole('region', {name: 'Expandable content'}); + + // Test initial collapsed state + await expect(expandButton).toHaveAttribute('aria-expanded', 'false'); + await expect(expandableContent).toBeHidden(); + + // Expand with keyboard + await expandButton.focus(); + await page.keyboard.press('Enter'); + + // Verify expanded state + await expect(expandButton).toHaveAttribute('aria-expanded', 'true'); + await expect(expandableContent).toBeVisible(); + + // Verify focus moves to content + await expect(expandableContent).toBeFocused(); + + // Collapse with keyboard + await expandButton.focus(); + await page.keyboard.press('Enter'); + + // Verify collapsed state + await expect(expandButton).toHaveAttribute('aria-expanded', 'false'); + await expect(expandableContent).toBeHidden(); +}); +``` + +### **4. Form Validation Accessibility** + +```typescript +test('form validation accessibility', async ({page}) => { + const form = page.getByRole('form'); + const requiredField = form.getByRole('textbox', {name: 'Email'}); + const submitButton = form.getByRole('button', {name: 'Submit'}); + + // Submit empty form to trigger validation + await submitButton.focus(); + await page.keyboard.press('Enter'); + + // Verify error state accessibility + await expect(requiredField).toHaveAttribute('aria-invalid', 'true'); + await expect(page.getByRole('alert')).toBeVisible(); + + // Verify focus moves to invalid field + await expect(requiredField).toBeFocused(); + + // Test error message association + const errorId = await page.getByRole('alert').getAttribute('id'); + await expect(requiredField).toHaveAttribute('aria-describedby', errorId); + + // Fix validation error + await requiredField.fill('valid@example.com'); + await submitButton.focus(); + await page.keyboard.press('Enter'); + + // Verify error cleared + await expect(requiredField).toHaveAttribute('aria-invalid', 'false'); + await expect(page.getByRole('alert')).toBeHidden(); +}); +``` + +### **5. Live Region Announcements** + +```typescript +test('dynamic content announcements', async ({page}) => { + const liveRegion = page.getByRole('status'); + const triggerButton = page.getByRole('button', {name: 'Load content'}); + + // Verify live region setup + await expect(liveRegion).toHaveAttribute('aria-live', 'polite'); + + // Trigger content update + await triggerButton.click(); + + // Wait for content to appear in live region + await expect(liveRegion).toContainText('Loading complete'); + + // Test urgent announcements + const urgentRegion = page.getByRole('alert'); + await page.getByRole('button', {name: 'Urgent action'}).click(); + await expect(urgentRegion).toContainText('Critical update'); +}); +``` + +### **6. Focus Visibility Testing** + +```typescript +test('focus indicators visibility', async ({page}) => { + const button = page.getByRole('button', {name: 'Save'}); + + // Tab to button (should show focus indicator) + await page.keyboard.press('Tab'); + await toBeFocusedWithFocusVisible(button); +}); +``` + +### **7. Complex Navigation Patterns** + +```typescript +test('hierarchical menu navigation', async ({page}) => { + await page.getByRole('button', {name: 'Menu'}).focus(); + await page.keyboard.press('Enter'); + + // Navigate to submenu trigger + const submenuTrigger = page.getByRole('menuitem', {name: /has submenu/}); + await submenuTrigger.focus(); + + // Open submenu with right arrow + await page.keyboard.press('ArrowRight'); + + // Verify submenu opened and focused + const submenu = page.getByRole('menu').nth(1); + await expect(submenu).toBeVisible(); + await expect(submenu.getByRole('menuitem').first()).toBeFocused(); + + // Navigate in submenu + await page.keyboard.press('ArrowDown'); + await expect(submenu.getByRole('menuitem').nth(1)).toBeFocused(); + + // Return to parent menu with left arrow + await page.keyboard.press('ArrowLeft'); + await expect(submenu).toBeHidden(); + await expect(submenuTrigger).toBeFocused(); +}); +``` diff --git a/e2e-tests/playwright/lib/src/test_fixture.ts b/e2e-tests/playwright/lib/src/test_fixture.ts index f8819cdf517..ead77492f4c 100644 --- a/e2e-tests/playwright/lib/src/test_fixture.ts +++ b/e2e-tests/playwright/lib/src/test_fixture.ts @@ -191,7 +191,7 @@ export class AxeBuilderExtended { readonly builder: (page: Page, options?: AxeBuilderOptions) => AxeBuilder; // See https://github.com/dequelabs/axe-core/blob/master/doc/API.md#axe-core-tags - readonly tags: string[] = ['wcag2a', 'wcag2aa']; + readonly tags: string[] = ['wcag2a', 'wcag2aa', 'wcag21aa']; constructor() { this.builder = (page: Page, options: AxeBuilderOptions = {}) => {