File: test_a11y.py

package info (click to toggle)
pydata-sphinx-theme 0.16.1%2Bdfsg-3
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 6,088 kB
  • sloc: python: 2,796; javascript: 701; makefile: 42; sh: 12
file content (293 lines) | stat: -rw-r--r-- 11,261 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
"""Using Axe-core, scan the Kitchen Sink pages for accessibility violations."""

from urllib.parse import urljoin

import pytest


# Using importorskip to ensure these tests are only loaded if Playwright is installed.
playwright = pytest.importorskip("playwright")
from playwright.sync_api import Page, expect  # noqa: E402


# Important note: automated accessibility scans can only find a fraction of
# potential accessibility issues.
#
# This test file scans pages from the Kitchen Sink examples with a JavaScript
# library called Axe-core, which checks the page for accessibility violations,
# such as places on the page with poor color contrast that would be hard for
# people with low vision to see.
#
# Just because a page passes the scan with no accessibility violations does
# *not* mean that it will be generally usable by a broad range of disabled
# people. It just means that page is free of common testable accessibility
# pitfalls.


def filter_ignored_violations(violations, url_pathname):
    """Filter out ignored axe-core violations.

    In some tests, we wish to ignore certain accessibility violations that we
    won't ever fix or that we don't plan to fix soon.
    """
    # we allow empty table headers
    # https://dequeuniversity.com/rules/axe/4.8/empty-table-header?application=RuleDescription
    if url_pathname == "/examples/pydata.html":
        return [v for v in violations if v["id"] != "empty-table-header"]
    elif url_pathname in [
        "/examples/kitchen-sink/generic.html",
        "/user_guide/theme-elements.html",
    ]:
        filtered = []
        for violation in violations:
            # TODO: remove this exclusion once the following update to Axe is
            # released and we upgrade:
            # https://github.com/dequelabs/axe-core/pull/4469
            if violation["id"] == "landmark-unique":
                # Ignore landmark-unique only for .sidebar targets. Don't ignore
                # it for other targets because then the test might fail to catch
                # a change that violates the rule in some other way.
                unexpected_nodes = []
                for node in violation["nodes"]:
                    # If some target is not .sidebar then we've found a rule
                    # violation we weren't expecting
                    if not all([".sidebar" in target for target in node["target"]]):
                        unexpected_nodes.append(node)
                if unexpected_nodes:
                    violation["nodes"] = unexpected_nodes
                    filtered.append(violation)
            else:
                filtered.append(violation)
        return filtered
    else:
        return violations


def format_violations(violations):
    """Return a pretty string representation of Axe-core violations."""
    result = f"""

        Found {len(violations)} accessibility violation(s):
        """

    for violation in violations:
        result += f"""

            - Rule violated:
              {violation["id"]} - {violation["help"]}
                - URL: {violation["helpUrl"]}
                - Impact: {violation["impact"]}
                - Tags: {" ".join(violation["tags"])}
                - Targets:"""

        for node in violation["nodes"]:
            for target in node["target"]:
                result += f"""
                    - {target}"""

        result += "\n\n"

    return result


@pytest.mark.a11y
@pytest.mark.parametrize("theme", ["light", "dark"])
@pytest.mark.parametrize(
    "url_pathname,selector",
    [
        (
            "/examples/kitchen-sink/admonitions.html",
            "#admonitions",
        ),
        (
            "/examples/kitchen-sink/api.html",
            "#api-documentation",
        ),
        ("/examples/kitchen-sink/blocks.html", "#blocks"),
        (
            "/examples/kitchen-sink/generic.html",
            "#generic-items",
        ),
        (
            "/examples/kitchen-sink/images.html",
            "#images-figures",
        ),
        ("/examples/kitchen-sink/lists.html", "#lists"),
        (
            "/examples/kitchen-sink/structure.html",
            "#structural-elements",
        ),
        (
            "/examples/kitchen-sink/structure.html",
            "#structural-elements-2",
        ),
        ("/examples/kitchen-sink/tables.html", "#tables"),
        (
            "/examples/kitchen-sink/typography.html",
            "#typography",
        ),
        ("/examples/pydata.html", "#PyData-Library-Styles"),
        (
            "/user_guide/theme-elements.html",
            "#theme-specific-elements",
        ),
        (
            "/user_guide/web-components.html",
            "#sphinx-design-components",
        ),
        (
            "/user_guide/extending.html",
            "#extending-the-theme",
        ),
        (
            "/user_guide/styling.html",
            "#theme-variables-and-css",
        ),
        # Using one of the simplest pages on the site, select the whole page for
        # testing in order to effectively test repeated website elements like
        # nav, sidebars, breadcrumbs, footer
        (
            "/user_guide/page-toc.html",
            "",  # select whole page
        ),
    ],
)
def test_axe_core(
    page: Page,
    url_base: str,
    theme: str,
    url_pathname: str,
    selector: str,
):
    """Should have no Axe-core violations at the provided theme and page section."""
    # Load the page at the provided path
    url_full = urljoin(url_base, url_pathname)
    page.goto(url_full)

    # Run a line of JavaScript that sets the light/dark theme on the page
    page.evaluate(f"document.documentElement.dataset.theme = '{theme}'")

    # Wait for CSS transitions (Bootstrap's transitions are 300 ms)
    page.wait_for_timeout(301)

    # On the PyData Library Styles page, wait for ipywidget to load and for our
    # JavaScript to apply tabindex="0" before running Axe checker (to avoid
    # false positives for scrollable-region-focusable).
    if url_pathname == "/examples/pydata.html":
        ipywidgets_pandas_table = page.locator("css=.jp-RenderedHTMLCommon").first
        expect(ipywidgets_pandas_table).to_have_attribute("tabindex", "0")

    # Inject the Axe-core JavaScript library into the page
    page.add_script_tag(path="node_modules/axe-core/axe.min.js")

    # Run the Axe-core library against a section of the page (unless the
    # selector is empty, then run against the whole page)
    results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')")

    # Check found violations against known violations that we do not plan to fix
    filtered_violations = filter_ignored_violations(results["violations"], url_pathname)

    # We expect notebook outputs on the PyData Library Styles page to have color
    # contrast failures.
    if url_pathname == "/examples/pydata.html":
        # All violations should be color contrast violations
        for violation in filtered_violations:
            assert (
                violation["id"] == "color-contrast"
            ), f"""Found {violation['id']} violation (expected color-contrast):
                    {format_violations([violation])}"""

        # Now check that when we exclude notebook outputs, the page has no violations

        results_sans_nbout = page.evaluate(
            f"axe.run({{ include: '{selector}', exclude: '.nboutput > .output_area' }})"
        )
        violations_sans_nbout = filter_ignored_violations(
            results_sans_nbout["violations"], url_pathname
        )

        # No violations on page when excluding notebook outputs
        assert len(violations_sans_nbout) == 0, format_violations(violations_sans_nbout)

        # TODO: for color contrast issues with common notebook outputs
        # (ipywidget tabbed panels, Xarray, etc.), should we override
        # third-party CSS with our own CSS or/and work with NbSphinx, MyST-NB,
        # ipywidgets, and other third parties to use higher contrast colors in
        # their CSS?
        pytest.xfail("notebook outputs have color contrast violations")

    assert len(filtered_violations) == 0, format_violations(filtered_violations)


@pytest.mark.a11y
def test_code_block_tab_stop(page: Page, url_base: str) -> None:
    """Code blocks that have scrollable content should be tab stops."""
    page.set_viewport_size({"width": 1440, "height": 720})
    page.goto(urljoin(url_base, "/examples/kitchen-sink/blocks.html"))

    code_block = page.locator(
        "css=#code-block pre", has_text="from typing import Iterator"
    )

    # Viewport is wide, so code block content fits, no overflow, no tab stop
    assert code_block.evaluate("el => el.scrollWidth > el.clientWidth") is False
    assert code_block.evaluate("el => el.tabIndex") != 0

    page.set_viewport_size({"width": 400, "height": 720})

    # Resize handler is debounced with 300 ms wait time
    page.wait_for_timeout(301)

    # Narrow viewport, content overflows and code block should be a tab stop
    assert code_block.evaluate("el => el.scrollWidth > el.clientWidth") is True
    assert code_block.evaluate("el => el.tabIndex") == 0


@pytest.mark.a11y
def test_notebook_output_tab_stop(page: Page, url_base: str) -> None:
    """Notebook outputs that have scrollable content should be tab stops."""
    page.goto(urljoin(url_base, "/examples/pydata.html"))

    # A "plain" notebook output
    nb_output = page.locator("css=#Pandas > .nboutput > .output_area")

    # At the default viewport size (1280 x 720) the Pandas data table has
    # overflow...
    assert nb_output.evaluate("el => el.scrollWidth > el.clientWidth") is True

    # ...and so our js code on the page should make it keyboard-focusable
    # (tabIndex = 0)
    assert nb_output.evaluate("el => el.tabIndex") == 0


@pytest.mark.a11y
def test_notebook_ipywidget_output_tab_stop(page: Page, url_base: str) -> None:
    """Notebook ipywidget outputs that have scrollable content should be tab stops."""
    page.goto(urljoin(url_base, "/examples/pydata.html"))

    # An ipywidget notebook output
    ipywidget = page.locator("css=.jp-RenderedHTMLCommon").first

    # As soon as the ipywidget is attached to the page it should trigger the
    # mutation observer, which has a 300 ms debounce
    ipywidget.wait_for(state="attached")
    page.wait_for_timeout(301)

    # At the default viewport size (1280 x 720) the data table inside the
    # ipywidget has overflow...
    assert ipywidget.evaluate("el => el.scrollWidth > el.clientWidth") is True

    # ...and so our js code on the page should make it keyboard-focusable
    # (tabIndex = 0)
    assert ipywidget.evaluate("el => el.tabIndex") == 0


def test_breadcrumb_expansion(page: Page, url_base: str) -> None:
    """Foo."""
    # page.goto(urljoin(url_base, "community/practices/merge.html"))
    # expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text("Merge and review policy") # noqa: E501
    page.set_viewport_size({"width": 1440, "height": 720})
    page.goto(urljoin(url_base, "community/topics/config.html"))
    expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
        "Update Sphinx configuration during the build"
    )