1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
|
"""Using Axe-core, scan the Kitchen Sink pages for accessibility violations."""
from urllib.parse import urljoin
import pytest
# Using importorskip to ensure these tests are only loaded if Playwright is installed.
playwright = pytest.importorskip("playwright")
from playwright.sync_api import Page, expect # noqa: E402
# Important note: automated accessibility scans can only find a fraction of
# potential accessibility issues.
#
# This test file scans pages from the Kitchen Sink examples with a JavaScript
# library called Axe-core, which checks the page for accessibility violations,
# such as places on the page with poor color contrast that would be hard for
# people with low vision to see.
#
# Just because a page passes the scan with no accessibility violations does
# *not* mean that it will be generally usable by a broad range of disabled
# people. It just means that page is free of common testable accessibility
# pitfalls.
def filter_ignored_violations(violations, url_pathname):
"""Filter out ignored axe-core violations.
In some tests, we wish to ignore certain accessibility violations that we
won't ever fix or that we don't plan to fix soon.
"""
# we allow empty table headers
# https://dequeuniversity.com/rules/axe/4.8/empty-table-header?application=RuleDescription
if url_pathname == "/examples/pydata.html":
return [v for v in violations if v["id"] != "empty-table-header"]
elif url_pathname in [
"/examples/kitchen-sink/generic.html",
"/user_guide/theme-elements.html",
]:
filtered = []
for violation in violations:
# TODO: remove this exclusion once the following update to Axe is
# released and we upgrade:
# https://github.com/dequelabs/axe-core/pull/4469
if violation["id"] == "landmark-unique":
# Ignore landmark-unique only for .sidebar targets. Don't ignore
# it for other targets because then the test might fail to catch
# a change that violates the rule in some other way.
unexpected_nodes = []
for node in violation["nodes"]:
# If some target is not .sidebar then we've found a rule
# violation we weren't expecting
if not all([".sidebar" in target for target in node["target"]]):
unexpected_nodes.append(node)
if unexpected_nodes:
violation["nodes"] = unexpected_nodes
filtered.append(violation)
else:
filtered.append(violation)
return filtered
else:
return violations
def format_violations(violations):
"""Return a pretty string representation of Axe-core violations."""
result = f"""
Found {len(violations)} accessibility violation(s):
"""
for violation in violations:
result += f"""
- Rule violated:
{violation["id"]} - {violation["help"]}
- URL: {violation["helpUrl"]}
- Impact: {violation["impact"]}
- Tags: {" ".join(violation["tags"])}
- Targets:"""
for node in violation["nodes"]:
for target in node["target"]:
result += f"""
- {target}"""
result += "\n\n"
return result
@pytest.mark.a11y
@pytest.mark.parametrize("theme", ["light", "dark"])
@pytest.mark.parametrize(
"url_pathname,selector",
[
(
"/examples/kitchen-sink/admonitions.html",
"#admonitions",
),
(
"/examples/kitchen-sink/api.html",
"#api-documentation",
),
("/examples/kitchen-sink/blocks.html", "#blocks"),
(
"/examples/kitchen-sink/generic.html",
"#generic-items",
),
(
"/examples/kitchen-sink/images.html",
"#images-figures",
),
("/examples/kitchen-sink/lists.html", "#lists"),
(
"/examples/kitchen-sink/structure.html",
"#structural-elements",
),
(
"/examples/kitchen-sink/structure.html",
"#structural-elements-2",
),
("/examples/kitchen-sink/tables.html", "#tables"),
(
"/examples/kitchen-sink/typography.html",
"#typography",
),
("/examples/pydata.html", "#PyData-Library-Styles"),
(
"/user_guide/theme-elements.html",
"#theme-specific-elements",
),
(
"/user_guide/web-components.html",
"#sphinx-design-components",
),
(
"/user_guide/extending.html",
"#extending-the-theme",
),
(
"/user_guide/styling.html",
"#theme-variables-and-css",
),
# Using one of the simplest pages on the site, select the whole page for
# testing in order to effectively test repeated website elements like
# nav, sidebars, breadcrumbs, footer
(
"/user_guide/page-toc.html",
"", # select whole page
),
],
)
def test_axe_core(
page: Page,
url_base: str,
theme: str,
url_pathname: str,
selector: str,
):
"""Should have no Axe-core violations at the provided theme and page section."""
# Load the page at the provided path
url_full = urljoin(url_base, url_pathname)
page.goto(url_full)
# Run a line of JavaScript that sets the light/dark theme on the page
page.evaluate(f"document.documentElement.dataset.theme = '{theme}'")
# Wait for CSS transitions (Bootstrap's transitions are 300 ms)
page.wait_for_timeout(301)
# On the PyData Library Styles page, wait for ipywidget to load and for our
# JavaScript to apply tabindex="0" before running Axe checker (to avoid
# false positives for scrollable-region-focusable).
if url_pathname == "/examples/pydata.html":
ipywidgets_pandas_table = page.locator("css=.jp-RenderedHTMLCommon").first
expect(ipywidgets_pandas_table).to_have_attribute("tabindex", "0")
# Inject the Axe-core JavaScript library into the page
page.add_script_tag(path="node_modules/axe-core/axe.min.js")
# Run the Axe-core library against a section of the page (unless the
# selector is empty, then run against the whole page)
results = page.evaluate("axe.run()" if selector == "" else f"axe.run('{selector}')")
# Check found violations against known violations that we do not plan to fix
filtered_violations = filter_ignored_violations(results["violations"], url_pathname)
# We expect notebook outputs on the PyData Library Styles page to have color
# contrast failures.
if url_pathname == "/examples/pydata.html":
# All violations should be color contrast violations
for violation in filtered_violations:
assert (
violation["id"] == "color-contrast"
), f"""Found {violation['id']} violation (expected color-contrast):
{format_violations([violation])}"""
# Now check that when we exclude notebook outputs, the page has no violations
results_sans_nbout = page.evaluate(
f"axe.run({{ include: '{selector}', exclude: '.nboutput > .output_area' }})"
)
violations_sans_nbout = filter_ignored_violations(
results_sans_nbout["violations"], url_pathname
)
# No violations on page when excluding notebook outputs
assert len(violations_sans_nbout) == 0, format_violations(violations_sans_nbout)
# TODO: for color contrast issues with common notebook outputs
# (ipywidget tabbed panels, Xarray, etc.), should we override
# third-party CSS with our own CSS or/and work with NbSphinx, MyST-NB,
# ipywidgets, and other third parties to use higher contrast colors in
# their CSS?
pytest.xfail("notebook outputs have color contrast violations")
assert len(filtered_violations) == 0, format_violations(filtered_violations)
@pytest.mark.a11y
def test_code_block_tab_stop(page: Page, url_base: str) -> None:
"""Code blocks that have scrollable content should be tab stops."""
page.set_viewport_size({"width": 1440, "height": 720})
page.goto(urljoin(url_base, "/examples/kitchen-sink/blocks.html"))
code_block = page.locator(
"css=#code-block pre", has_text="from typing import Iterator"
)
# Viewport is wide, so code block content fits, no overflow, no tab stop
assert code_block.evaluate("el => el.scrollWidth > el.clientWidth") is False
assert code_block.evaluate("el => el.tabIndex") != 0
page.set_viewport_size({"width": 400, "height": 720})
# Resize handler is debounced with 300 ms wait time
page.wait_for_timeout(301)
# Narrow viewport, content overflows and code block should be a tab stop
assert code_block.evaluate("el => el.scrollWidth > el.clientWidth") is True
assert code_block.evaluate("el => el.tabIndex") == 0
@pytest.mark.a11y
def test_notebook_output_tab_stop(page: Page, url_base: str) -> None:
"""Notebook outputs that have scrollable content should be tab stops."""
page.goto(urljoin(url_base, "/examples/pydata.html"))
# A "plain" notebook output
nb_output = page.locator("css=#Pandas > .nboutput > .output_area")
# At the default viewport size (1280 x 720) the Pandas data table has
# overflow...
assert nb_output.evaluate("el => el.scrollWidth > el.clientWidth") is True
# ...and so our js code on the page should make it keyboard-focusable
# (tabIndex = 0)
assert nb_output.evaluate("el => el.tabIndex") == 0
@pytest.mark.a11y
def test_notebook_ipywidget_output_tab_stop(page: Page, url_base: str) -> None:
"""Notebook ipywidget outputs that have scrollable content should be tab stops."""
page.goto(urljoin(url_base, "/examples/pydata.html"))
# An ipywidget notebook output
ipywidget = page.locator("css=.jp-RenderedHTMLCommon").first
# As soon as the ipywidget is attached to the page it should trigger the
# mutation observer, which has a 300 ms debounce
ipywidget.wait_for(state="attached")
page.wait_for_timeout(301)
# At the default viewport size (1280 x 720) the data table inside the
# ipywidget has overflow...
assert ipywidget.evaluate("el => el.scrollWidth > el.clientWidth") is True
# ...and so our js code on the page should make it keyboard-focusable
# (tabIndex = 0)
assert ipywidget.evaluate("el => el.tabIndex") == 0
def test_breadcrumb_expansion(page: Page, url_base: str) -> None:
"""Foo."""
# page.goto(urljoin(url_base, "community/practices/merge.html"))
# expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text("Merge and review policy") # noqa: E501
page.set_viewport_size({"width": 1440, "height": 720})
page.goto(urljoin(url_base, "community/topics/config.html"))
expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
"Update Sphinx configuration during the build"
)
|