1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
|
"""
Standalone file utils.
Nothing in this module should have an knowledge of config or the layout
and structure of the site and pages in the site.
"""
from __future__ import annotations
import functools
import logging
import os
import posixpath
import re
import shutil
import sys
import warnings
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import PurePath
from typing import TYPE_CHECKING, Collection, Iterable, MutableSequence, TypeVar
from urllib.parse import urlsplit
if sys.version_info >= (3, 10):
from importlib.metadata import EntryPoint, entry_points
else:
from importlib_metadata import EntryPoint, entry_points
from mkdocs import exceptions
from mkdocs.utils.yaml import get_yaml_loader, yaml_load # noqa: F401 - legacy re-export
if TYPE_CHECKING:
from mkdocs.structure.pages import Page
T = TypeVar('T')
log = logging.getLogger(__name__)
markdown_extensions = (
'.markdown',
'.mdown',
'.mkdn',
'.mkd',
'.md',
)
def get_build_timestamp(*, pages: Collection[Page] | None = None) -> int:
"""
Returns the number of seconds since the epoch for the latest updated page.
In reality this is just today's date because that's how pages' update time is populated.
"""
if pages:
# Lexicographic comparison is OK for ISO date.
date_string = max(p.update_date for p in pages)
dt = datetime.fromisoformat(date_string).replace(tzinfo=timezone.utc)
else:
dt = get_build_datetime()
return int(dt.timestamp())
def get_build_datetime() -> datetime:
"""
Returns an aware datetime object.
Support SOURCE_DATE_EPOCH environment variable for reproducible builds.
See https://reproducible-builds.org/specs/source-date-epoch/
"""
source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH')
if source_date_epoch is None:
return datetime.now(timezone.utc)
return datetime.fromtimestamp(int(source_date_epoch), timezone.utc)
def get_build_date() -> str:
"""
Returns the displayable date string.
Support SOURCE_DATE_EPOCH environment variable for reproducible builds.
See https://reproducible-builds.org/specs/source-date-epoch/
"""
return get_build_datetime().strftime('%Y-%m-%d')
if sys.version_info >= (3, 9):
_removesuffix = str.removesuffix
else:
def _removesuffix(s: str, suffix: str) -> str:
if suffix and s.endswith(suffix):
return s[: -len(suffix)]
return s
def reduce_list(data_set: Iterable[T]) -> list[T]:
"""Reduce duplicate items in a list and preserve order."""
return list(dict.fromkeys(data_set))
if sys.version_info >= (3, 10):
from bisect import insort
else:
def insort(a: MutableSequence[T], x: T, *, key=lambda v: v) -> None:
kx = key(x)
i = len(a)
while i > 0 and kx < key(a[i - 1]):
i -= 1
a.insert(i, x)
def copy_file(source_path: str, output_path: str) -> None:
"""
Copy source_path to output_path, making sure any parent directories exist.
The output_path may be a directory.
"""
output_dir = os.path.dirname(output_path)
os.makedirs(output_dir, exist_ok=True)
if os.path.isdir(output_path):
output_path = os.path.join(output_path, os.path.basename(source_path))
shutil.copyfile(source_path, output_path)
def write_file(content: bytes, output_path: str) -> None:
"""Write content to output_path, making sure any parent directories exist."""
output_dir = os.path.dirname(output_path)
os.makedirs(output_dir, exist_ok=True)
with open(output_path, 'wb') as f:
f.write(content)
def clean_directory(directory: str) -> None:
"""Remove the content of a directory recursively but not the directory itself."""
if not os.path.exists(directory):
return
for entry in os.listdir(directory):
# Don't remove hidden files from the directory. We never copy files
# that are hidden, so we shouldn't delete them either.
if entry.startswith('.'):
continue
path = os.path.join(directory, entry)
if os.path.isdir(path):
shutil.rmtree(path, True)
else:
os.unlink(path)
def is_markdown_file(path: str) -> bool:
"""
Return True if the given file path is a Markdown file.
https://superuser.com/questions/249436/file-extension-for-markdown-files
"""
return path.endswith(markdown_extensions)
_ERROR_TEMPLATE_RE = re.compile(r'^\d{3}\.html?$')
def is_error_template(path: str) -> bool:
"""Return True if the given file path is an HTTP error template."""
return bool(_ERROR_TEMPLATE_RE.match(path))
@functools.lru_cache(maxsize=None)
def _norm_parts(path: str) -> list[str]:
if not path.startswith('/'):
path = '/' + path
path = posixpath.normpath(path)[1:]
return path.split('/') if path else []
def get_relative_url(url: str, other: str) -> str:
"""
Return given url relative to other.
Both are operated as slash-separated paths, similarly to the 'path' part of a URL.
The last component of `other` is skipped if it contains a dot (considered a file).
Actual URLs (with schemas etc.) aren't supported. The leading slash is ignored.
Paths are normalized ('..' works as parent directory), but going higher than the
root has no effect ('foo/../../bar' ends up just as 'bar').
"""
# Remove filename from other url if it has one.
dirname, _, basename = other.rpartition('/')
if '.' in basename:
other = dirname
other_parts = _norm_parts(other)
dest_parts = _norm_parts(url)
common = 0
for a, b in zip(other_parts, dest_parts):
if a != b:
break
common += 1
rel_parts = ['..'] * (len(other_parts) - common) + dest_parts[common:]
relurl = '/'.join(rel_parts) or '.'
return relurl + '/' if url.endswith('/') else relurl
def normalize_url(path: str, page: Page | None = None, base: str = '') -> str:
"""Return a URL relative to the given page or using the base."""
path, relative_level = _get_norm_url(path)
if relative_level == -1:
return path
if page is not None:
result = get_relative_url(path, page.url)
if relative_level > 0:
result = '../' * relative_level + result
return result
return posixpath.join(base, path)
@functools.lru_cache(maxsize=None)
def _get_norm_url(path: str) -> tuple[str, int]:
if not path:
path = '.'
elif '\\' in path:
log.warning(
f"Path '{path}' uses OS-specific separator '\\'. "
f"That will be unsupported in a future release. Please change it to '/'."
)
path = path.replace('\\', '/')
# Allow links to be fully qualified URLs
parsed = urlsplit(path)
if parsed.scheme or parsed.netloc or path.startswith(('/', '#')):
return path, -1
# Relative path - preserve information about it
norm = posixpath.normpath(path) + '/'
relative_level = 0
while norm.startswith('../', relative_level * 3):
relative_level += 1
return path, relative_level
def create_media_urls(
path_list: Iterable[str], page: Page | None = None, base: str = ''
) -> list[str]:
"""Soft-deprecated, do not use."""
return [normalize_url(path, page, base) for path in path_list]
def path_to_url(path):
warnings.warn(
"path_to_url is never used in MkDocs and will be removed soon.", DeprecationWarning
)
return path.replace('\\', '/')
def get_theme_dir(name: str) -> str:
"""Return the directory of an installed theme by name."""
theme = get_themes()[name]
return os.path.dirname(os.path.abspath(theme.load().__file__))
@functools.lru_cache(maxsize=None)
def get_themes() -> dict[str, EntryPoint]:
"""Return a dict of all installed themes as {name: EntryPoint}."""
themes: dict[str, EntryPoint] = {}
eps: dict[EntryPoint, None] = dict.fromkeys(entry_points(group='mkdocs.themes'))
builtins = {ep.name for ep in eps if ep.dist is not None and ep.dist.name == 'mkdocs'}
for theme in eps:
assert theme.dist is not None
if theme.name in builtins and theme.dist.name != 'mkdocs':
raise exceptions.ConfigurationError(
f"The theme '{theme.name}' is a builtin theme but the package '{theme.dist.name}' "
"attempts to provide a theme with the same name."
)
elif theme.name in themes:
other_dist = themes[theme.name].dist
assert other_dist is not None
log.warning(
f"A theme named '{theme.name}' is provided by the Python packages '{theme.dist.name}' "
f"and '{other_dist.name}'. The one in '{theme.dist.name}' will be used."
)
themes[theme.name] = theme
return themes
def get_theme_names() -> Collection[str]:
"""Return a list of all installed themes by name."""
return get_themes().keys()
def dirname_to_title(dirname: str) -> str:
"""Return a page tile obtained from a directory name."""
title = dirname
title = title.replace('-', ' ').replace('_', ' ')
# Capitalize if the dirname was all lowercase, otherwise leave it as-is.
if title.lower() == title:
title = title.capitalize()
return title
def get_markdown_title(markdown_src: str) -> str | None:
"""Soft-deprecated, do not use."""
lines = markdown_src.replace('\r\n', '\n').replace('\r', '\n').split('\n')
while lines:
line = lines.pop(0).strip()
if not line.strip():
continue
if not line.startswith('# '):
return None
return line.lstrip('# ')
return None
def find_or_create_node(branch, key):
"""
Given a list, look for dictionary with a key matching key and return it's
value. If it doesn't exist, create it with the value of an empty list and
return that.
"""
for node in branch:
if not isinstance(node, dict):
continue
if key in node:
return node[key]
new_branch = []
node = {key: new_branch}
branch.append(node)
return new_branch
def nest_paths(paths):
"""
Given a list of paths, convert them into a nested structure that will match
the pages config.
"""
nested = []
for path in paths:
parts = PurePath(path).parent.parts
branch = nested
for part in parts:
part = dirname_to_title(part)
branch = find_or_create_node(branch, part)
branch.append(path)
return nested
class DuplicateFilter:
"""Avoid logging duplicate messages."""
def __init__(self) -> None:
self.msgs: set[str] = set()
def __call__(self, record: logging.LogRecord) -> bool:
rv = record.msg not in self.msgs
self.msgs.add(record.msg)
return rv
class CountHandler(logging.NullHandler):
"""Counts all logged messages >= level."""
def __init__(self, **kwargs) -> None:
self.counts: dict[int, int] = defaultdict(int)
super().__init__(**kwargs)
def handle(self, record):
rv = self.filter(record)
if rv:
# Use levelno for keys so they can be sorted later
self.counts[record.levelno] += 1
return rv
def get_counts(self) -> list[tuple[str, int]]:
return [(logging.getLevelName(k), v) for k, v in sorted(self.counts.items(), reverse=True)]
class weak_property:
"""Same as a read-only property, but allows overwriting the field for good."""
def __init__(self, func):
self.func = func
self.__doc__ = func.__doc__
def __get__(self, instance, owner=None):
if instance is None:
return self
return self.func(instance)
def __getattr__(name: str):
if name == 'warning_filter':
warnings.warn(
"warning_filter doesn't do anything since MkDocs 1.2 and will be removed soon. "
"All messages on the `mkdocs` logger get counted automatically.",
DeprecationWarning,
)
return logging.Filter()
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|