File: normalize.py

package info (click to toggle)
markdown-it-py 3.0.0-3
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 1,780 kB
  • sloc: python: 5,214; xml: 39; sh: 29; makefile: 24
file content (18 lines) | stat: -rw-r--r-- 402 bytes parent folder | download | duplicates (8) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
 
"""Normalize input string."""
import re

from .state_core import StateCore

# https://spec.commonmark.org/0.29/#line-ending
NEWLINES_RE = re.compile(r"\r\n?|\n")
NULL_RE = re.compile(r"\0")


def normalize(state: StateCore) -> None:
    # Normalize newlines
    string = NEWLINES_RE.sub("\n", state.src)

    # Replace NULL characters
    string = NULL_RE.sub("\uFFFD", string)

    state.src = string