from __future__ import annotations from pathlib import Path import lxml.html import pytest from html_text import ( DOUBLE_NEWLINE_TAGS, NEWLINE_TAGS, cleaned_selector, cleaner, etree_to_text, extract_text, parse_html, selector_to_text, ) ROOT = Path(__file__).parent @pytest.fixture( params=[ {"guess_punct_space": True, "guess_layout": False}, {"guess_punct_space": False, "guess_layout": False}, {"guess_punct_space": True, "guess_layout": True}, {"guess_punct_space": False, "guess_layout": True}, ] ) def all_options(request): return request.param def test_extract_no_text_html(all_options): html = ( '

' ) assert extract_text(html, **all_options) == "" def test_extract_text(all_options): html = "

Hello, world!" assert extract_text(html, **all_options) == "Hello, world!" def test_declared_encoding(all_options): html = ( '' "" "Hello, world!

" ) assert extract_text(html, **all_options) == "Hello, world!" def test_empty(all_options): assert extract_text("", **all_options) == "" assert extract_text(" ", **all_options) == "" assert extract_text(None, **all_options) == "" def test_comment(all_options): assert extract_text("", **all_options) == "" def test_comment_fragment(all_options): node = lxml.html.fragment_fromstring("") assert extract_text(node, **all_options) == "" def test_processing_instruction(all_options): assert extract_text('', **all_options) == "" def test_processing_instruction_fragment(all_options): node = lxml.html.fragment_fromstring('') assert extract_text(node, **all_options) == "" def test_extract_text_from_tree(all_options): html = "

Hello, world!" tree = parse_html(html) assert extract_text(tree, **all_options) == "Hello, world!" def test_extract_text_from_node(all_options): html = "

Hello, world!

" tree = parse_html(html) node = tree.xpath("//p")[0] assert extract_text(node, **all_options) == "Hello, world!" def test_inline_tags_whitespace(all_options): html = "fieldvalue of" assert extract_text(html, **all_options) == "field value of" def test_extract_text_from_fail_html(): html = "" tree = parse_html(html) node = tree.xpath("/html/frameset")[0] assert extract_text(node) == "" def test_punct_whitespace(): html = "
field, and more
" assert extract_text(html, guess_punct_space=False) == "field , and more" assert extract_text(html, guess_punct_space=True) == "field, and more" def test_punct_whitespace_preserved(): html = ( "
поле, and , " "more !now
a (boo)" ) text = extract_text(html, guess_punct_space=True, guess_layout=False) assert text == "по ле, and , more ! now a (boo)" @pytest.mark.xfail(reason="code punctuation should be handled differently") def test_bad_punct_whitespace(): html = ( "
trees "
        "= webstruct"
        ".load_trees"
        "("train/*.html""
        ")
" ) text = extract_text(html, guess_punct_space=False, guess_layout=False) assert text == 'trees = webstruct . load_trees ( "train/*.html" )' text = extract_text(html, guess_punct_space=True, guess_layout=False) assert text == 'trees = webstruct.load_trees("train/*.html")' def test_selectors(all_options): pytest.importorskip("parsel") html = ( 'textmore' "and more text and some more " ) # Selector sel = cleaned_selector(html) assert ( selector_to_text(sel, **all_options) == "text more and more text and some more" ) # SelectorList subsel = sel.xpath('//span[@id="extract-me"]') assert selector_to_text(subsel, **all_options) == "text more" subsel = sel.xpath("//a") assert selector_to_text(subsel, **all_options) == "more and some more" subsel = sel.xpath('//a[@id="extract-me"]') assert selector_to_text(subsel, **all_options) == "" subsel = sel.xpath("//foo") assert selector_to_text(subsel, **all_options) == "" def test_nbsp(): html = "

Foo Bar

" assert extract_text(html) == "Foo Bar" def test_guess_layout(): html = ( " title
text_1.

text_2 text_3

" '

' "

text_6text_7text_8

text_9
" '

...text_10

' ) text = ( "title text_1. text_2 text_3 text_4 text_5 text_6 text_7 " "text_8 text_9 ...text_10" ) assert extract_text(html, guess_punct_space=False, guess_layout=False) == text text = ( "title\n\ntext_1.\n\ntext_2 text_3\n\ntext_4\ntext_5" "\n\ntext_6 text_7 text_8\n\ntext_9\n\n...text_10" ) assert extract_text(html, guess_punct_space=False, guess_layout=True) == text text = ( "title text_1. text_2 text_3 text_4 text_5 text_6 text_7 " "text_8 text_9...text_10" ) assert extract_text(html, guess_punct_space=True, guess_layout=False) == text text = ( "title\n\ntext_1.\n\ntext_2 text_3\n\ntext_4\ntext_5\n\n" "text_6 text_7 text_8\n\ntext_9\n\n...text_10" ) assert extract_text(html, guess_punct_space=True, guess_layout=True) == text def test_basic_newline(): html = "
a
b
" assert extract_text(html, guess_punct_space=False, guess_layout=False) == "a b" assert extract_text(html, guess_punct_space=False, guess_layout=True) == "a\nb" assert extract_text(html, guess_punct_space=True, guess_layout=False) == "a b" assert extract_text(html, guess_punct_space=True, guess_layout=True) == "a\nb" def test_adjust_newline(): html = "
text 1

text 2

" assert extract_text(html, guess_layout=True) == "text 1\n\ntext 2" def test_personalize_newlines_sets(): html = ( "textmore" "and more text and some more " ) text = extract_text(html, guess_layout=True, newline_tags=NEWLINE_TAGS | {"a"}) assert text == "text\nmore\nand more text\nand some more" text = extract_text( html, guess_layout=True, double_newline_tags=DOUBLE_NEWLINE_TAGS | {"a"} ) assert text == "text\n\nmore\n\nand more text\n\nand some more" def _webpage_paths() -> list[tuple[Path, Path]]: webpages = sorted((ROOT / "test_webpages").glob("*.html")) extracted = sorted((ROOT / "test_webpages").glob("*.txt")) return list(zip(webpages, extracted)) @pytest.mark.parametrize(("page", "extracted"), _webpage_paths()) def test_webpages(page, extracted): html = page.read_text(encoding="utf-8") expected = extracted.read_text(encoding="utf-8") assert extract_text(html) == expected tree = cleaner.clean_html(parse_html(html)) assert etree_to_text(tree) == expected def test_deep_html(): """Make sure we don't crash due to recursion limit.""" # Build a deep tree manually as default parser would only allow # for 255 depth, but deeper trees are possible with other parsers n = 5000 parent = root = None for _ in range(n): el = lxml.html.Element("div") el.text = "foo" if parent is None: root = el parent = el else: parent.append(el) parent = el assert extract_text(root) == ("foo\n" * n).strip()