#!/usr/bin/env python # -*- coding: utf-8 -*- import pytest from selectolax.parser import HTMLParser from selectolax.lexbor import LexborHTMLParser, SelectolaxError """ We'are testing only our own code. Many functionality are already tested in the Modest engine, so there is no reason to test every case. """ _PARSERS_PARAMETRIZER = ( "parser", (HTMLParser, LexborHTMLParser), ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_selector(parser): html = "

text

" selector = "p#p3" for node in parser(html).css(selector): assert node.text() == "text" assert node.tag == "p" assert node.parent.tag == "div" assert node.parent.next.tag == "p" assert node.parent.prev.tag == "span" assert node.parent.last_child.attributes["id"] == "p3" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_css_multiple_matches(parser): html = "
" assert len(parser(html).css("div")) == 3 @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_css_matches(parser): html = "
" assert parser(html).css_matches("div") @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_any_css_matches(parser): html = "
" assert parser(html).any_css_matches(("h1", "span")) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_css_one(parser): html = "

text

sd

" selector = ".s3" assert parser(html).css_first(selector) is None selector = "p.p3" assert parser(html).css_first(selector).text() == "text" with pytest.raises(ValueError): parser(html).css_first(selector, strict=True) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_text_when_html_is_empty(parser): html_parser = parser("") assert html_parser.text() == "" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_css_first_default(parser): html = "

text

sd

" selector = ".s3" assert parser(html).css_first(selector, default="lorem ipsum") == "lorem ipsum" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_id_property(parser): html = "

text

" assert parser(html).css_first("p").id == "main_text" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_tag_property(parser): html = "

text

" assert parser(html).css_first("h1").tag == "h1" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_attributes(parser): html = "

text

" selector = "p#p3" for node in parser(html).css(selector): assert "id" in node.attributes assert node.attributes["id"] == "p3" html = "

text

" selector = "p#p3" for node in HTMLParser(html).css(selector): assert "attr" in node.attributes assert node.attributes["attr"] is None @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_decompose(parser): html = "

text

" html_parser = parser(html) for node in html_parser.tags("p"): node.decompose() assert html_parser.body.child.html == "
" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_html_property(parser): html = "Hi there" html_parser = parser(html) assert html_parser.body.child.html == "Hi there" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_root_property(parser): html = "Hi there" html_parser = parser(html) assert html_parser.root.html == "Hi there" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_head_property(parser): html = """ rushter.com """ html_parser = parser(html) assert html_parser.head.html == "rushter.com" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_body_property(parser): html = "Hi there" html_parser = parser(html) assert html_parser.body.html == "Hi there" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_strip_tags(parser): html = "
" html_parser = parser(html) html_parser.root.strip_tags(["div", "script"]) assert html_parser.html == "" with pytest.raises(TypeError): html_parser.strip_tags(1) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_malformed_attributes(parser): html = '
' html_parser = parser(html) for tag in html_parser.tags("meta"): assert tag @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_iter_with_text(parser): html = """

Title

text
foo
""" html_parser = parser(html) expected_tags = ["-text", "h1", "-text", "div", "-text", "img", "-text"] actual_tags = [ node.tag for node in html_parser.css_first("#description").iter(include_text=True) ] assert expected_tags == actual_tags @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_text_deep_gh61(parser): html = """
this is a test

Heading

""" output = [] tree = parser(html) for node in tree.root.traverse(include_text=True): if node.tag == "-text": text = node.text(deep=True) if text: output.append(text) assert output == ["this is a test ", "Heading"] @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_iter_no_text(parser): html = """

Title

text
foo
""" html_parser = parser(html) expected_tags = ["h1", "div", "img"] actual_tags = [ node.tag for node in html_parser.css_first("#description").iter(include_text=False) ] assert expected_tags == actual_tags @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_navigation(parser): html = ( '

Title

' '
foo
' ) html_parser = parser(html) main_node = html_parser.css_first("#test_node") assert main_node.prev.id == "prev" assert main_node.next.id == "next" assert main_node.parent.id == "parent" assert main_node.child.id == "child" @pytest.mark.parametrize( "html,expected, parser", [ ("
", "my_node", HTMLParser), ("
", None, HTMLParser), ("
", "my_node", LexborHTMLParser), ("
", None, LexborHTMLParser), ], ) def test_get_node_id(html, expected, parser): html_parser = parser(html) node = html_parser.css_first("div") assert node.id == expected @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_html_attribute_works_for_text(parser): html = "
foo bar
" html_parser = parser(html) node = html_parser.css_first("div").child assert node.html == "foo bar" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_text_node_returns_text(parser): html = "
foo bar
" html_parser = parser(html) node = html_parser.css_first("div").child assert node.text(deep=False) == "foo bar" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_text_node_returns_text_parent(parser): html = "
foo bar
" html_parser = parser(html) node = html_parser.css_first("div") assert node.text(deep=False) == "foo bar" def test_text_node_returns_text_when_deep(): html = "
foo bar
" html_parser = HTMLParser(html) node = html_parser.css_first("div").child assert node.text(deep=True) == "foo bar" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_unwrap(parser): html = 'I linked to rushter.com' html_parser = parser(html) node = html_parser.css_first("i") node.unwrap() assert ( html_parser.body.child.html == 'I linked to rushter.com' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_unwrap_empty_tag(parser): html = 'I linked to rushter.com' html_parser = parser(html) node = html_parser.css_first("i") node.unwrap(delete_empty=True) assert ( html_parser.body.child.html == 'I linked to rushter.com' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_unwrap_tags(parser): html_parser = parser("
Hello world!
") html_parser.body.unwrap_tags(["i", "a"]) assert html_parser.body.html == "
Hello world!
" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_unwrap_empty_tags(parser): html_parser = parser("
Hello world!
") html_parser.body.unwrap_tags(["i", "a"], delete_empty=True) assert html_parser.body.html == "
Hello world!
" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_unwraps_multiple_child_nodes(parser): html = """
foo bar Lorems I dummy
text
""" html_parser = parser(html) html_parser.body.unwrap_tags(["span", "i"]) assert ( html_parser.body.child.html == '
\n foo bar Lorems I dummy
text
\n
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_unwraps_multiple_child_nodes_with_empty(parser): html = """
foo bar Lorems I dummy
text
""" html_parser = parser(html) html_parser.body.unwrap_tags(["span", "i"], delete_empty=True) assert ( html_parser.body.child.html == '
\n foo bar Lorems I dummy
text
\n
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_replace_with(parser): html_parser = parser('
Get Laptop
') img = html_parser.css_first("img") img.replace_with(img.attributes.get("alt", "")) assert html_parser.body.child.html == "
Get Laptop
" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_replace_with_multiple_nodes(parser): html_parser = parser( '
Get
/div>
' ) img = html_parser.css_first("span") img.replace_with(img.attributes.get("alt", "")) assert html_parser.body.child.html == "
Get Laptop
" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_replace_with(parser): html_parser = parser( '
Get
' ) html_parser2 = parser("
Test
") img_node = html_parser.css_first("img") img_node.replace_with(html_parser2.body.child) assert ( html_parser.body.child.html == '
Get
Test
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_replace_with_empty_string(parser): html_parser = parser('
Get Laptop
') img = html_parser.css_first("img") img.replace_with("") assert html_parser.body.child.html == "
Get
" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_replace_with_invalid_value_passed_exception(parser): with pytest.raises(TypeError) as excinfo: html_parser = parser('
Get Laptop
') img = html_parser.css_first("img") img.replace_with(None) assert "No matching signature found" in str(excinfo.value) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_insert_before(parser): html_parser = parser('
Get Laptop
') img = html_parser.css_first("img") img.insert_before(img.attributes.get("alt", "")) assert ( html_parser.body.child.html == '
Get LaptopLaptop
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_insert_before(parser): html_parser = parser( '
Get
' ) html_parser2 = parser("
Test
") img_node = html_parser.css_first("img") img_node.insert_before(html_parser2.body.child) assert ( html_parser.body.child.html == '
Get
Test
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_insert_after(parser): html_parser = parser('
Get Laptop
') img = html_parser.css_first("img") img.insert_after(img.attributes.get("alt", "")) assert ( html_parser.body.child.html == '
Get LaptopLaptop
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_insert_after(parser): html_parser = parser( '
Get
' ) html_parser2 = parser("
Test
") img_node = html_parser.css_first("img") img_node.insert_after(html_parser2.body.child) assert ( html_parser.body.child.html == '
Get
Test
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_insert_child(parser): html_parser = parser('
Get
') div = html_parser.css_first("div") div.insert_child("Laptop") assert html_parser.body.child.html == '
Get Laptop
' @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_insert_child(parser): html_parser = parser('
Get
Laptop
') html_parser2 = parser("
Test
") span_node = html_parser.css_first("span") span_node.insert_child(html_parser2.body.child) assert ( html_parser.body.child.html == '
Get
Laptop
Test
' ) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_attrs_adds_attribute(parser): html_parser = parser('
') node = html_parser.css_first("div") node.attrs["new_att"] = "new" assert node.attributes == {"id": "id", "new_att": "new"} @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_attrs_sets_attribute(parser): html_parser = parser('
') node = html_parser.css_first("div") node.attrs["id"] = "new_id" assert node.attributes == {"id": "new_id"} @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_attrs_removes_attribute(parser): html_parser = parser('
') node = html_parser.css_first("div") del node.attrs["id"] assert node.attributes == {} @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_attrs_test_dict_features(parser): html_parser = parser('
') node = html_parser.css_first("div") node.attrs["new_att"] = "new" assert list(node.attrs.keys()) == ["id", "v", "data-id", "new_att"] assert list(node.attrs.values()) == ["id", None, "foo", "new"] assert len(node.attrs) == 4 assert node.attrs.get("unknown_field", "default_value") == "default_value" assert "id" in node.attrs assert "vid" not in node.attrs @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_traverse(parser): html = ( '

Title

' '
foo
' ) html_parser = parser(html) actual = [node.tag for node in html_parser.root.traverse()] expected = ["html", "head", "body", "div", "div", "div", "h1", "div", "img", "div"] assert actual == expected @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_traverse_with_text(parser): html = ( '

Title

' '
foo
' ) html_parser = parser(html) actual = [node.tag for node in html_parser.root.traverse(include_text=True)] expected = [ "html", "head", "body", "div", "div", "div", "h1", "-text", "div", "-text", "img", "div", ] assert actual == expected @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_comparison(parser): html = """
H3ll0

Lorem ipsum dolor sit amet, ea quo modus meliore platonem.

""" html_parser = parser(html) nodes = [node for node in html_parser.root.traverse(include_text=False)] same_node_path_one = nodes[-1].parent same_node_path_two = nodes[-2] same_node_path_three = html_parser.css_first("#tt") assert same_node_path_one == same_node_path_two == same_node_path_three @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_comprassion_with_strings(parser): html = """
""" html_parser = parser(html) node = html_parser.css_first("#test") assert node == '
' @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_node_comparison_fails(parser): html = """
""" html_parser = parser(html) node = html_parser.css_first("#test") assert node is not None assert node != 123 def test_raw_value(): html_parser = HTMLParser("
<test>
") selector = html_parser.css_first("div") assert selector.child.raw_value == b"<test>" assert selector.child.html == "<test>" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_adavanced_selector(parser): html_parser = parser(""" """) selector = html_parser.select("script").text_contains("super_value") assert selector.any_matches @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_script_contain(parser): html_parser = parser(""" """) assert html_parser.scripts_contain("super_value") @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_hash_nodes(parser): tree = parser("""

John

Doe

""") node = tree.css_first("div") assert node.mem_id == hash(node) @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_srcs_contain(parser): html_parser = parser("""""") assert html_parser.script_srcs_contain(("analytics.js",)) @pytest.mark.parametrize("parser", (HTMLParser,)) def test_css_chaining(parser): html = """
""" tree = parser(html) assert len(tree.select("div").css("span").css(".red").matches) == 2 @pytest.mark.parametrize("parser", (HTMLParser,)) def test_css_chaining_two(parser): html = """ """ tree = parser(html) query = ( tree.select("script") .text_contains("var counter = ") .css(".weird_script") .attribute_longer_than("integrity", 25) ) assert query @pytest.mark.parametrize("parser", (HTMLParser, LexborHTMLParser)) def test_content_method(parser): html = """
SuperTest
""" tree = parser(html) assert tree.css_first("#main").child.text_content == "SuperTest" assert tree.css_first("#main").text_content is None @pytest.mark.parametrize("parser", (HTMLParser, LexborHTMLParser)) def test_merge_text_nodes(parser): html = """

John

Doe

""" tree = parser(html) tree.unwrap_tags(["strong"]) node = tree.css_first("div", strict=True) node.merge_text_nodes() assert node.html == "

John

Doe

" text = tree.text(deep=True, separator=" ", strip=True) assert text == "John Doe" @pytest.mark.parametrize("parser", (HTMLParser, LexborHTMLParser)) def test_merge_text_nodes_complex(parser): from textwrap import dedent html = dedent("""

Hello World

This is a test

Nested text nodes
with more nesting here
""").strip() tree = parser(html) tree.unwrap_tags(["em", "strong", "span", "b", "i"]) root = tree.css_first("article", strict=True) root.merge_text_nodes() assert root.css_first("h1").text() == "Hello World" assert root.css_first("p").text() == "This is a test" assert "Nested text nodes" in root.css_first("section").text() assert root.css_first("section > div").text() == "with more nesting here" @pytest.mark.parametrize("parser", (HTMLParser, LexborHTMLParser)) def test_merge_text_nodes_three_plus(parser): html = """
One Two Three
""" tree = parser(html) tree.unwrap_tags( ["em", "strong", "b", "i", "span", "u", "small", "big", "mark", "sub", "sup"] ) div = tree.css_first("div", strict=True) div.merge_text_nodes() assert div.text() == "One Two Three" @pytest.mark.parametrize(*_PARSERS_PARAMETRIZER) def test_css_first_first(parser): html = '

(1:1, 0:0, 0:0, 5:3)

' selector = "h2.list-details__item__partial" find_first = parser(html).css_first(selector) assert find_first.css_first(selector) is not None @pytest.mark.parametrize("parser", (LexborHTMLParser,)) def test_any_css_matches_fails(parser): html = """

Test

""" tree = parser(html) with pytest.raises(SelectolaxError): tree.any_css_matches(("##",))