import re import unittest import weakref import libxml2 from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector from scrapy.selector.document import Libxml2Document from scrapy.utils.test import libxml2debug class XPathSelectorTestCase(unittest.TestCase): @libxml2debug def test_selector_simple(self): """Simple selector tests""" body = "

" response = TextResponse(url="http://example.com", body=body) xpath = HtmlXPathSelector(response) xl = xpath.select('//input') self.assertEqual(2, len(xl)) for x in xl: assert isinstance(x, HtmlXPathSelector) self.assertEqual(xpath.select('//input').extract(), [x.extract() for x in xpath.select('//input')]) self.assertEqual([x.extract() for x in xpath.select("//input[@name='a']/@name")], [u'a']) self.assertEqual([x.extract() for x in xpath.select("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) self.assertEqual(xpath.select("concat('xpath', 'rules')").extract(), [u'xpathrules']) self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) @libxml2debug def test_selector_same_type(self): """Test XPathSelector returning the same type in x() method""" text = '

test

' assert isinstance(XmlXPathSelector(text=text).select("//p")[0], XmlXPathSelector) assert isinstance(HtmlXPathSelector(text=text).select("//p")[0], HtmlXPathSelector) @libxml2debug def test_selector_xml_html(self): """Test that XML and HTML XPathSelector's behave differently""" # some text which is parsed differently by XML and HTML flavors text = '

Hello

' self.assertEqual(XmlXPathSelector(text=text).select("//div").extract(), [u'

Hello

']) self.assertEqual(HtmlXPathSelector(text=text).select("//div").extract(), [u'

Hello

']) @libxml2debug def test_selector_nested(self): """Nested selector tests""" body = """
""" response = HtmlResponse(url="http://example.com", body=body) x = HtmlXPathSelector(response) divtwo = x.select('//div[@class="two"]') self.assertEqual(divtwo.select("//li").extract(), ["
  • one
    • ", "
  • two
    • ", "
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) self.assertEqual(divtwo.select("./ul/li").extract(), ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) self.assertEqual(divtwo.select(".//li").extract(), ["
  • four
    • ", "
  • five
    • ", "
  • six
    • "]) self.assertEqual(divtwo.select("./li").extract(), []) @libxml2debug def test_selector_re(self): body = """
    Name: Mary Age: 20
    """ response = HtmlResponse(url="http://example.com", body=body) x = HtmlXPathSelector(response) name_re = re.compile("Name: (\w+)") self.assertEqual(x.select("//ul/li").re(name_re), ["John", "Paul"]) self.assertEqual(x.select("//ul/li").re("Age: (\d+)"), ["10", "20"]) @libxml2debug def test_selector_over_text(self): hxs = HtmlXPathSelector(text='lala') self.assertEqual(hxs.extract(), u'lala') xxs = XmlXPathSelector(text='lala') self.assertEqual(xxs.extract(), u'lala') xxs = XmlXPathSelector(text='lala') self.assertEqual(xxs.select('.').extract(), [u'lala']) @libxml2debug def test_selector_namespaces_simple(self): body = """ found """ response = XmlResponse(url="http://example.com", body=body) x = XmlXPathSelector(response) x.register_namespace("somens", "http://scrapy.org") self.assertEqual(x.select("//somens:a").extract(), ['']) @libxml2debug def test_selector_namespaces_multiple(self): body = """ hello value 90Dried Rose """ response = XmlResponse(url="http://example.com", body=body) x = XmlXPathSelector(response) x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") x.register_namespace("p", "http://www.scrapy.org/product") x.register_namespace("b", "http://somens.com") self.assertEqual(len(x.select("//xmlns:TestTag")), 1) self.assertEqual(x.select("//b:Operation/text()").extract()[0], 'hello') self.assertEqual(x.select("//xmlns:TestTag/@b:att").extract()[0], 'value') self.assertEqual(x.select("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') self.assertEqual(x.select("//p:SecondTestTag").select("./xmlns:price/text()")[0].extract(), '90') self.assertEqual(x.select("//p:SecondTestTag/xmlns:material").extract()[0], '') @libxml2debug def test_selector_invalid_xpath(self): response = XmlResponse(url="http://example.com", body="") x = HtmlXPathSelector(response) xpath = "//test[@foo='bar]" try: x.select(xpath) except ValueError, e: assert xpath in str(e), "Exception message does not contain invalid xpath" except Exception: raise AssertionError("A invalid XPath does not raise ValueError") else: raise AssertionError("A invalid XPath does not raise an exception") @libxml2debug def test_http_header_encoding_precedence(self): # u'\xa3' = pound symbol in unicode # u'\xc2\xa3' = pound symbol in utf-8 # u'\xa3' = pound symbol in latin-1 (iso-8859-1) meta = u'' head = u'' + meta + u'' body_content = u'\xa3' body = u'' + body_content + u'' html = u'' + head + body + u'' encoding = 'utf-8' html_utf8 = html.encode(encoding) headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = HtmlXPathSelector(response) self.assertEquals(x.select("//span[@id='blank']/text()").extract(), [u'\xa3']) @libxml2debug def test_null_bytes(self): hxs = HtmlXPathSelector(text='la\x00la') self.assertEqual(hxs.extract(), u'lala') xxs = XmlXPathSelector(text='la\x00la') self.assertEqual(xxs.extract(), u'lala') @libxml2debug def test_unquote(self): xmldoc = '\n'.join(( '', ' lala', ' ', ' blabla&moreatestoh', ' PPPPppp&la]]>', ' ', ' pff', '')) xxs = XmlXPathSelector(text=xmldoc) self.assertEqual(xxs.extract_unquoted(), u'') self.assertEqual(xxs.select('/root').extract_unquoted(), [u'']) self.assertEqual(xxs.select('/root/text()').extract_unquoted(), [ u'\n lala\n ', u'\n pff\n']) self.assertEqual(xxs.select('//*').extract_unquoted(), [u'', u'', u'']) self.assertEqual(xxs.select('//text()').extract_unquoted(), [ u'\n lala\n ', u'\n blabla&more', u'a', u'test', u'oh\n ', u'lalalal&pppppPPPPppp&la', u'\n ', u'\n pff\n']) @libxml2debug def test_empty_bodies(self): r1 = TextResponse('http://www.example.com', body='') hxs = HtmlXPathSelector(r1) # shouldn't raise error xxs = XmlXPathSelector(r1) # shouldn't raise error @libxml2debug def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" for cls in [XPathSelector, HtmlXPathSelector, XmlXPathSelector]: x = cls() weakref.ref(x) assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ x.__class__.__name__ class Libxml2DocumentTest(unittest.TestCase): @libxml2debug def test_response_libxml2_caching(self): r1 = HtmlResponse('http://www.example.com', body='') r2 = r1.copy() doc1 = Libxml2Document(r1) doc2 = Libxml2Document(r1) doc3 = Libxml2Document(r2) # make sure it's cached assert doc1 is doc2 assert doc1.xmlDoc is doc2.xmlDoc assert doc1 is not doc3 assert doc1.xmlDoc is not doc3.xmlDoc # don't leave libxml2 documents in memory to avoid wrong libxml2 leaks reports del doc1, doc2, doc3 @libxml2debug def test_null_char(self): # make sure bodies with null char ('\x00') don't raise a TypeError exception self.body_content = 'test problematic \x00 body' response = TextResponse('http://example.com/catalog/product/blabla-123', headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) Libxml2Document(response) class Libxml2Test(unittest.TestCase): @libxml2debug def test_libxml2_bug_2_6_27(self): # this test will fail in version 2.6.27 but passes on 2.6.29+ html = "123" node = libxml2.htmlParseDoc(html, 'utf-8') result = [str(r) for r in node.xpathEval('//text()')] self.assertEquals(result, ['1', '2', '3']) node.freeDoc() if __name__ == "__main__": unittest.main()