# -*- coding: utf-8 -*- import unittest import six from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, add_http_if_no_scheme, guess_scheme, parse_url) __doctests__ = ['scrapy.utils.url'] class UrlUtilsTest(unittest.TestCase): def test_url_is_from_any_domain(self): url = 'http://www.wheele-bin-art.co.uk/get/product/123' self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) url = 'http://wheele-bin-art.co.uk/get/product/123' self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) url = 'http://www.Wheele-Bin-Art.co.uk/get/product/123' self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.CO.UK'])) self.assertTrue(url_is_from_any_domain(url, ['WHEELE-BIN-ART.CO.UK'])) url = 'http://192.169.0.15:8080/mypage.html' self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) def test_url_is_from_spider(self): spider = Spider(name='example.com') self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', spider)) def test_url_is_from_spider_class_attributes(self): class MySpider(Spider): name = 'example.com' self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) def test_url_is_from_spider_with_allowed_domains(self): spider = Spider(name='example.com', allowed_domains=['example.org', 'example.net']) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) spider = Spider(name='example.com', allowed_domains=set(('example.com', 'example.net'))) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) spider = Spider(name='example.com', allowed_domains=('example.com', 'example.net')) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) def test_url_is_from_spider_with_allowed_domains_class_attributes(self): class MySpider(Spider): name = 'example.com' allowed_domains = ('example.org', 'example.net') self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) class AddHttpIfNoScheme(unittest.TestCase): def test_add_scheme(self): self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com') def test_without_subdomain(self): self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com') def test_path(self): self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'), 'http://www.example.com/some/page.html') def test_port(self): self.assertEqual(add_http_if_no_scheme('www.example.com:80'), 'http://www.example.com:80') def test_fragment(self): self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'), 'http://www.example.com/some/page#frag') def test_query(self): self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), 'http://www.example.com/do?a=1&b=2&c=3') def test_username_password(self): self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'), 'http://username:password@www.example.com') def test_complete_url(self): self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_http(self): self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com') def test_preserve_http_without_subdomain(self): self.assertEqual(add_http_if_no_scheme('http://example.com'), 'http://example.com') def test_preserve_http_path(self): self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'), 'http://www.example.com/some/page.html') def test_preserve_http_port(self): self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'), 'http://www.example.com:80') def test_preserve_http_fragment(self): self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'), 'http://www.example.com/some/page#frag') def test_preserve_http_query(self): self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), 'http://www.example.com/do?a=1&b=2&c=3') def test_preserve_http_username_password(self): self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'), 'http://username:password@www.example.com') def test_preserve_http_complete_url(self): self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_protocol_relative(self): self.assertEqual(add_http_if_no_scheme('//www.example.com'), 'http://www.example.com') def test_protocol_relative_without_subdomain(self): self.assertEqual(add_http_if_no_scheme('//example.com'), 'http://example.com') def test_protocol_relative_path(self): self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'), 'http://www.example.com/some/page.html') def test_protocol_relative_port(self): self.assertEqual(add_http_if_no_scheme('//www.example.com:80'), 'http://www.example.com:80') def test_protocol_relative_fragment(self): self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'), 'http://www.example.com/some/page#frag') def test_protocol_relative_query(self): self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), 'http://www.example.com/do?a=1&b=2&c=3') def test_protocol_relative_username_password(self): self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'), 'http://username:password@www.example.com') def test_protocol_relative_complete_url(self): self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_https(self): self.assertEqual(add_http_if_no_scheme('https://www.example.com'), 'https://www.example.com') def test_preserve_ftp(self): self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com') class GuessSchemeTest(unittest.TestCase): pass def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) assert url.startswith(args[1]), \ 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( args[0], url, args[1]) return do_expected def create_skipped_scheme_t(args): def do_expected(self): raise unittest.SkipTest(args[2]) url = guess_scheme(args[0]) assert url.startswith(args[1]) return do_expected for k, args in enumerate ([ ('/index', 'file://'), ('/index.html', 'file://'), ('./index.html', 'file://'), ('../index.html', 'file://'), ('../../index.html', 'file://'), ('./data/index.html', 'file://'), ('.hidden/data/index.html', 'file://'), ('/home/user/www/index.html', 'file://'), ('//home/user/www/index.html', 'file://'), ('file:///home/user/www/index.html', 'file://'), ('index.html', 'http://'), ('example.com', 'http://'), ('www.example.com', 'http://'), ('www.example.com/index.html', 'http://'), ('http://example.com', 'http://'), ('http://example.com/index.html', 'http://'), ('localhost', 'http://'), ('localhost/index.html', 'http://'), # some corner cases (default to http://) ('/', 'http://'), ('.../test', 'http://'), ], start=1): t_method = create_guess_scheme_t(args) t_method.__name__ = 'test_uri_%03d' % k setattr (GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation for k, args in enumerate ([ ('C:\absolute\path\to\a\file.html', 'file://', 'Windows filepath are not supported for scrapy shell'), ], start=1): t_method = create_skipped_scheme_t(args) t_method.__name__ = 'test_uri_skipped_%03d' % k setattr (GuessSchemeTest, t_method.__name__, t_method) if __name__ == "__main__": unittest.main()