ó
ç'ÊUc @ sí d d l Z d d l Z d d l Z d d l m Z d d l m Z d d l m Z d d l m
Z
d d l m Z d d l
m Z m Z m Z m Z m Z d d l m Z d d l m Z m Z m Z m Z m Z m Z m Z d d
l m Z d d l m Z d d l! m" Z" d d
l# m$ Z$ d d l% m& Z& d e j' f d „ ƒ YZ( d e( f d „ ƒ YZ) d e( f d „ ƒ YZ* d e( f d „ ƒ YZ+ d e( f d „ ƒ YZ, d e( f d „ ƒ YZ- d e j' f d „ ƒ YZ. e/ d k rée j0 ƒ n d S( iÿÿÿÿN( t BytesIO( t
LogCapture( t unittest( t signals( t Settings( t Requestt Responset TextResponset XmlResponset HtmlResponse( t
InitSpider( t Spidert
BaseSpidert CrawlSpidert Rulet
XMLFeedSpidert
CSVFeedSpidert
SitemapSpider( t
LinkExtractor( t ScrapyDeprecationWarning( t
object_ref( t get_crawler( t mockt
SpiderTestc B sƒ e Z e Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z
d „ Z d „ Z d „ Z
d
„ Z d „ Z d „ Z RS(
c C s t j d ƒ d S( Nt always( t warningst simplefilter( t self( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt setUp s c C s t j ƒ d S( N( R t
resetwarnings( R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt tearDown s c C s9 | j d ƒ } | j | j d ƒ | j | j g ƒ d S( Ns example.com( t spider_classt assertEqualt namet
start_urls( R t spider( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_base_spider! s c C sK | j d ƒ } | j ƒ } | j t j | ƒ ƒ | j t | ƒ g ƒ d S( Ns example.com( R t start_requestst
assertTruet inspectt isgeneratorR t list( R R# R% ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_start_requests& s c C s, | j d d d ƒ} | j | j d ƒ d S( s7 Constructor arguments are assigned to spider attributess example.comt foot barN( R R R+ ( R R# ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_spider_args, s c C s0 | j t | j ƒ | j t | j d d ƒd S( s7 Constructor arguments are assigned to spider attributest someargR+ N( t assertRaisest
ValueErrorR ( R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_spider_without_name1 s c C s¶ | j d ƒ } t ƒ } t j d t ƒ † } | j | ƒ | j d t | d j ƒ ƒ | j t
| d ƒ ƒ | j | j | ƒ | j t
| d ƒ ƒ | j | j
| j
ƒ Wd QXd S( Ns example.comt recordt set_crawleri t crawlert settings( R R R t catch_warningst TrueR3 t assertInt strt messageR& t hasattrt assertIsR4 R5 ( R R# R4 t w( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt" test_deprecated_set_crawler_method6 s
c C sw t ƒ } | j j | d ƒ } | j t | d ƒ ƒ | j | j | ƒ | j t | d ƒ ƒ | j | j | j ƒ d S( Ns example.comR4 R5 ( R R t from_crawlerR& R; R< R4 R5 ( R R4 R# ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt1 test_from_crawler_crawler_and_settings_populationA s c C sZ t j j | j d d d ƒ6 } | j j t ƒ d d d ƒ| j d d d ƒWd QXd S( Nt __init__t return_values example.comR+ R, ( R t patcht objectR t NoneR? R t assert_called_once_with( R t mock_init( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_from_crawler_init_callI s
c C s† d | j f d „ ƒ Y} t ƒ } | j | d ƒ } | j j d t j d | ƒ | j j d t j d | d d ƒ | j | j ƒ d S( Nt
TestSpiderc B s e Z e Z d „ Z RS( c S s
t | _ d S( N( R7 t
closed_called( R t reason( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt closedT s ( t __name__t
__module__t FalseRJ RL ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRI Q s s example.comt signalR# RK (
R R R? R t send_catch_logt
spider_openedt
spider_closedRE R& RJ ( R RI R4 R# ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_closed_signal_callP s
c C s¥ i d d 6d d 6} i d d 6d d 6} | | j _ t | d d ƒ} | j j | ƒ | j | j d ƒ d ƒ | j | j d ƒ d ƒ | j | j d ƒ d ƒ d S( NR# t TEST1t TEST2t projectt TEST3t priority( R t custom_settingsR t update_settingsR t get( R t spider_settingst project_settingsR5 ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_update_settings_ s c C su | j d ƒ } t ƒ } | j j d ƒ Wd QX| j d ƒ | j d } | j d | j ƒ | j | j | ƒ d S( Ns example.coms test log msgt INFOi R# ( s example.coms INFOs test log msg(
R R t loggert infot checkt recordsR8 t __dict__R< R# ( R R# t lR2 ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_loggerj s
c C sN | j d ƒ } t j d ƒ } | j d d ƒ Wd QX| j j d d ƒ d S( Ns example.coms scrapy.spiders.Spider.loggers test log msgR` ( R R RC t logRF ( R R# t mock_logger( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_logt s ( RM RN R R R R R$ R* R- R1 R> R@ RH RT R_ Rg Rj ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR s
t InitSpiderTestc B s e Z e Z RS( ( RM RN R
R ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRk { s t XMLFeedSpiderTestc B s e Z e Z d „ Z RS( c C sì d } t d d d | ƒ } d | j f d „ ƒ Y} x± d D]© } | d d
| ƒ} t | j | ƒ ƒ } | j t | ƒ d | ƒ | j | i d g d
6d g d 6d g d 6d g d 6i g d
6d g d 6d g d 6g d 6g | ƒ q; Wd S( Ns¸
http://www.example.com/Special-Offers.html2009-08-16
http://www.example.com/2009-08-16
t urls http://example.com/sitemap.xmlt bodyt
_XMLSpiderc B s# e Z d Z d d f Z d „ Z RS( Rm t as* http://www.google.com/schemas/sitemap/0.84t bs) http://www.example.com/schemas/extras/1.0c s sa i | j d ƒ j ƒ d 6| j d ƒ j ƒ d 6| j d ƒ j ƒ d 6| j d ƒ j ƒ d 6Vd S( Ns a:loc/text()t locs b:updated/text()t updateds other/@valuet others other/@b:customt custom( t xpatht extract( R t responset selector( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt
parse_node” s
( Rp s* http://www.google.com/schemas/sitemap/0.84( Rq s) http://www.example.com/schemas/extras/1.0( RM RN t itertagt
namespacesRz ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRo s t iternodest xmlt examplet iteratori u* http://www.example.com/Special-Offers.htmlRr u
2009-08-16Rs u fuuRu u barRt u foo( s iternodess xml( R R R) t parseR t len( R Rn Rx Ro R€ R# t output( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_register_namespace„ s"
( RM RN R R R„ ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRl € s t CSVFeedSpiderTestc B s e Z e Z RS( ( RM RN R R ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR… ¬ s t CrawlSpiderTestc B sA e Z d Z e Z d „ Z d „ Z d „ Z d „ Z d „ Z RS( s Page title<title>
<body>
<p><a href="item/12.html">Item 12</a></p>
<div class='links'>
<p><a href="/about.html">About us</a></p>
</div>
<div>
<p><a href="/nofollow.html">This shouldn't be followed</a></p>
</div>
</body></html>c C s´ t d d | j ƒ} d | j f d „ ƒ Y} | ƒ } t | j | ƒ ƒ } | j t | ƒ d ƒ | j t t d „ | ƒ ƒ ƒ | j
g | D] } | j ^ qŽ d d d g ƒ d S(
Ns& http://example.org/somepage/index.htmlRn t _CrawlSpiderc B s8 e Z d Z d g Z e e ƒ d d ƒf Z d „ Z RS( t tests example.orgt
process_linkst dummy_process_linksc S s | S( N( ( R t links( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRŠ Ë s ( RM RN R! t allowed_domainsR R t rulesRŠ ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR‡ Ä s i c S s
t | t ƒ S( N( t
isinstanceR ( t r( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt <lambda>Ñ s s( http://example.org/somepage/item/12.htmls http://example.org/about.htmls http://example.org/nofollow.html( R t test_bodyR R) t _requests_to_followR R‚ R& t allt mapt assertEqualsRm ( R Rx R‡ R# Rƒ R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_process_links¿ s
c C s± t d d | j ƒ} d | j f d „ ƒ Y} | ƒ } t | j | ƒ ƒ } | j t | ƒ d ƒ | j t t d „ | ƒ ƒ ƒ | j
g | D] } | j ^ qŽ d d g ƒ d S( Ns& http://example.org/somepage/index.htmlRn R‡ c B sS e Z d d l Z d Z d g Z e e ƒ d d ƒf Z e j d ƒ Z d „ Z
RS( iÿÿÿÿNRˆ s example.orgR‰ t filter_process_linkst nofollowc S s, g | D]! } | j j | j ƒ s | ^ q S( N( t _test_regext searchRm ( R R‹ t link( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR— å s
( RM RN t reR! RŒ R R R t compileR™ R— ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR‡ Ü s i c S s
t | t ƒ S( N( RŽ R ( R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR ì s s( http://example.org/somepage/item/12.htmls http://example.org/about.html( R R‘ R R) R’ R R‚ R& R“ R” R• Rm ( R Rx R‡ R# Rƒ R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_process_links_filter× s
c C s´ t d d | j ƒ} d | j f d „ ƒ Y} | ƒ } t | j | ƒ ƒ } | j t | ƒ d ƒ | j t t d „ | ƒ ƒ ƒ | j
g | D] } | j ^ qŽ d d d g ƒ d S(
Ns& http://example.org/somepage/index.htmlRn R‡ c B s8 e Z d Z d g Z e e ƒ d d ƒf Z d „ Z RS( Rˆ s example.orgR‰ RŠ c s s x | D] } | Vq Wd S( N( ( R R‹ R› ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRŠ ý s
( RM RN R! RŒ R R R RŠ ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR‡ ö s i c S s
t | t ƒ S( N( RŽ R ( R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR s s( http://example.org/somepage/item/12.htmls http://example.org/about.htmls http://example.org/nofollow.html( R R‘ R R) R’ R R‚ R& R“ R” R• Rm ( R Rx R‡ R# Rƒ R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_process_links_generatorñ s c C sŸ t ƒ } | j j | d ƒ } | j t | d ƒ ƒ | j | j ƒ i t d 6} t d | ƒ } | j j | d ƒ } | j t | d ƒ ƒ | j | j ƒ d S( Ns example.comt
_follow_linkst CRAWLSPIDER_FOLLOW_LINKSt
settings_dict( R R R? R& R; R RO t assertFalse( R R4 R# R¢ ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt& test_follow_links_attribute_population
s
c C s· | j d ƒ } | j t | d ƒ ƒ | j t ƒ ƒ | j t | d ƒ ƒ | j | j ƒ | j d ƒ } i t d 6} | j t d | ƒ ƒ | j t | d ƒ ƒ | j | j ƒ d S( Ns example.comR R¡ R¢ ( R R£ R; R3 R R& R RO ( R R# R¢ ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt1 test_follow_links_attribute_deprecated_population s
(
RM RN R‘ R
R R– Rž RŸ R¤ R¥ ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR† ± s t SitemapSpiderTestc B sa e Z e Z d Z e ƒ Z e j d e d d ƒ Z e j
e ƒ e j ƒ e j ƒ Z
d „ Z RS( t SITEMAPt fileobjt modes w+bc C sR | j d ƒ } t d d d | j ƒ } | j | j | ƒ | j ƒ t d d d | j ƒ } | j | j | ƒ d ƒ t d d d | j ƒ } | j | j | ƒ d ƒ t d d d | j d i d d 6ƒ } | j | j | ƒ | j ƒ t d d
d | j ƒ } | j | j | ƒ | j ƒ t d d d | j ƒ } | j | j | ƒ | j ƒ d S( Ns example.comRm s http://www.example.com/Rn s" http://www.example.com/favicon.icos http://www.example.com/sitemapt headerss application/gzips content-types" http://www.example.com/sitemap.xmls% http://www.example.com/sitemap.xml.gz(
R R t BODYR t _get_sitemap_bodyR RE R t GZBODYR ( R R# R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_get_sitemap_body0 s % ( RM RN R R R« R t ft gzipt GzipFilet gt writet closet getvalueR R® ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR¦ % s
t BaseSpiderDeprecationTestc B s, e Z d „ Z d „ Z d „ Z d „ Z RS( c C s† t j d t ƒ n } d t f d „ ƒ Y} | j t | ƒ d ƒ | j | d j t ƒ | j | d j t j
| ƒ d ƒ Wd QXd S( NR2 t MySpider1c B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR· K s i i ( R R6 R7 R R R‚ t categoryR t linenoR' t getsourcelines( R R= R· ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_basespider_is_deprecatedH s
c C s² d t f d „ ƒ Y} d | f d „ ƒ Y} d t f d „ ƒ Y} d t f d „ ƒ Y} t | t ƒ sm t ‚ t | t ƒ s‚ t ‚ t | t ƒ s˜ t ‚ t | t ƒ s® t ‚ d S( Nt MySpider2c B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR¼ S s t
MySpider2ac B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR½ V s t Fooc B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR¾ Y s t Foo2c B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR¿ \ s ( R RD R t
issubclassR t AssertionError( R R¼ R½ R¾ R¿ ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_basespider_issubclassR s c C s¾ d t f d „ ƒ Y} d | f d „ ƒ Y} d t f d „ ƒ Y} d t f d „ ƒ Y} t | ƒ t ƒ sp t ‚ t | ƒ t ƒ sˆ t ‚ t | ƒ t ƒ s¡ t ‚ t | ƒ t ƒ sº t ‚ d S( Nt MySpider3c B s e Z d Z RS( t myspider3( RM RN R! ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRÃ e s t
MySpider3ac B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyRÅ h s R¾ c B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR¾ k s R¿ c B s e Z RS( ( RM RN ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR¿ n s ( R RD R RŽ R RÁ ( R RÃ RÅ R¾ R¿ ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_basespider_isinstanced s c C sj t t t ƒ s t ‚ t t t ƒ s* t ‚ t t d d ƒ t ƒ sH t ‚ t t d d ƒ t ƒ sf t ‚ d S( NR! R+ ( RÀ R
R RÁ R RŽ ( R ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt test_crawl_spiderv s ( RM RN R» RÂ RÆ RÇ ( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyR¶ F s
t __main__(1 R° R' R t ioR t testfixturesR t
twisted.trialR t scrapyR t scrapy.settingsR t scrapy.httpR R R R R t scrapy.spiders.initR
t scrapy.spidersR R R
R R R R t scrapy.linkextractorsR t scrapy.exceptionsR t scrapy.utils.trackrefR t scrapy.utils.testR t testsR t TestCaseR Rk Rl R… R† R¦ R¶ RM t main( ( ( s5 /home/travis/build/scrapy/scrapy/tests/test_spider.pyt <module> s0 (4d,t!7