ó ç'ÊUc@sÜddlZddlZddlZddlmZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZddlmZdd lmZd efd „ƒYZdS( iÿÿÿÿN(t LogCapture(tdefer(tTestCase(t get_crawler(tmock(tFollowAllSpidert DelaySpidert SimpleSpidertBrokenStartRequestsSpidertSingleRequestSpidertDuplicateStartRequestsSpider(t MockServer(tRequestt CrawlTestCasecBsUeZd„Zd„Zejd„ƒZejd„ƒZejd„ƒZejd„ƒZ ejd„ƒZ ejd„ƒZ ejd„ƒZ ejd „ƒZ ejd „ƒZejd „ƒZejd „ƒZejd „ƒZejd„ƒZejd„ƒZejd„ƒZd„Zejd„ƒZejd„ƒZRS(cCstƒ|_|jjƒdS(N(R t mockservert __enter__(tself((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pytsetUps cCs|jjdddƒdS(N(Rt__exit__tNone(R((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttearDownsccs7ttƒ}|jƒV|jt|jjƒdƒdS(Ni (RRtcrawlt assertEqualtlentspidert urls_visited(Rtcrawler((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_follow_alls  ccsH|jdtƒV|jdtƒV|jdtƒV|jdtƒVdS(Ngš™™™™™É?i(t _test_delaytFalsetTrue(R((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyt test_delay sc cs¢i|d6|d6}tt|ƒ}|jd|dƒV|jj}|d|d}|t|ƒd}|rvdnd }|j||d|kd |ƒdS( NtDOWNLOAD_DELAYtRANDOMIZE_DOWNLOAD_DELAYt maxlatencyiiÿÿÿÿiig333333ã?gš™™™™™É?sdownload delay too small: %s(RRRRttimesRt assertTrue( Rtdelayt randomizetsettingsRttt totaltimetavgdt tolerance((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyR)s ccsrttƒ}|jddƒV|j|jjdkƒ|j|jjdkƒ|j|jj|jjkƒdS(Ntngà?i(RRRR$Rtt1tt2(RR((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_timeout_success5s  ccsttidd6ƒ}|jddƒV|j|jjdkƒ|j|jjdkƒ|j|jjdkƒ|j|jj|jjkƒ|jddddƒV|j|jjdkƒ|j|jjdkƒ|j|jjdkƒ|j|jj|jjkƒdS(NgffffffÖ?tDOWNLOAD_TIMEOUTR,gà?itbi(RRRR$RR-R.tt2_err(RR((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_timeout_failure=sccs=ttƒ}tƒ}|jdƒVWdQX|j|ƒdS(Ns"http://localhost:8998/status?n=503(RRRRt_assert_retried(RRtl((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_retry_503Ls  ccs=ttƒ}tƒ}|jdƒVWdQX|j|ƒdS(Ns#http://localhost:65432/status?n=503(RRRRR4(RRR5((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_retry_conn_failedSs  c csetjddtjddƒƒ>ttƒ}tƒ}|jdƒVWdQX|j|ƒWdQXdS(Nssocket.gethostbynamet side_effectiûÿÿÿs#No address associated with hostnameshttp://example.com/( RtpatchtsockettgaierrorRRRRR4(RRR5((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_retry_dns_errorZs    ccsŒtddtjƒ$}ttƒ}|jddƒVWdQX|jt|jƒdƒ|jd}|j |j ƒ|j |j dt ƒdS(Ntscrapytleveltfail_before_yieldii( RtloggingtERRORRRRRRtrecordstassertIsNotNonetexc_infotassertIstZeroDivisionError(RR5Rtrecord((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyt$test_start_requests_bug_before_yieldcs  ccsŒtddtjƒ$}ttƒ}|jddƒVWdQX|jt|jƒdƒ|jd}|j |j ƒ|j |j dt ƒdS(NR=R>t fail_yieldingii( RR@RARRRRRRBRCRDRERF(RR5RRG((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyt test_start_requests_bug_yieldingns  ccs+idd6}tt|ƒ}|jƒVdS(NitCONCURRENT_REQUESTS(RRR(RR'R((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_start_requests_lazynessys ccs†idd6}tt|ƒ}|jdtddddƒV|j|jjdƒ|jdtdddd ƒV|j|jjdƒdS( NiRKt dont_filtert distinct_urlsit dupe_factoriii(RR RRRRtvisitedR(RR'R((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_start_requests_dupes‚s  ccs{ddlm}|idd6ƒ}ttƒ}tƒ}|jdj|ƒƒVWdQX|jt|ƒj dƒdƒdS(Niÿÿÿÿ(t urlencodesÊHTTP/1.1 200 OK Server: Apache-Coyote/1.1 X-Powered-By: Servlet 2.4; JBoss-4.2.3.GA (build: SVNTag=JBoss_4_2_3_GA date=200807181417)/JBossWeb-2.0 Set-Cookie: JSESSIONID=08515F572832D0E659FD2B0D8031D75F; Path=/ Pragma: no-cache Expires: Thu, 01 Jan 1970 00:00:00 GMT Cache-Control: no-cache Cache-Control: no-store Content-Type: text/html;charset=UTF-8 Content-Language: en Date: Tue, 27 Aug 2013 13:05:05 GMT Connection: close foo body with multiples lines trawshttp://localhost:8998/raw?{0}sGot response 200i( turllibRRRRRRtformatRtstrtcount(RRRtqueryRR5((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_unbounded_responseŒs   ccs=ttƒ}tƒ}|jdƒVWdQX|j|ƒdS(Ns"http://localhost:8998/drop?abort=0(RRRRR4(RRR5((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_retry_conn_lost§s  ccs=ttƒ}tƒ}|jdƒVWdQX|j|ƒdS(Ns"http://localhost:8998/drop?abort=1(RRRRR4(RRR5((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyttest_retry_conn_aborted¯s  cCsB|jt|ƒjdƒdƒ|jt|ƒjdƒdƒdS(NtRetryingisGave up retryingi(RRVRW(Rtlog((s4/home/travis/build/scrapy/scrapy/tests/test_crawl.pyR4·sc cs¯tdddƒ}|jƒ}|jdid d6ƒ}|jdidd6ƒ}||jd<||jd<||jds   .