ó ç'ÊUc@@sÛddlmZddlZddlmZmZddlmZddlm Z ddl m Z ddl m Z ddlmZmZdd lmZmZdd lmZdd lmZd e jfd „ƒYZdS(i(tabsolute_importN(treactorterror(tDeferred(tfailure(tunittest(tRobotsTxtMiddleware(t IgnoreRequestt NotConfigured(tRequesttResponse(tSettings(tmocktRobotsTxtMiddlewareTestcB@sGeZd„Zd„Zd„Zd„Zd„Zd„Zd„ZRS(c@saˆjƒ‰ˆjtdƒˆƒ‡‡fd†}tƒ}|j|ƒtj|jdƒ|S(Nshttp://site.localc@sFˆjtdƒˆƒˆjtdƒˆƒˆjtdƒˆƒdS(Nshttp://site.local/allowedshttp://site.local/admin/mainshttp://site.local/static/(tassertNotIgnoredR t assertIgnored(tr(t middlewaretself(sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyttests( t_get_middlewareRR Rt addCallbackRtcallFromThreadtcallbacktNone(RRtdeferred((RRsM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyttest_robotstxts   c@switd6‰ˆjƒ‰ˆjtddˆƒˆƒ‡‡‡fd†}tƒ}|j|ƒtj|jdƒ|S(Ntdont_obey_robotstxtshttp://site.localtmetac@sXˆjtddˆƒˆƒˆjtddˆƒˆƒˆjtddˆƒˆƒdS(Nshttp://site.local/allowedRshttp://site.local/admin/mainshttp://site.local/static/(RR (R(RRR(sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyR$s( tTrueRRR RRRRRR(RRR((RRRsM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyttest_robotstxt_meta s    c@sÔtjƒ}tƒ|_|jjdtƒtjƒ|j_tj dƒ‰‡fd†}||jj_ t |ƒ‰tjƒˆ_ ˆj tdƒdƒtƒ}|j‡‡fd†ƒtj|jdƒ|S(NtROBOTSTXT_OBEYsRobotstxt address not foundc@s)tƒ}tj|jtjˆƒƒ|S(N(RRRterrbackRtFailure(trequesttspiderR(terr(sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pytreturn_failure3s shttp://site.localc@sˆjˆjjƒƒS(N(t assertIsNonet _logerrortassert_any_call(t_(RR(sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyt=s(R t MagicMockR tsettingstsetRtenginetdownloadRtDNSLookupErrort side_effectRR'tprocess_requestR RRt addErrbackRRR(RtcrawlerR%R((R$RRsM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyttest_robotstxt_error-s    cC@s#d}|j|j||ƒƒdS(N(RR&R2(RR"RR#((sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyRAscC@s#d}|jt|j||ƒdS(N(Rt assertRaisesRR2(RR"RR#((sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyREsc@s¬tjƒ}tƒ|_|jjddƒ|jtt|ƒ|jjdtƒtjƒ|j _ t j dddƒ}t dd|ƒ‰‡fd †}||j j _|S( Nt USER_AGENTt CustomAgentRs^\s+(?m)tsT User-Agent: * Disallow: /admin/ Disallow: /static/ shttp://site.local/robots.txttbodyc@s tƒ}tj|jˆƒ|S(N(RRRR(R"R#R(tresponse(sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pytreturn_responseVs (R R+R R,R-R6RRRR.R/tretsubR R1(RR4tROBOTSR<((R;sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyt _get_crawlerIs    cC@s|jƒ}t|ƒS(N(R@R(RR4((sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyR]s ( t__name__t __module__RRR5RRR@R(((sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyR s     (t __future__RR=ttwisted.internetRRttwisted.internet.deferRttwisted.pythonRt twisted.trialRt&scrapy.downloadermiddlewares.robotstxtRtscrapy.exceptionsRRt scrapy.httpR R tscrapy.settingsR ttestsR tTestCaseR (((sM/home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_robotstxt.pyts