ó
ç'ÊUc           @   s­  d  d l  Z  d  d l Z d  d l Z d  d l m Z d  d l m Z d  d l m Z d  d l	 m
 Z
 m Z m Z d  d l m Z m Z m Z m Z d  d l m Z m Z m Z m Z m Z d  d l m Z m Z d  d	 l m Z m Z m Z d  d
 l m Z m  Z  d  d l! m" Z" d  d l# m$ Z$ d  d l% m& Z& d  d l' m( Z( d  d l) m* Z* m+ Z+ d  d l, m- Z- d  d l. m/ Z/ d  d l0 m1 Z1 d  d l2 m3 Z3 d  d l4 m5 Z5 d  d l6 m7 Z7 d  d l8 m9 Z9 d  d l# m: Z: d  d l; m< Z< d  d l= m> Z> d  d l? m@ Z@ d  d lA mB ZB d eC f d „  ƒ  YZD d eC f d „  ƒ  YZE d  e jF f d! „  ƒ  YZG d" e jF f d# „  ƒ  YZH d$ e jF f d% „  ƒ  YZI d& eI f d' „  ƒ  YZJ d( eI f d) „  ƒ  YZK d* eI f d+ „  ƒ  YZL d, e jF f d- „  ƒ  YZM d. e jN f d/ „  ƒ  YZO d0 e jF f d1 „  ƒ  YZP d2 e jF f d3 „  ƒ  YZQ d4 eP f d5 „  ƒ  YZR d6 eP f d7 „  ƒ  YZS d8 eC f d9 „  ƒ  YZT d: e jF f d; „  ƒ  YZU d< e jF f d= „  ƒ  YZV d S(>   iÿÿÿÿN(   t   unittest(   t   WrappingFactory(   t   FilePath(   t   reactort   defert   error(   t   servert   statict   utilt   resource(   t   ForeverTakingResourcet   NoLengthResourcet   HostHeaderResourcet   PayloadResourcet   BrokenDownloadResource(   t   FTPRealmt
   FTPFactory(   t   portalt   checkerst   credentials(   t	   FTPClientt   ConnectionLost(   t   path_to_file_uri(   t   twisted_version(   t   DownloadHandlers(   t   FileDownloadHandler(   t   HTTPDownloadHandlert   HttpDownloadHandler(   t   HTTP10DownloadHandler(   t   HTTP11DownloadHandler(   t   S3DownloadHandler(   t   FTPDownloadHandler(   t   Spider(   t   Request(   t   Settings(   t   optional_features(   t   get_crawler(   t   NotConfigured(   t
   MockServer(   t   SingleRequestSpidert   DummyDHc           B   s   e  Z d  „  Z RS(   c         C   s   d  S(   N(    (   t   selft   crawler(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   __init__'   s    (   t   __name__t
   __module__R+   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR(   %   s   t   OffDHc           B   s   e  Z d  „  Z RS(   c         C   s
   t  ‚ d  S(   N(   R%   (   R)   R*   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR+   -   s    (   R,   R-   R+   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR.   +   s   t   LoadTestCasec           B   s#   e  Z d  „  Z d „  Z d „  Z RS(   c         C   sY   i d d 6} t  d i | d 6ƒ } t | ƒ } |  j d | j ƒ |  j d | j ƒ d  S(   Ns&   tests.test_downloader_handlers.DummyDHt   schemet   settings_dictt   DOWNLOAD_HANDLERS(   R$   R   t   assertInt	   _handlerst   assertNotInt   _notconfigured(   R)   t   handlersR*   t   dh(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_enabled_handler3   s
    c         C   sY   i d d 6} t  d i | d 6ƒ } t | ƒ } |  j d | j ƒ |  j d | j ƒ d  S(   Ns$   tests.test_downloader_handlers.OffDHR0   R1   R2   (   R$   R   R5   R4   R3   R6   (   R)   R7   R*   R8   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_not_configured_handler:   s
    c         C   sY   i d  d 6} t d i | d 6ƒ } t | ƒ } |  j d | j ƒ |  j d | j ƒ d  S(   NR0   R1   R2   (   t   NoneR$   R   R5   R4   R6   (   R)   R7   R*   R8   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_disabled_handlerA   s
    (   R,   R-   R9   R:   R<   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR/   1   s   		t   FileTestCasec           B   s#   e  Z d  „  Z d „  Z d „  Z RS(   c         C   sU   |  j  ƒ  |  _ t |  j d d ƒ } | j d ƒ | j ƒ  t t ƒ  ƒ j |  _ d  S(   Nt   ^t   wt
   0123456789(   t   mktempt   tmpnamet   opent   writet   closeR   R"   t   download_request(   R)   t   fd(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   setUpK   s
    
c            sh   ‡  ‡ f d †  } t  t ˆ j d ƒ ƒ ‰  ˆ  j j ƒ  j d ƒ sI t ‚ ˆ j ˆ  t d ƒ ƒ j	 | ƒ S(   Nc            s@   ˆ j  |  j ˆ  j ƒ ˆ j  |  j d ƒ ˆ j  |  j d ƒ d  S(   NiÈ   R@   (   t   assertEqualst   urlt   statust   body(   t   response(   t   requestR)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   _testS   s    R>   s   %5Et   foo(
   R!   R   RB   RJ   t   uppert   endswitht   AssertionErrorRF   R    t   addCallback(   R)   RO   (    (   RN   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_downloadR   s    c         C   s>   t  d |  j ƒ  ƒ } |  j | t d ƒ ƒ } |  j | t ƒ S(   Ns	   file://%sRP   (   R!   RA   RF   R    t   assertFailuret   IOError(   R)   RN   t   d(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_non_existent\   s    (   R,   R-   RH   RU   RY   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR=   I   s   		
t   HttpTestCasec           B   sƒ   e  Z e Z d  „  Z e j d „  ƒ Z d „  Z d „  Z	 d „  Z
 d „  Z d „  Z e j d „  ƒ Z d „  Z d	 „  Z d
 „  Z RS(   c         C   sZ  |  j  ƒ  } t j | ƒ t | ƒ j d ƒ j d ƒ t j | ƒ } | j d t	 j
 d ƒ ƒ | j d t ƒ  ƒ | j d t d t ƒ ƒ | j d t ƒ  ƒ | j d	 t ƒ  ƒ | j d
 t ƒ  ƒ | j d t ƒ  ƒ t j | d d  ƒ|  _ t |  j ƒ |  _ t j d |  j d d ƒ|  _ |  j j ƒ  j |  _ |  j t ƒ  ƒ |  _ |  j j |  _ d  S(   Nt   fileR@   t   redirects   /filet   waits   hang-after-headersRD   t   nolengtht   hostt   payloadt   brokent   timeouti    t	   interfaces	   127.0.0.1(    RA   t   ost   mkdirR   t   childt
   setContentR   t   Filet   putChildR   t   RedirectR
   t   TrueR   R   R   R   R   t   SiteR;   t   siteR   t   wrapperR   t	   listenTCPt   portt   getHostt   portnot   download_handler_clsR"   t   download_handlerRF   (   R)   t   namet   r(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRH   f   s"    c         c   s5   |  j  j ƒ  Vt |  j d ƒ r1 |  j j ƒ  Vn  d  S(   NRE   (   Rp   t   stopListeningt   hasattrRt   RE   (   R)   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   tearDowny   s    c         C   s   d |  j  | f S(   Ns   http://127.0.0.1:%d/%s(   Rr   (   R)   t   path(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   getURL   s    c         C   sT   t  |  j d ƒ ƒ } |  j | t d ƒ ƒ } | j d „  ƒ | j |  j d ƒ | S(   NR[   RP   c         S   s   |  j  S(   N(   RL   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   <lambda>…   s    R@   (   R!   R{   RF   R    RT   RI   (   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRU   ‚   s
    c         C   sZ   t  |  j d ƒ d d ƒ} |  j | t d ƒ ƒ } | j d „  ƒ | j |  j d ƒ | S(   NR[   t   methodt   HEADRP   c         S   s   |  j  S(   N(   RL   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   Œ   s    t    (   R!   R{   RF   R    RT   RI   (   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_download_head‰   s
    c         C   sT   t  |  j d ƒ ƒ } |  j | t d ƒ ƒ } | j d „  ƒ | j |  j d ƒ | S(   NR\   RP   c         S   s   |  j  S(   N(   RK   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   “   s    i.  (   R!   R{   RF   R    RT   RI   (   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_redirect_status   s
    c         C   sZ   t  |  j d ƒ d d ƒ} |  j | t d ƒ ƒ } | j d „  ƒ | j |  j d ƒ | S(   NR\   R}   R~   RP   c         S   s   |  j  S(   N(   RK   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   š   s    i.  (   R!   R{   RF   R    RT   RI   (   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_redirect_status_head—   s
    c         c   s«   t  d ƒ } i d d 6} t |  j d ƒ d | ƒ} |  j | | ƒ } |  j | t j t j ƒ Vt |  j d ƒ d | ƒ} |  j | | ƒ } |  j | t j t j ƒ Vd  S(   NRP   gš™™™™™É?t   download_timeoutR]   t   metas   hang-after-headers(   R    R!   R{   RF   RV   R   t   TimeoutErrorR   (   R)   t   spiderR„   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt!   test_timeout_download_from_spiderž   s    c            sF   ‡  ‡ f d †  } t  ˆ j d ƒ ƒ ‰  ˆ j ˆ  t d ƒ ƒ j | ƒ S(   Nc            s1   ˆ j  |  j d ˆ j ƒ ˆ j  ˆ  j i  ƒ d  S(   Ns   127.0.0.1:%d(   RI   RL   Rr   t   headers(   RM   (   RN   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   ¬   s    R_   RP   (   R!   R{   RF   R    RT   (   R)   RO   (    (   RN   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt'   test_host_header_not_in_request_headers«   s    c            s’   ‡  ‡ f d †  } t  ˆ j d ƒ d i d d 6ƒ‰  ˆ j ˆ  t d ƒ ƒ j | ƒ Sˆ j ˆ  t d ƒ ƒ } | j d „  ƒ | j ˆ j d ƒ | S(   Nc            s3   ˆ j  |  j d ƒ ˆ j  ˆ  j j d ƒ d ƒ d  S(   Ns   example.comt   Host(   RI   RL   Rˆ   t   get(   RM   (   RN   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   ´   s    R_   Rˆ   s   example.comRŠ   RP   c         S   s   |  j  S(   N(   RL   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   ¼   s    (   R!   R{   RF   R    RT   RI   (   R)   RO   RX   (    (   RN   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt)   test_host_header_seted_in_request_headers³   s    "c         C   sj   d d } t  |  j d ƒ d d d | ƒ} |  j | t d ƒ ƒ } | j d „  ƒ | j |  j | ƒ | S(	   Nt   1id   R`   R}   t   POSTRL   RP   c         S   s   |  j  S(   N(   RL   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   Ä   s    (   R!   R{   RF   R    RT   RI   (   R)   RL   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_payloadÀ   s    
!(   R,   R-   R   Rs   RH   R   t   inlineCallbacksRy   R{   RU   R€   R   R‚   R‡   R‰   RŒ   R   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRZ   b   s   								t   DeprecatedHttpTestCasec           B   s   e  Z d  Z e Z RS(   s   HTTP 1.0 test case(   R,   R-   t   __doc__R   Rs   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR‘   É   s   t   Http10TestCasec           B   s   e  Z d  Z e Z RS(   s   HTTP 1.0 test case(   R,   R-   R’   R   Rs   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR“   Î   s   t   Http11TestCasec           B   sq   e  Z d  Z e Z d e k r' d Z n  d „  Z e j	 d „  ƒ Z
 e j	 d „  ƒ Z e j	 d „  ƒ Z d „  Z RS(   s   HTTP 1.1 test caset   http11s)   HTTP1.1 not supported in twisted < 11.1.0c         C   sT   t  |  j d ƒ ƒ } |  j | t d ƒ ƒ } | j d „  ƒ | j |  j d ƒ | S(   NR[   RP   c         S   s   |  j  S(   N(   RL   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   Ü   s    R@   (   R!   R{   RF   R    RT   RI   (   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt#   test_download_without_maxsize_limitÙ   s
    c         c   s—   t  |  j d ƒ ƒ } |  j | t d d d ƒƒ } | j d „  ƒ | j |  j d ƒ | V|  j | t d d d ƒƒ } |  j | t j t	 j
 ƒ Vd  S(   NR[   RP   t   download_maxsizei
   c         S   s   |  j  S(   N(   RL   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   ç   s    R@   i	   (   R!   R{   RF   R    RT   RI   RV   R   t   CancelledErrorR   t   ConnectionAborted(   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_download_with_maxsizeà   s    c         c   s^   i d d 6} t  |  j d ƒ d | ƒ} |  j | t d ƒ ƒ } |  j | t j t j ƒ Vd  S(   Ni   R—   R[   R„   RP   (	   R!   R{   RF   R    RV   R   R˜   R   R™   (   R)   R„   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt"   test_download_with_maxsize_per_reqî   s    c         c   sQ   t  |  j d ƒ ƒ } |  j | t d d d ƒƒ } |  j | t j t j ƒ Vd  S(   NR[   RP   R—   i   (	   R!   R{   RF   R    RV   R   R˜   R   R™   (   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt+   test_download_with_small_maxsize_per_spiderõ   s    c         C   sZ   t  |  j d ƒ ƒ } |  j | t d d d ƒƒ } | j d „  ƒ | j |  j d ƒ | S(   NR[   RP   R—   id   c         S   s   |  j  S(   N(   RL   (   Rv   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR|   þ   s    R@   (   R!   R{   RF   R    RT   RI   (   R)   RN   RX   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt+   test_download_with_large_maxsize_per_spiderû   s
    (   R,   R-   R’   R   Rs   R#   t   skipR–   R   R   Rš   R›   Rœ   R   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR”   Ó   s   		t   Http11MockServerTestCasec           B   sk   e  Z d  Z d e k r! d Z n  d „  Z d „  Z e j d „  ƒ Z	 e j d „  ƒ Z
 e j d „  ƒ Z RS(   s"   HTTP 1.1 test case with MockServerR•   s)   HTTP1.1 not supported in twisted < 11.1.0c         C   s   t  ƒ  |  _ |  j j ƒ  d  S(   N(   R&   t
   mockservert	   __enter__(   R)   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRH     s    c         C   s   |  j  j d  d  d  ƒ d  S(   N(   R    t   __exit__R;   (   R)   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRy     s    c      	   c   s]   t  t ƒ } | j d t d d d i d d 6ƒ ƒ V| j j d } |  j | j t j	 ƒ d  S(   Nt   seedRJ   s   http://localhost:8998/partialR„   iè  R—   t   failure(
   R$   R'   t   crawlR!   R†   R„   t   assertIsInstancet   valueR   R˜   (   R)   R*   R¤   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt!   test_download_with_content_length  s    'c         c   sr   t  t ƒ } | j d t d d ƒ ƒ V| j j j d ƒ } |  j | d  k ƒ | j j d } |  j | d ƒ d  S(   NR£   RJ   s   http://localhost:8998R¤   t   close_reasont   finished(	   R$   R'   R¥   R!   R†   R„   R‹   t
   assertTrueR;   (   R)   R*   R¤   t   reason(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRU     s    c      
   c   s  t  j rt d k rt t ƒ } d d } t d d d d	 | d
 i d d 6ƒ} | j d | ƒ V| j j d } |  j	 | j
 t j ƒ | j j d d ƒ | j d d ƒ } | j d | ƒ V| j j j d ƒ } |  j | d  k ƒ | j j d } |  j | d ƒ n t j d ƒ ‚ d  S(   Ni   i   i    R   id   s   http://localhost:8998/payloadR}   RŽ   RL   R„   i2   R—   R£   R¤   s   Accept-Encodings   gzip,deflateRJ   s   http://localhost:8998/xpayloadR©   Rª   sN   xpayload and payload endpoint only enabled for twisted > 12.3.0 and python 2.x(   i   i   i    (   t   sixt   PY2R   R$   R'   R!   R¥   R†   R„   R¦   R§   R   R˜   Rˆ   t
   setdefaultt   replaceR‹   R«   R;   R    t   SkipTest(   R)   R*   RL   RN   R¤   R¬   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_download_gzip_response!  s    
%(   R,   R-   R’   R#   Rž   RH   Ry   R   R   R¨   RU   R²   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRŸ     s   					t   UriResourcec           B   s    e  Z d  Z d „  Z d „  Z RS(   s&   Return the full uri that was requestedc         C   s   |  S(   N(    (   R)   Rz   RN   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   getChild>  s    c         C   s   | j  S(   N(   t   uri(   R)   RN   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   renderA  s    (   R,   R-   R’   R´   R¶   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR³   ;  s   	t   HttpProxyTestCasec           B   sM   e  Z e Z d  „  Z e j d „  ƒ Z d „  Z d „  Z	 d „  Z
 d „  Z RS(   c         C   s|   t  j t ƒ  d d  ƒ} t | ƒ } t j d | d d ƒ|  _ |  j j ƒ  j |  _	 |  j
 t ƒ  ƒ |  _ |  j j |  _ d  S(   NRb   i    Rc   s	   127.0.0.1(   R   Rl   R³   R;   R   R   Ro   Rp   Rq   Rr   Rs   R"   Rt   RF   (   R)   Rm   Rn   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRH   H  s    c         c   s5   |  j  j ƒ  Vt |  j d ƒ r1 |  j j ƒ  Vn  d  S(   NRE   (   Rp   Rw   Rx   Rt   RE   (   R)   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRy   P  s    c         C   s   d |  j  | f S(   Ns   http://127.0.0.1:%d/%s(   Rr   (   R)   Rz   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR{   V  s    c            sY   ‡  ‡ f d †  } ˆ j  d ƒ } t d d i | d 6ƒ‰  ˆ j ˆ  t d ƒ ƒ j | ƒ S(   Nc            s@   ˆ j  |  j d ƒ ˆ j  |  j ˆ  j ƒ ˆ j  |  j d ƒ d  S(   NiÈ   s   http://example.com(   RI   RK   RJ   RL   (   RM   (   RN   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   Z  s    R   s   http://example.comR„   t   proxyRP   (   R{   R!   RF   R    RT   (   R)   RO   t
   http_proxy(    (   RN   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_download_with_proxyY  s    c            s]   ‡  ‡ f d †  } d ˆ j  d ƒ } t d d i | d 6ƒ‰  ˆ j ˆ  t d ƒ ƒ j | ƒ S(   Nc            s@   ˆ j  |  j d ƒ ˆ j  |  j ˆ  j ƒ ˆ j  |  j d ƒ d  S(   NiÈ   s   https://example.com(   RI   RK   RJ   RL   (   RM   (   RN   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   d  s    s   %s?noconnectR   s   https://example.comR„   R¸   RP   (   R{   R!   RF   R    RT   (   R)   RO   R¹   (    (   RN   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt(   test_download_with_proxy_https_noconnectc  s    c            sF   ‡  ‡ f d †  } t  ˆ j d ƒ ƒ ‰  ˆ j ˆ  t d ƒ ƒ j | ƒ S(   Nc            s@   ˆ j  |  j d ƒ ˆ j  |  j ˆ  j ƒ ˆ j  |  j d ƒ d  S(   NiÈ   s   /path/to/resource(   RI   RK   RJ   RL   (   RM   (   RN   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   n  s    s   path/to/resourceRP   (   R!   R{   RF   R    RT   (   R)   RO   (    (   RN   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_download_without_proxym  s    (   R,   R-   R   Rs   RH   R   R   Ry   R{   Rº   R»   R¼   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR·   E  s   			
	
t   DeprecatedHttpProxyTestCasec           B   s   e  Z d  Z e Z RS(   s5   Old deprecated reference to http10 downloader handler(   R,   R-   R’   R   Rs   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR½   w  s   t   Http10ProxyTestCasec           B   s   e  Z e Z RS(    (   R,   R-   R   Rs   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR¾   |  s   t   Http11ProxyTestCasec           B   s#   e  Z e Z d  e k r! d Z n  RS(   R•   s)   HTTP1.1 not supported in twisted < 11.1.0(   R,   R-   R   Rs   R#   Rž   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR¿   €  s   t   HttpDownloadHandlerMockc           B   s   e  Z d  „  Z d „  Z RS(   c         C   s   d  S(   N(    (   R)   t   settings(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyR+   ‡  s    c         C   s   | S(   N(    (   R)   RN   R†   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRF   Š  s    (   R,   R-   R+   RF   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRÀ   †  s   	t
   S3TestCasec           B   sn   e  Z d  e k o d Z d Z d Z d „  Z d „  Z d „  Z d „  Z	 d „  Z
 d	 „  Z d
 „  Z d „  Z RS(   t   botos   missing boto libraryt   0PN5J17HBGZHT7JJ3X82s(   uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2oc         C   s@   t  t ƒ  |  j |  j d t ƒ} | j |  _ t d ƒ |  _ d  S(   Nt   httpdownloadhandlerRP   (   R   R"   t   AWS_ACCESS_KEY_IDt   AWS_SECRET_ACCESS_KEYRÀ   RF   R    R†   (   R)   t   s3reqh(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRH   —  s
    		c         C   sI   t  d d i d d 6ƒ} |  j | |  j ƒ } |  j | j d d ƒ d  S(   Ns   s3://johnsmith/photos/puppy.jpgRˆ   s   Tue, 27 Mar 2007 19:36:42 +0000t   Datet   Authorizations5   AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=(   R!   RF   R†   t   assertEqualRˆ   (   R)   t   reqt   httpreq(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_request_signing1ž  s
    	c         C   s]   t  d d d d i d d 6d d 6d	 d
 6ƒ} |  j | |  j ƒ } |  j | j d d ƒ d  S(   Ns   s3://johnsmith/photos/puppy.jpgR}   t   PUTRˆ   s
   image/jpegs   Content-Types   Tue, 27 Mar 2007 21:15:45 +0000RÉ   t   94328s   Content-LengthRÊ   s5   AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=(   R!   RF   R†   RË   Rˆ   (   R)   RÌ   RÍ   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_request_signing2¦  s    c         C   sV   t  d d d d i d d 6d d 6ƒ} |  j | |  j ƒ } |  j | j d	 d
 ƒ d  S(   Ns6   s3://johnsmith/?prefix=photos&max-keys=50&marker=puppyR}   t   GETRˆ   s   Mozilla/5.0s
   User-Agents   Tue, 27 Mar 2007 19:42:41 +0000RÉ   RÊ   s5   AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=(   R!   RF   R†   RË   Rˆ   (   R)   RÌ   RÍ   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_request_signing3±  s    		c         C   sO   t  d d d d i d d 6ƒ} |  j | |  j ƒ } |  j | j d d ƒ d  S(	   Ns   s3://johnsmith/?aclR}   RÒ   Rˆ   s   Tue, 27 Mar 2007 19:44:46 +0000RÉ   RÊ   s5   AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=(   R!   RF   R†   RË   Rˆ   (   R)   RÌ   RÍ   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_request_signing4¼  s
    	c         C   sV   t  d d d d i d d 6d d 6ƒ} |  j | |  j ƒ } |  j | j d	 d
 ƒ d  S(   Ns   s3://johnsmith/photos/puppy.jpgR}   t   DELETERˆ   s   Tue, 27 Mar 2007 21:20:27 +0000RÉ   s   Tue, 27 Mar 2007 21:20:26 +0000s
   x-amz-dateRÊ   s5   AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=(   R!   RF   R†   RË   Rˆ   (   R)   RÌ   RÍ   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_request_signing5Ä  s    		c         C   sœ   t  d d d d i d d 6d d 6d	 d
 6d d 6d d 6d d 6d d 6d d 6d d 6d d 6d d 6d d 6ƒ} |  j | |  j ƒ } |  j | j d d ƒ d  S(   Ns/   s3://static.johnsmith.net:8080/db-backup.dat.gzR}   RÏ   Rˆ   s   curl/7.15.5s
   User-Agents   static.johnsmith.net:8080RŠ   s   Tue, 27 Mar 2007 21:06:08 +0000RÉ   s   public-reads	   x-amz-acls   application/x-downloads   content-types   4gJE4saaMU4BqNR0kLY+lw==s   Content-MD5s$   joe@johnsmith.net,jane@johnsmith.nets   X-Amz-Meta-ReviewedByt
   0x02661779s   X-Amz-Meta-FileChecksumt   crc32s   X-Amz-Meta-ChecksumAlgorithms!   attachment; filename=database.dats   Content-Dispositiont   gzips   Content-Encodingt   5913339s   Content-LengthRÊ   s5   AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=(   R!   RF   R†   RË   Rˆ   (   R)   RÌ   RÍ   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_request_signing6Ð  s"    		c         C   sO   t  d d d d i d d 6ƒ} |  j | |  j ƒ } |  j | j d d ƒ d  S(	   NsL   s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpgR}   RÒ   Rˆ   s   Tue, 27 Mar 2007 19:42:41 +0000RÉ   RÊ   s5   AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=(   R!   RF   R†   RË   Rˆ   (   R)   RÌ   RÍ   (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_request_signing7å  s    
(   R,   R-   R#   Rž   RÆ   RÇ   RH   RÎ   RÑ   RÓ   RÔ   RÖ   RÛ   RÜ   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRÂ     s   							t   FTPTestCasec           B   sn   e  Z d  Z d Z e d k  r' d Z n  d „  Z d d d „ Z d „  Z	 d	 „  Z
 d
 „  Z d „  Z d „  Z RS(   t   scrapyt   passwdi
   i   i    sB   Twisted pre 10.2.0 doesn't allow to set home path other than /homec         C   sL  |  j  ƒ  |  _ t j |  j ƒ t j j |  j |  j ƒ } t j | ƒ t | ƒ } | j d ƒ j	 d ƒ | j d ƒ j	 d ƒ t
 d |  j d |  j ƒ } t j | ƒ } t j ƒ  } | j |  j |  j ƒ | j | t j ƒ t d | ƒ |  _ t j d |  j d	 d
 ƒ|  _ |  j j ƒ  j |  _ t t ƒ  ƒ |  _ |  j |  j j ƒ d  S(   Ns   file.txts   I have the power!s   file with spaces.txts   Moooooooooo power!t   anonymousRoott   userHomeR   i    Rc   s	   127.0.0.1(    RA   t	   directoryRd   Re   Rz   t   joint   usernameR   Rf   Rg   R   R   t   PortalR   t'   InMemoryUsernamePasswordDatabaseDontUset   addUsert   passwordt   registerCheckerR   t   IUsernamePasswordR   t   factoryR   Ro   Rp   Rq   t   portNumR   R"   Rt   t
   addCleanupRw   (   R)   t   userdirt   fpt   realmt   pt   users_checker(    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRH   ü  s"    c            sL   ‡  f d †  } | j  | ƒ | r2 | j  | ƒ n  | rH | j | ƒ n  | S(   Nc            s   ˆ  j  j j j ƒ  |  S(   N(   Rt   t   clientt	   transportt   loseConnection(   t   data(   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   _clean  s    (   RT   t
   addErrback(   R)   t   deferredt   callbackt   errbackR÷   (    (   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   _add_test_callbacks  s    c            sd   t  d d ˆ  j d i ˆ  j d 6ˆ  j d 6ƒ } ˆ  j j | d  ƒ } ‡  f d †  } ˆ  j | | ƒ S(   NRJ   s   ftp://127.0.0.1:%s/file.txtR„   t   ftp_usert   ftp_passwordc            sQ   ˆ  j  |  j d ƒ ˆ  j  |  j d ƒ ˆ  j  |  j i d g d 6d g d 6ƒ d  S(   NiÈ   s   I have the power!R   s   Local Filenamet   17t   Size(   RË   RK   RL   Rˆ   (   Rv   (   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   "  s    (   R!   Rì   Rä   Rè   Rt   RF   R;   Rü   (   R)   RN   RX   RO   (    (   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_ftp_download_success  s
    c            sd   t  d d ˆ  j d i ˆ  j d 6ˆ  j d 6ƒ } ˆ  j j | d  ƒ } ‡  f d †  } ˆ  j | | ƒ S(   NRJ   s'   ftp://127.0.0.1:%s/file with spaces.txtR„   Rý   Rþ   c            sQ   ˆ  j  |  j d ƒ ˆ  j  |  j d ƒ ˆ  j  |  j i d g d 6d g d 6ƒ d  S(   NiÈ   s   Moooooooooo power!R   s   Local Filenamet   18R   (   RË   RK   RL   Rˆ   (   Rv   (   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   /  s    (   R!   Rì   Rä   Rè   Rt   RF   R;   Rü   (   R)   RN   RX   RO   (    (   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt"   test_ftp_download_path_with_spaces(  s    c            sd   t  d d ˆ  j d i ˆ  j d 6ˆ  j d 6ƒ } ˆ  j j | d  ƒ } ‡  f d †  } ˆ  j | | ƒ S(   NRJ   s   ftp://127.0.0.1:%s/notexist.txtR„   Rý   Rþ   c            s   ˆ  j  |  j d ƒ d  S(   Ni”  (   RË   RK   (   Rv   (   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   :  s    (   R!   Rì   Rä   Rè   Rt   RF   R;   Rü   (   R)   RN   RX   RO   (    (   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_ftp_download_notexist5  s
    c            st   d ‰  t  d d ˆ j d i ˆ j d 6ˆ j d 6ˆ  d 6ƒ } ˆ j j | d  ƒ } ‡  ‡ f d †  } ˆ j | | ƒ S(	   Ns   /tmp/file.txtRJ   s   ftp://127.0.0.1:%s/file.txtR„   Rý   Rþ   t   ftp_local_filenamec            s   ˆ j  |  j ˆ  ƒ ˆ j  |  j i d g d 6d g d 6ƒ ˆ j t j j ˆ  ƒ ƒ t ˆ  ƒ  } ˆ j  | j ƒ  d ƒ Wd  QXt j	 ˆ  ƒ d  S(   Ns   /tmp/file.txts   Local FilenameRÿ   R   s   I have the power!(
   RË   RL   Rˆ   R«   Rd   Rz   t   existsRC   t   readt   remove(   Rv   t   f(   t   local_fnameR)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   D  s    '(   R!   Rì   Rä   Rè   Rt   RF   R;   Rü   (   R)   RN   RX   RO   (    (   R
  R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_ftp_local_filename>  s    $c            sd   t  d d ˆ  j d i ˆ  j d 6d d 6ƒ } ˆ  j j | d  ƒ } ‡  f d †  } ˆ  j | d | ƒS(	   NRJ   s   ftp://127.0.0.1:%s/file.txtR„   Rý   t   invalidRþ   c            s   ˆ  j  |  j t ƒ d  S(   N(   RË   t   typeR   (   Rv   (   R)   (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRO   R  s    Rû   (   R!   Rì   Rä   Rt   RF   R;   Rü   (   R)   RN   RX   RO   (    (   R)   sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   test_invalid_credentialsM  s
    (   i
   i   i    N(   R,   R-   Rä   Rè   R   Rž   RH   R;   Rü   R  R  R  R  R  (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyRÝ   ô  s   							(W   Rd   t   twistedR­   t   twisted.trialR    t   twisted.protocols.policiesR   t   twisted.python.filepathR   t   twisted.internetR   R   R   t   twisted.webR   R   R   R	   t   twisted.web.test.test_webclientR
   R   R   R   R   t   twisted.protocols.ftpR   R   t   twisted.credR   R   R   R   R   t	   w3lib.urlR   RÞ   R   t   scrapy.core.downloader.handlersR   t$   scrapy.core.downloader.handlers.fileR   t$   scrapy.core.downloader.handlers.httpR   R   t&   scrapy.core.downloader.handlers.http10R   t&   scrapy.core.downloader.handlers.http11R   t"   scrapy.core.downloader.handlers.s3R   t#   scrapy.core.downloader.handlers.ftpR   t   scrapy.spidersR    t   scrapy.httpR!   t   scrapy.settingsR"   R#   t   scrapy.utils.testR$   t   scrapy.exceptionsR%   t   tests.mockserverR&   t   tests.spidersR'   t   objectR(   R.   t   TestCaseR/   R=   RZ   R‘   R“   R”   RŸ   t   ResourceR³   R·   R½   R¾   R¿   RÀ   RÂ   RÝ   (    (    (    sB   /home/travis/build/scrapy/scrapy/tests/test_downloader_handlers.pyt   <module>   sZ   "(g08
2g