ó
ç'ÊUc           @  s.  d  d l  m Z d  d l m Z d  d l m Z d  d l m Z d  d l m	 Z	 d  d l
 m Z m Z d  d l m Z m Z d  d l m Z d  d	 l m Z d  d
 l m Z d  d l m Z d  d l m Z d  d l m Z d „  Z d e j f d „  ƒ  YZ d e f d „  ƒ  YZ d e f d „  ƒ  YZ  d S(   iÿÿÿÿ(   t   print_function(   t
   LogCapture(   t   unittest(   t   Failure(   t   reactor(   t   Deferredt   inlineCallbacks(   t   Requestt   Response(   t   Spider(   t   request_fingerprint(   t   MediaPipeline(   t   failure_to_exc_info(   t   disconnect_all(   t   signalsc         C  s)   |  j  j d ƒ } t | ƒ r% | ƒ  S| S(   Nt   response(   t   metat   gett   callable(   t   requestt   infoR   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   _mocked_download_func   s    t   BaseMediaPipelineTestCasec           B  s\   e  Z e Z d  „  Z d „  Z d „  Z d „  Z d „  Z d „  Z	 d „  Z
 e d „  ƒ Z RS(   c         C  sJ   t  d ƒ |  _ |  j d t ƒ |  _ |  j j |  j ƒ |  j j |  _ d  S(   Ns	   media.comt   download_func(   R	   t   spidert   pipeline_classR   t   pipet   open_spidert
   spiderinfoR   (   t   self(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   setUp   s    c         C  sC   x< t  t ƒ j ƒ  D]( \ } } | j d ƒ s t | ƒ q q Wd  S(   Nt   _(   t   varsR   t   itemst
   startswithR   (   R   t   namet   signal(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   tearDown    s    c         C  s4   t  d ƒ } |  j j | |  j ƒ d  k s0 t ‚ d  S(   Ns
   http://url(   R   R   t   media_to_downloadR   t   Nonet   AssertionError(   R   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_default_media_to_download%   s    c         C  s7   t  d d ƒ } |  j j | |  j ƒ d  k s3 t ‚ d  S(   NR#   (   t   dictR   t   get_media_requestsR   R'   R(   (   R   t   item(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_default_get_media_requests)   s    c         C  sI   t  d ƒ } t d d d ƒ} |  j j | | |  j ƒ | k sE t ‚ d  S(   Ns
   http://urlt   bodyt    (   R   R   R   t   media_downloadedR   R(   (   R   R   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_default_media_downloaded-   s    c         C  sF   t  d ƒ } t t ƒ  ƒ } |  j j | | |  j ƒ | k sB t ‚ d  S(   Ns
   http://url(   R   R   t	   ExceptionR   t   media_failedR   R(   (   R   R   t   fail(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_default_media_failed2   s    c      	   C  s\  t  d d ƒ } |  j j g  | |  j ƒ | k s6 t ‚ t t ƒ  ƒ } t d f t | f g } t	 ƒ  " } |  j j | | |  j ƒ } Wd  QX| | k sœ t ‚ t
 | j ƒ d k s· t ‚ | j d } | j d k sÙ t ‚ |  j | j t | ƒ ƒ t |  j _ t	 ƒ  " } |  j j | | |  j ƒ } Wd  QX| | k s=t ‚ t
 | j ƒ d k sXt ‚ d  S(   NR#   i   i    t   ERROR(   R*   R   t   item_completedR   R(   R   R2   t   Truet   FalseR   t   lent   recordst	   levelnamet   assertTupleEqualt   exc_infoR   t   LOG_FAILED_RESULTS(   R   R,   R4   t   resultst   lt   new_itemt   record(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_default_item_completed7   s     '!!c         c  s>   t  d d ƒ } |  j j | |  j ƒ V} | | k s: t ‚ d  S(   NR#   (   R*   R   t   process_itemR   R(   (   R   R,   RB   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_default_process_itemO   s    (   t   __name__t
   __module__R   R   R   R%   R)   R-   R1   R5   RD   R   RF   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyR      s   							t   MockedMediaPipelinec           B  sG   e  Z d  „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z RS(   c         O  s&   t  t |  ƒ j | | Ž  g  |  _ d  S(   N(   t   superRI   t   __init__t   _mockcalled(   R   t   argst   kwargs(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyRK   X   s    c         C  s)   |  j  j d ƒ t t |  ƒ j | | ƒ S(   Nt   download(   RL   t   appendRJ   RI   RO   (   R   R   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyRO   \   s    c         C  sH   |  j  j d ƒ d | j k r/ | j j d ƒ St t |  ƒ j | | ƒ S(   NR&   t   result(   RL   RP   R   R   RJ   RI   R&   (   R   R   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyR&   `   s    c         C  s   |  j  j d ƒ | j d ƒ S(   NR+   t   requests(   RL   RP   R   (   R   R,   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyR+   f   s    c         C  s,   |  j  j d ƒ t t |  ƒ j | | | ƒ S(   NR0   (   RL   RP   RJ   RI   R0   (   R   R   R   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyR0   j   s    c         C  s,   |  j  j d ƒ t t |  ƒ j | | | ƒ S(   NR3   (   RL   RP   RJ   RI   R3   (   R   t   failureR   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyR3   n   s    c         C  s<   |  j  j d ƒ t t |  ƒ j | | | ƒ } | | d <| S(   NR7   R@   (   RL   RP   RJ   RI   R7   (   R   R@   R,   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyR7   r   s    
(	   RG   RH   RK   RO   R&   R+   R0   R3   R7   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyRI   V   s   						t   MediaPipelineTestCasec           B  s†   e  Z e Z e d  „  ƒ Z e d „  ƒ Z e d „  ƒ Z e d „  ƒ Z e d „  ƒ Z	 e d „  ƒ Z
 e d „  ƒ Z e d „  ƒ Z RS(   c         #  s¿   ‡  f d †  } ‡  f d †  } t  d ƒ } t d d t d | ƒ d | d | ƒ} t d | ƒ } ˆ  j j | ˆ  j ƒ V} ˆ  j | d	 t | f g ƒ ˆ  j ˆ  j j d
 d d d d g ƒ d  S(   Nc           s   ˆ  j  j j d ƒ p |  S(   Nt   request_callback(   R   RL   RP   (   R   (   R   (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   <lambda>   s    c           s   ˆ  j  j j d ƒ p |  S(   Nt   request_errback(   R   RL   RP   (   R   (   R   (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyRV   €   s    s   http://url1R   R   t   callbackt   errbackRR   R@   R+   R&   R0   RU   R7   (	   R   R   R*   R   RE   R   t   assertEqualR8   RL   (   R   t   cbt   ebt   rspt   reqR,   RB   (    (   R   s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_result_succeed}   s    'c         #  sÎ   t  ˆ  j _ ‡  f d †  } ‡  f d †  } t t ƒ  ƒ } t d d t d | ƒ d | d | ƒ} t d | ƒ } ˆ  j j | ˆ  j ƒ V} ˆ  j	 | d	 t  | f g ƒ ˆ  j	 ˆ  j j
 d
 d d d d g ƒ d  S(   Nc           s   ˆ  j  j j d ƒ p |  S(   NRU   (   R   RL   RP   (   R   (   R   (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyRV      s    c           s   ˆ  j  j j d ƒ p |  S(   NRW   (   R   RL   RP   (   R   (   R   (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyRV   Ž   s    s   http://url1R   R   RX   RY   RR   R@   R+   R&   R3   RW   R7   (   R9   R   R?   R   R2   R   R*   RE   R   RZ   RL   (   R   R[   R\   R4   R^   R,   RB   (    (   R   s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_result_failureŠ   s    'c         c  sf  t  |  j _ t d ƒ } t d d t d | ƒ ƒ} t t ƒ  ƒ } t d d t d | ƒ ƒ} t d | | g ƒ } |  j j | |  j	 ƒ V} |  j
 | d t | f t  | f g ƒ |  j j } |  j
 | d d ƒ |  j
 | j d ƒ d	 ƒ |  j
 | j d
 ƒ d	 ƒ |  j
 | d d
 ƒ |  j
 | j d ƒ d ƒ |  j
 | j d ƒ d	 ƒ |  j
 | j d ƒ d	 ƒ d  S(   Ns   http://url1R   R   s   http://url2RR   R@   i    R+   i   R7   iÿÿÿÿR&   i   R0   R3   (   R9   R   R?   R   R   R*   R   R2   RE   R   RZ   R8   RL   t   count(   R   t   rsp1t   req1R4   t   req2R,   RB   t   m(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_mix_of_success_and_failure˜   s     &c         c  s  t  d ƒ } t d | ƒ } |  j j | |  j ƒ V} | | k sF t ‚ t | ƒ |  j j k sd t ‚ t  d ƒ } t  d ƒ } t d t	 | | g ƒ ƒ } |  j j | |  j ƒ V} | | k sÂ t ‚ t | ƒ |  j j k sà t ‚ t | ƒ |  j j k sþ t ‚ d  S(   Ns
   http://urlRR   s   http://url1s   http://url2(
   R   R*   R   RE   R   R(   R
   R   t
   downloadedt   iter(   R   R^   R,   RB   Rc   Rd   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   test_get_media_requests®   s    c         c  s  t  d ƒ } t d d t d | ƒ ƒ} t d | ƒ } |  j j | |  j ƒ V} |  j | | k ƒ |  j | d t | f g ƒ t | j	 d t d t  d ƒ ƒ ƒ} t d | ƒ } |  j j | |  j ƒ V} |  j | | k ƒ |  j t
 | ƒ t
 | ƒ ƒ |  j | d t | f g ƒ d  S(   Ns   http://url1R   R   RR   R@   s   http://donot.download.me(   R   R   R*   R   RE   R   t
   assertTrueRZ   R8   t   urlR
   (   R   Rb   Rc   R,   RB   Rd   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt-   test_results_are_cached_across_multiple_itemsÀ   s    $c         c  s¶   t  d ƒ } t d d t d | ƒ ƒ} t | j d t d t  d ƒ ƒ ƒ} t d | | g ƒ } |  j j | |  j ƒ V} |  j | | k ƒ |  j | d t	 | f t	 | f g ƒ d  S(   Ns   http://url1R   R   s   http://donot.download.meRR   R@   (
   R   R   R*   Rk   R   RE   R   Rj   RZ   R8   (   R   Rb   Rc   Rd   R,   RB   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt3   test_results_are_cached_for_requests_of_single_itemÑ   s    $c         #  sÐ   ‡ ‡ f d †  ‰  t  d ƒ ‰ ‡  ‡ f d †  } ‡ f d †  } t d d t d | ƒ ƒ‰ t ˆ j d t d | ƒ ƒ} t d ˆ | g ƒ } ˆ j j | ˆ j ƒ V} ˆ j | d t ˆ f t ˆ f g ƒ d  S(	   Nc           s{   t  ˆ  ƒ } ˆ j | ˆ j j k ƒ ˆ j | ˆ j j k ƒ ˆ j | ˆ j j k ƒ ˆ j t ˆ j j | ƒ d ƒ |  S(   Ni   (   R
   Rj   R   t   downloadingt   waitingRg   RZ   R:   (   R   t   fp(   Rc   R   (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   _check_downloadingÝ   s     s
   http://urlc            s,   t  ƒ  j ˆ  ƒ }  t j d |  j ˆ ƒ |  S(   Ngš™™™™™¹?(   R   t   addCallbackR   t	   callLaterRX   (   t   dfd(   Rq   Rb   (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt	   rsp1_funcæ   s    c             s   ˆ  j  d ƒ d  S(   Ns8   it must cache rsp1 result and must not try to redownload(   R4   (    (   R   (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt	   rsp2_funcë   s    R   R   RR   R@   (	   R   R   R*   Rk   R   RE   R   RZ   R8   (   R   Ru   Rv   Rd   R,   RB   (    (   Rq   Rc   Rb   R   s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt#   test_wait_if_request_is_downloadingÛ   s    c         c  sŒ   t  d d t d d d |  j ƒ ƒ} t d | ƒ } |  j j | |  j ƒ V} |  j | d t d f g ƒ |  j |  j j d d	 d
 g ƒ d  S(   Ns
   http://urlR   RQ   t   ITSMER   RR   R@   R+   R&   R7   (	   R   R*   R4   R   RE   R   RZ   R8   RL   (   R   R^   R,   RB   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt!   test_use_media_to_download_resultô   s    $(   RG   RH   RI   R   R   R_   R`   Rf   Ri   Rl   Rm   Rw   Ry   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyRT   y   s   
N(!   t
   __future__R    t   testfixturesR   t   twisted.trialR   t   twisted.python.failureR   t   twisted.internetR   t   twisted.internet.deferR   R   t   scrapy.httpR   R   t   scrapy.spidersR	   t   scrapy.utils.requestR
   t   scrapy.pipelines.mediaR   t   scrapy.utils.logR   t   scrapy.utils.signalR   t   scrapyR   R   t   TestCaseR   RI   RT   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_media.pyt   <module>   s    	@#