ó
ç'ÊUc           @   s  d  d l  Z  d  d l Z d  d l Z d  d l Z d  d l m Z d  d l m Z d  d l m	 Z	 d  d l
 m Z d  d l m Z m Z d  d l m Z m Z d  d l m Z m Z d  d	 l m Z d  d
 l m Z d „  Z d e	 j f d „  ƒ  YZ d e f d „  ƒ  YZ d e	 j f d „  ƒ  YZ d e	 j f d „  ƒ  YZ d e f d „  ƒ  YZ d „  Z  d „  Z! e" d k r}e	 j# ƒ  n  d S(   iÿÿÿÿN(   t   mkdtemp(   t   rmtree(   t   unittest(   t   defer(   t   FilesPipelinet   FSFilesStore(   t   Itemt   Field(   t   Requestt   Response(   t   Settings(   t   mockc         C   s)   |  j  j d ƒ } t | ƒ r% | ƒ  S| S(   Nt   response(   t   metat   gett   callable(   t   requestt   infoR   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   _mocked_download_func   s    t   FilesPipelineTestCasec           B   sP   e  Z d  „  Z d „  Z d „  Z d „  Z e j d „  ƒ Z e j d „  ƒ Z	 RS(   c         C   sN   t  ƒ  |  _ t j t i |  j d 6ƒ ƒ |  _ t |  j _ |  j j d  ƒ d  S(   Nt   FILES_STORE(
   R    t   tempdirR   t   from_settingsR
   t   pipelineR   t   download_funct   open_spidert   None(   t   self(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   setUp   s    "c         C   s   t  |  j ƒ d  S(   N(   R   R   (   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   tearDown    s    c         C   sé   |  j  j } |  j | t d ƒ ƒ d ƒ |  j | t d ƒ ƒ d ƒ |  j | t d ƒ ƒ d ƒ |  j | t d ƒ ƒ d ƒ |  j | t d	 ƒ ƒ d
 ƒ |  j | t d ƒ ƒ d ƒ |  j | t d ƒ d t d ƒ d t ƒ  ƒd ƒ d  S(   Ns!   https://dev.mydeco.com/mydeco.pdfs1   full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdfsK   http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txts1   full/4ce274dd83db0368bafd7e406f382ae088e39219.txts9   https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.docs1   full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.docsE   http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpgs1   full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpgs3   http://www.dorma.co.uk/images/product_details/2532/s-   full/97ee6f8a46cbbb418ea91502fd24176865cf39b2s2   http://www.dorma.co.uk/images/product_details/2532s-   full/244e0dd7d96a3b7b01f54eded250c9e272577aa1R   R   (   R   t	   file_patht   assertEqualR   R	   t   object(   R   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   test_file_path#   s"    	c         C   s~   t  |  j j t ƒ s t ‚ |  j |  j j j |  j ƒ d } t j	 j
 |  j d d d ƒ } |  j |  j j j | ƒ | ƒ d  S(   Ns   some/image/key.jpgt   somet   images   key.jpg(   t
   isinstanceR   t   storeR   t   AssertionErrorR   t   basedirR   t   ost   patht   joint   _get_filesystem_path(   R   R)   t   fullpath(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   test_fs_store6   s
    c         c   sé   d } t  | ƒ } t j j t d d t ƒt j j t d d i d d 6t j ƒ  d 6ƒt j j t d d t | ƒ g ƒg } x | D] } | j	 ƒ  q„ W|  j
 j | d  ƒ V} |  j | d	 d
 d d ƒ x | D] } | j ƒ  qÑ Wd  S(   Ns   http://example.com/file.pdft	   inc_statst   return_valuet	   stat_filet   abct   checksumt   last_modifiedt   get_media_requestst   filesi    (   t   _create_item_with_filesR   t   patchR    R   t   TrueR   t   timet   _prepare_request_objectt   startR   t   process_itemR   R   t   stop(   R   t   item_urlt   itemt   patcherst   pt   result(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   test_file_not_expired>   s    c         c   s   d } t  | ƒ } t j j t d d i d d 6t j ƒ  t j d d d d d	 6ƒt j j t d
 d t | ƒ g ƒt j j t d d t	 ƒg } x | D] } | j
 ƒ  q› W|  j j | d  ƒ V} |  j | d d d d ƒ x | D] } | j ƒ  qè Wd  S(   Ns   http://example.com/file2.pdfR0   R/   R1   R2   i<   i   i   R3   R4   R.   R5   i    (   R6   R   R7   R    R   R9   R   t   EXPIRESR:   R8   R;   R   R<   R   t   assertNotEqualR=   (   R   R>   R?   R@   RA   RB   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   test_file_expiredR   s    '(
   t   __name__t
   __module__R   R   R!   R-   R   t   inlineCallbacksRC   RF   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyR      s   				t   DeprecatedFilesPipelinec           B   s   e  Z d  „  Z RS(   c         C   s9   t  j | ƒ j ƒ  } t j j | ƒ d } d | | f S(   Ni   s
   empty/%s%s(   t   hashlibt   sha1t	   hexdigestR(   R)   t   splitext(   R   t   urlt
   media_guidt	   media_ext(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   file_keyi   s    (   RG   RH   RR   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyRJ   h   s   t   DeprecatedFilesPipelineTestCasec           B   s5   e  Z d  „  Z d „  Z d „  Z d „  Z d „  Z RS(   c         C   s   t  ƒ  |  _ d  S(   N(   R    R   (   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyR   p   s    c         C   sB   | j  t i |  j d 6ƒ ƒ |  _ t |  j _ |  j j d  ƒ d  S(   NR   (   R   R
   R   R   R   R   R   R   (   R   t   pipeline_class(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   init_pipelines   s    "c      	   C   s‹   |  j  t ƒ t j d t ƒ f } t j d ƒ |  j |  j j d ƒ d ƒ |  j t	 | ƒ d ƒ |  j
 d t | d j ƒ k ƒ Wd  QXd  S(   Nt   recordt   alwayss!   https://dev.mydeco.com/mydeco.pdfs1   full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdfi   s"   file_key(url) method is deprecatediÿÿÿÿ(   RU   R   t   warningst   catch_warningsR8   t   simplefilterR   R   RR   t   lent
   assertTruet   strt   message(   R   t   w(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   test_default_file_key_methodx   s    c      	   C   s‘   |  j  t ƒ t j d t ƒ l } t j d ƒ |  j |  j j t	 d ƒ ƒ d ƒ |  j t
 | ƒ d ƒ |  j d t | d j ƒ k ƒ Wd  QXd  S(   NRV   RW   s!   https://dev.mydeco.com/mydeco.pdfs2   empty/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdfi   s"   file_key(url) method is deprecatediÿÿÿÿ(   RU   RJ   RX   RY   R8   RZ   R   R   R   R   R[   R\   R]   R^   (   R   R_   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   test_overridden_file_key_method   s    c         C   s   t  |  j ƒ d  S(   N(   R   R   (   R   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyR   Š   s    (   RG   RH   R   RU   R`   Ra   R   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyRS   o   s
   						t   FilesPipelineTestCaseFieldsc           B   s   e  Z d  „  Z d „  Z RS(   c         C   sç   d t  f d „  ƒ  Y} xÊ | t f D]¼ } d } | i d d 6| g d 6ƒ } t j t i d d 6ƒ ƒ } t | j | d  ƒ ƒ } |  j | d	 j	 | ƒ t
 i | d
 6f g } | j | | d  ƒ |  j | d | d	 d g ƒ q# Wd  S(   Nt   TestItemc           B   s#   e  Z e ƒ  Z e ƒ  Z e ƒ  Z RS(    (   RG   RH   R   t   namet	   file_urlsR5   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyRc   ‘   s   		s"   http://www.example.com/files/1.txtt   item1Rd   Re   s   s3://example/files/R   i    RO   R5   i   (   R   t   dictR   R   R
   t   listR4   R   R   RO   R8   t   item_completed(   R   Rc   t   clsRO   R?   R   t   requestst   results(    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   test_item_fields_default   s    c         C   sõ   d t  f d „  ƒ  Y} xØ | t f D]Ê } d } | i d d 6| g d 6ƒ } t j t i d d 6d d	 6d
 d 6ƒ ƒ } t | j | d  ƒ ƒ } |  j | d j	 | ƒ t
 i | d 6f g } | j | | d  ƒ |  j | d
 | d d g ƒ q# Wd  S(   NRc   c           B   s#   e  Z e ƒ  Z e ƒ  Z e ƒ  Z RS(    (   RG   RH   R   Rd   R5   t   stored_file(    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyRc   ¡   s   		s"   http://www.example.com/files/1.txtRf   Rd   R5   s   s3://example/files/R   t   FILES_URLS_FIELDRn   t   FILES_RESULT_FIELDi    RO   i   (   R   Rg   R   R   R
   Rh   R4   R   R   RO   R8   Ri   (   R   Rc   Rj   RO   R?   R   Rk   Rl   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt"   test_item_fields_override_settings    s    (   RG   RH   Rm   Rq   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyRb   Ž   s   	t   ItemWithFilesc           B   s   e  Z e ƒ  Z e ƒ  Z RS(    (   RG   RH   R   Re   R5   (    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyRr   µ   s   	c          G   s   t  ƒ  } |  | d <| S(   NRe   (   Rr   (   R5   R?   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyR6   º   s    	
c      
   C   s)   t  |  d i t |  d d d d ƒd 6ƒS(   NR   t   statusiÈ   t   bodyt   dataR   (   R   R	   (   R>   (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyR:   À   s    t   __main__($   R(   R9   RK   RX   t   tempfileR    t   shutilR   t   twisted.trialR   t   twisted.internetR   t   scrapy.pipelines.filesR   R   t   scrapy.itemR   R   t   scrapy.httpR   R	   t   scrapy.settingsR
   t   testsR   R   t   TestCaseR   RJ   RS   Rb   Rr   R6   R:   RG   t   main(    (    (    s=   /home/travis/build/scrapy/scrapy/tests/test_pipeline_files.pyt   <module>   s,   	P'		