ó
ç'ÊUc           @@  s­  d  d l  m Z d  d l Z d  d l Z d  d l Z d  d l m Z d  d l Z d  d l Z d  d l	 Z	 d  d l
 m Z d  d l m Z d  d l m Z d  d l m Z d  d l m Z d  d	 l m Z d  d
 l m Z d  d l m Z d  d l Z d  d l m Z m Z m Z m Z m  Z  d  d l! m" Z" d e j# f d „  ƒ  YZ$ d e j# f d „  ƒ  YZ% d e j# f d „  ƒ  YZ& d e j# f d „  ƒ  YZ' d e j# f d „  ƒ  YZ( d S(   i    (   t   absolute_importN(   t   BytesIO(   t   urlparse(   t   verifyObject(   t   unittest(   t   defer(   t   CrawlerRunner(   t   Settings(   t
   MockServer(   t   path_to_file_uri(   t   IFeedStoraget   FileFeedStoraget   FTPFeedStoraget   S3FeedStoraget   StdoutFeedStorage(   t   assert_aws_environt   FileFeedStorageTestc           B@  sG   e  Z d  „  Z d „  Z d „  Z d „  Z d „  Z e j d „  ƒ Z	 RS(   c         C@  s:   t  j j |  j ƒ  ƒ } t | ƒ } |  j t | ƒ | ƒ S(   N(   t   ost   patht   abspatht   mktempR	   t   _assert_storesR   (   t   selfR   t   uri(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_store_file_uri   s    c         C@  sU   t  j j |  j ƒ  ƒ } t  j j | d d d ƒ } t | ƒ } |  j t | ƒ | ƒ S(   Nt   moret   pathss   file.txt(   R   R   R   R   t   joinR	   R   R   (   R   R   R   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_store_file_uri_makedirs"   s    c         C@  s.   t  j j |  j ƒ  ƒ } |  j t | ƒ | ƒ S(   N(   R   R   R   R   R   R   (   R   R   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_store_direct_path(   s    c         C@  s"   |  j  ƒ  } |  j t | ƒ | ƒ S(   N(   R   R   R   (   R   R   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_store_direct_path_relative,   s    c         C@  s)   |  j  ƒ  } t | ƒ } t t | ƒ d  S(   N(   R   R   R   R
   (   R   R   t   st(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_interface0   s    c         c@  s„   t  j d ƒ } | j | ƒ } | j d ƒ | j | ƒ V|  j t j j | ƒ ƒ t | d ƒ  } |  j	 | j
 ƒ  d ƒ Wd  QXd  S(   Nt   defaultt   contentt   rb(   t   scrapyt   Spidert   opent   writet   storet
   assertTrueR   R   t   existst   assertEqualt   read(   R   t   storageR   t   spidert   filet   fp(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR   5   s    (
   t   __name__t
   __module__R   R   R   R   R    R   t   inlineCallbacksR   (    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR      s   					t   FTPFeedStorageTestc           B@  s#   e  Z d  „  Z e j d „  ƒ Z RS(   c         C@  sk   t  j j d ƒ } t  j j d ƒ } | o- | sB t j d ƒ ‚ n  t | ƒ } t t | ƒ |  j | | ƒ S(   Nt   FEEDTEST_FTP_URIt   FEEDTEST_FTP_PATHs#   No FTP server available for testing(	   R   t   environt   getR   t   SkipTestR   R   R
   R   (   R   R   R   R   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt
   test_storeB   s    c         c@  sÆ   t  j d ƒ } | j | ƒ } | j d ƒ | j | ƒ V|  j t j j | ƒ ƒ t | d ƒ  } |  j	 | j
 ƒ  d ƒ Wd  QX| j t d ƒ ƒ Vt | d ƒ  } |  j	 | j
 ƒ  d ƒ Wd  QXd  S(   NR!   R"   R#   s   new content(   R$   R%   R&   R'   R(   R)   R   R   R*   R+   R,   R   (   R   R-   R   R.   R/   R0   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR   K   s    (   R1   R2   R:   R   R3   R   (    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR4   @   s   		t   S3FeedStorageTestc           B@  s   e  Z e j d  „  ƒ Z RS(   c         c@  sÚ   t  ƒ  t j j d ƒ } | s1 t j d ƒ ‚ n  d d l m } t | ƒ } t	 t
 | ƒ | j t j d ƒ ƒ } | j d ƒ | j | ƒ Vt | ƒ } | ƒ  j | j d t ƒj | j ƒ } |  j | j ƒ  d ƒ d  S(   Nt   FEEDTEST_S3_URIs   No S3 URI available for testingi    (   t
   connect_s3R!   R"   t   validate(   R   R   R7   R8   R   R9   t   botoR=   R   R   R
   R&   R$   R%   R'   R(   R   t
   get_buckett   hostnamet   Falset   get_keyR   R+   t   get_contents_as_string(   R   R   R=   R-   R/   t   ut   key(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR:   \   s    '(   R1   R2   R   R3   R:   (    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR;   Z   s   t   StdoutFeedStorageTestc           B@  s   e  Z e j d  „  ƒ Z RS(   c         c@  sh   t  ƒ  } t d d | ƒ} | j t j d ƒ ƒ } | j d ƒ | j | ƒ V|  j | j ƒ  d ƒ d  S(   Ns   stdout:t   _stdoutR!   R"   (	   R   R   R&   R$   R%   R'   R(   R+   t   getvalue(   R   t   outR-   R/   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR:   o   s    	(   R1   R2   R   R3   R:   (    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyRG   m   s   t   FeedExportTestc           B@  sß   e  Z e j Z d  e j f d „  ƒ  YZ e j	 d d „ ƒ Z e j	 d „  ƒ Z e j	 d e d „ ƒ Z e j	 d d „ ƒ Z e j	 d e d „ ƒ Z e j	 d „  ƒ Z e j	 d „  ƒ Z e j	 d	 „  ƒ Z e j	 d
 „  ƒ Z RS(   t   MyItemc           B@  s,   e  Z e j ƒ  Z e j ƒ  Z e j ƒ  Z RS(    (   R1   R2   R$   t   Fieldt   foot   eggt   baz(    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyRL   }   s   c   	      c@  s·   t  j ƒ  } | d } i d | d 6d d 6} | j | p= i  ƒ za t ƒ  ' } t t | ƒ ƒ } | j | ƒ VWd QXt | d ƒ  } t j	 | j
 ƒ  ƒ Wd QXWd t j | ƒ Xd S(   s;    Run spider with specified settings; return exported data. s   /ress   file://t   FEED_URIt   csvt   FEED_FORMATNR#   (   t   tempfilet   mkdtempt   updateR   R   R   t   crawlR&   R   t   returnValueR,   t   shutilt   rmtree(	   R   t
   spider_clst   settingst   tmpdirt   res_namet   defaultst   st   runnert   f(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   run_and_export‚   s    

c         #@  sC   d t  j f ‡  f d †  ƒ  Y} |  j | | ƒ V} t j | ƒ d S(   sV   
        Return exported data which a spider yielding ``items`` would return.
        t
   TestSpiderc           @  s&   e  Z d  Z d g Z ‡  f d †  Z RS(   t
   testspiders   http://localhost:8998/c         3@  s   x ˆ  D] } | Vq Wd  S(   N(    (   R   t   responset   item(   t   items(    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   parse    s    (   R1   R2   t   namet
   start_urlsRi   (    (   Rh   (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyRd   œ   s   	N(   R$   R%   Rc   R   RX   (   R   Rh   R\   Rd   t   data(    (   Rh   s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   exported_data—   s    c   	      c@  s£   | p	 i  } | j  i d d 6ƒ |  j | | ƒ V} t j | j ƒ  ƒ } t | ƒ } | rp |  j | j | ƒ n |  j t | j ƒ t | ƒ ƒ |  j | | ƒ d  S(   NRR   RS   (	   RV   Rm   RR   t
   DictReadert
   splitlinest   listR+   t
   fieldnamest   set(	   R   Rh   t   headert   rowsR\   t   orderedRl   t   readert   got_rows(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   assertExportedCsv§   s    c         c@  s˜   | p	 i  } | j  i d d 6ƒ |  j | | ƒ V} g  | j ƒ  D] } t j | ƒ ^ q@ } g  | D] } d „  | j ƒ  Dƒ ^ qb } |  j | | ƒ d  S(   Nt   jlRS   c         S@  s%   i  |  ] \ } } | r | | “ q S(    (    (   t   .0t   kt   v(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pys
   <dictcomp>¼   s   	 (   RV   Rm   Ro   t   jsont   loadsRh   R+   (   R   Rh   Rt   R\   Rl   t   linet   parsedt   row(    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   assertExportedJsonLines¶   s    ()c         c@  s2   |  j  | | | | | ƒ V|  j | | | ƒ Vd  S(   N(   Rx   R‚   (   R   Rh   Rs   Rt   R\   Ru   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   assertExported¿   s    c         c@  s§   |  j  i d d 6d d 6ƒ |  j  i d d 6d d 6d d 6ƒ g } i d d 6d d 6d	 d 6i d d 6d d 6d d 6g } |  j  j j ƒ  } |  j | | | d
 t ƒVd  S(   Nt   bar1RN   t   spam1RO   t   bar2t   spam2t   quux2RP   t    Ru   (   RL   t   fieldst   keysRƒ   RB   (   R   Rh   Rt   Rs   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_export_itemsÄ   s    'c   	      c@  s%  d t  j f d „  ƒ  Y} |  j i d d 6d d 6ƒ | i d d 6d	 d 6ƒ |  j i d
 d 6d d 6d d 6ƒ i d d 6d d 6g } |  j j j ƒ  } i d d 6d d 6d d 6i d d 6d	 d 6d d 6i d d 6d
 d 6d d 6i d d 6d d 6d d 6g } g  | D] } t | ƒ ^ q} |  j | | | d t ƒV|  j | | ƒ Vi g  d 6} |  j | | | d t ƒV|  j | | | ƒ Vd d d g } i | d 6} i d d 6d d 6d d 6i d	 d 6d d 6d d 6i d
 d 6d d 6d d 6i d d 6d d 6d d 6g } |  j	 | | | d | d t
 ƒVd  S(   Nt   MyItem2c           B@  s    e  Z e j ƒ  Z e j ƒ  Z RS(    (   R1   R2   R$   RM   RN   t   hello(    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyR   Õ   s   R„   RN   R…   RO   t   world2RŽ   R†   t   bar3t   spam3t   quux3RP   t   world4t   spam4R‰   Ru   t   FEED_EXPORT_FIELDSR\   (   R$   t   ItemRL   RŠ   R‹   t   dictRx   RB   R‚   Rƒ   t   True(	   R   R   Rh   Rs   t   rows_csvR   t   rows_jlR\   Rt   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt!   test_export_multiple_item_classesÒ   s0    !c         c@  s’   i d d 6d d 6i d d 6d d 6d d 6g } i d d 6d d 6i d d 6d d 6g } | } |  j  | d d g | d t ƒV|  j | | ƒ Vd  S(   Nt   barRN   t   spamRO   t   quuxRP   Ru   (   Rx   RB   R‚   (   R   Rh   R™   Rš   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_export_dictsþ   s     c      	   c@  s   x|  j  t g D]} | i d d 6d d 6ƒ | i d d 6d d 6d d 6ƒ g } i d	 d
 6} i d d 6d d 6d d 6i d d 6d d 6d d 6g } |  j | d d d g | d | d t ƒVi d d
 6} i d d 6d d 6i d d 6d d 6g } |  j | d d g | d | d t ƒVq Wd  S(   NR„   RN   R…   RO   R†   R‡   Rˆ   RP   s   foo,baz,eggR•   R‰   R\   Ru   s   egg,baz(   RL   R—   Rƒ   R˜   (   R   t   item_clsRh   R\   Rt   (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   test_export_feed_export_fields  s    $N(   R1   R2   t   sixt   PY2t   skipR$   R–   RL   R   R3   t   NoneRc   Rm   R˜   Rx   R‚   Rƒ   RŒ   R›   RŸ   R¡   (    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyRK   y   s   
,()   t
   __future__R    R   RR   R}   t   ioR   RT   RY   R¢   t   six.moves.urllib.parseR   t   zope.interface.verifyR   t   twisted.trialR   t   twisted.internetR   t   scrapy.crawlerR   t   scrapy.settingsR   t   tests.mockserverR   t	   w3lib.urlR	   R$   t   scrapy.extensions.feedexportR
   R   R   R   R   t   scrapy.utils.testR   t   TestCaseR   R4   R;   RG   RK   (    (    (    s9   /home/travis/build/scrapy/scrapy/tests/test_feedexport.pyt   <module>   s.   (%