ó
ç'ÊUc           @  sº  d  d l  m Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l m	 Z	 d  d l
 Z
 d  d l m Z m Z m Z d  d l m Z d  d l m Z d  d l m Z d  d l m Z d  d	 l m Z d
 e j f d „  ƒ  YZ d e f d „  ƒ  YZ d e f d „  ƒ  YZ d e f d „  ƒ  YZ d e f d „  ƒ  YZ d e f d „  ƒ  YZ d e f d „  ƒ  YZ  d e f d „  ƒ  YZ! d e f d „  ƒ  YZ" e# d k r¶e j$ ƒ  n  d S(   iÿÿÿÿ(   t   print_functionN(   t   contextmanager(   t   Responset   HtmlResponset   Request(   t   Spider(   t   Settings(   t   IgnoreRequest(   t   get_crawler(   t   HttpCacheMiddlewaret	   _BaseTestc           B  s€   e  Z d  Z d Z d „  Z d „  Z d „  Z e d „  ƒ Z e d „  ƒ Z	 e d „  ƒ Z
 d „  Z d	 „  Z d
 „  Z d „  Z RS(   s+   scrapy.extensions.httpcache.DbmCacheStorages)   scrapy.extensions.httpcache.RFC2616Policyc         C  sá   t  j j t j ƒ  d ƒ |  _ t  j j ƒ  |  _ t  j j t j ƒ  d ƒ |  _ t t ƒ |  _	 |  j	 j
 d ƒ |  _ t j ƒ  |  _ t d d i d d 6ƒ|  _ t d d i d d 6d	 d
 d d ƒ|  _ |  j	 j j |  j ƒ d  S(   Ni€Q s   example.coms   http://www.example.comt   headerst   tests
   User-Agents	   text/htmls   Content-Typet   bodys	   test bodyt   statusiÊ   (   t   emailt   utilst
   formatdatet   timet	   yesterdayt   todayt   tomorrowR   R   t   crawlert   _create_spidert   spidert   tempfilet   mkdtempt   tmpdirR   t   requestR   t   responset   statst   open_spider(   t   self(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   setUp   s    		c         C  s-   |  j  j j |  j d ƒ t j |  j ƒ d  S(   Nt    (   R   R   t   close_spiderR   t   shutilt   rmtreeR   (   R    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   tearDown&   s    c         K  sP   i t  d 6|  j d 6d d 6g  d 6|  j d 6|  j d 6} | j | ƒ t | ƒ S(   Nt   HTTPCACHE_ENABLEDt   HTTPCACHE_DIRi   t   HTTPCACHE_EXPIRATION_SECSt   HTTPCACHE_IGNORE_HTTP_CODESt   HTTPCACHE_POLICYt   HTTPCACHE_STORAGE(   t   TrueR   t   policy_classt   storage_classt   updateR   (   R    t   new_settingst   settings(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   _get_settings*   s    

c         k  s$   |  j  |    } | j VWd  QXd  S(   N(   t   _middlewaret   storage(   R    R1   t   mw(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   _storage6   s    c         k  s$   |  j  |    } | j VWd  QXd  S(   N(   R4   t   policy(   R    R1   R6   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   _policy;   s    c         k  sU   |  j  |   } t | |  j j ƒ } | j |  j ƒ z	 | VWd  | j |  j ƒ Xd  S(   N(   R3   R	   R   R   t   spider_openedR   t   spider_closed(   R    R1   R2   R6   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyR4   @   s    	c         C  s\   |  j  | j | j ƒ |  j  | j | j ƒ |  j  | j | j ƒ |  j  | j | j ƒ d  S(   N(   t   assertEqualt   urlR   R   R   (   R    t	   response1t	   response2(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   assertEqualResponseJ   s    c         C  sF   |  j  | j | j ƒ |  j  | j | j ƒ |  j  | j | j ƒ d  S(   N(   R<   R=   R   R   (   R    t   request1t   request2(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   assertEqualRequestP   s    c           s|   |  j  | j ˆ  j ƒ d | j k s+ t ‚ d | j k s@ t ‚ t ‡  f d †  d Dƒ ƒ sb t ‚ |  j  | j ˆ  j ƒ d  S(   Ns   If-None-Matchs   If-Modified-Sincec         3  s   |  ] } | ˆ  j  k Vq d  S(   N(   R   (   t   .0t   h(   RB   (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pys	   <genexpr>Y   s    (   s   If-None-Matchs   If-Modified-Since(   R<   R=   R   t   AssertionErrort   anyR   (   R    RA   RB   (    (   RB   sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt(   assertEqualRequestButWithCacheValidatorsU   s
    "c      	   C  sô   |  j  ƒ  X } t |  j j d <| j |  j |  j |  j ƒ |  j | j j	 |  j |  j ƒ d  ƒ Wd  QX|  j  ƒ  | } t |  j j d <| j |  j |  j |  j ƒ | j j |  j |  j ƒ rê |  j | j j	 |  j |  j ƒ |  j j ƒ n  Wd  QXd  S(   Nt
   dont_cache(   R4   R-   R   t   metat   process_responseR   R   R<   R5   t   retrieve_responset   Nonet   FalseR8   t   should_cache_responset   assertIsInstancet	   __class__(   R    R6   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_dont_cache\   s    +(   t   __name__t
   __module__R/   R.   R!   R&   R3   R   R7   R9   R4   R@   RC   RH   RR   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyR
      s   			
			t   DefaultStorageTestc           B  s   e  Z d  „  Z d „  Z RS(   c         C  sÐ   |  j  ƒ  ¾ } |  j j ƒ  } | j |  j | ƒ d  k s? t ‚ | j |  j |  j |  j ƒ | j |  j | ƒ } t	 | t
 ƒ s… t ‚ |  j |  j | ƒ t j d ƒ | j |  j | ƒ d  k sÆ t ‚ Wd  QXd  S(   Ni   (   R7   R   t   copyRL   R   RM   RF   t   store_responseR   t
   isinstanceR   R@   R   t   sleep(   R    R5   RB   R?   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_storagek   s    !c         C  sŠ   |  j  d d ƒ r } | j |  j |  j ƒ d  k s9 t ‚ | j |  j |  j |  j ƒ t j	 d ƒ | j |  j |  j ƒ s€ t ‚ Wd  QXd  S(   NR)   i    g      à?(
   R7   RL   R   R   RM   RF   RW   R   R   RY   (   R    R5   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_storage_never_expirex   s
    $(   RS   RT   RZ   R[   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyRU   i   s   	t   DbmStorageTestc           B  s   e  Z d  Z RS(   s+   scrapy.extensions.httpcache.DbmCacheStorage(   RS   RT   R/   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyR\   €   s   t!   DbmStorageWithCustomDbmModuleTestc           B  s    e  Z d  Z d „  Z d „  Z RS(   s   tests.mocks.dummydbmc         K  s)   | j  d |  j ƒ t t |  ƒ j |   S(   Nt   HTTPCACHE_DBM_MODULE(   t
   setdefaultt
   dbm_modulet   superR]   R3   (   R    R1   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyR3   ‰   s    c         C  s2   |  j  ƒ    } |  j | j j |  j ƒ Wd  QXd  S(   N(   R7   R<   t   dbmoduleRS   R`   (   R    R5   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_custom_dbm_module_loaded   s    (   RS   RT   R`   R3   Rc   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyR]   …   s   	t   FilesystemStorageTestc           B  s   e  Z d  Z RS(   s2   scrapy.extensions.httpcache.FilesystemCacheStorage(   RS   RT   R/   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyRd   “   s   t   FilesystemStorageGzipTestc           B  s   e  Z d  „  Z RS(   c         K  s&   | j  d t ƒ t t |  ƒ j |   S(   Nt   HTTPCACHE_GZIP(   R_   R-   Ra   Rd   R3   (   R    R1   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyR3   ™   s    (   RS   RT   R3   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyRe   —   s   t   LeveldbStorageTestc           B  s   e  Z e j d  ƒ d Z RS(   t   leveldbs/   scrapy.extensions.httpcache.LeveldbCacheStorage(   RS   RT   t   pytestt   importorskipR/   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyRg      s   t   DummyPolicyTestc           B  s;   e  Z d  Z d „  Z d „  Z d „  Z d „  Z d „  Z RS(   s'   scrapy.extensions.httpcache.DummyPolicyc         C  s®   |  j  ƒ  œ } | j |  j |  j ƒ d  k s3 t ‚ | j |  j |  j |  j ƒ | j |  j |  j ƒ } t | t	 ƒ s| t ‚ |  j
 |  j | ƒ d | j k s¤ t ‚ Wd  QXd  S(   Nt   cached(   R4   t   process_requestR   R   RM   RF   RK   R   RX   R   R@   t   flags(   R    R6   R   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_middleware§   s    $c         C  s·   |  j  ƒ  ¥ } t d ƒ } t d ƒ } | j | |  j ƒ d  k sH t ‚ | j | | |  j ƒ | j | |  j ƒ } t | t ƒ sˆ t ‚ |  j	 | | ƒ d | j
 k s­ t ‚ Wd  QXd  S(   Ns   http://host.com/paths   http://host2.net/test.htmlRl   (   R4   R   R   Rm   R   RM   RF   RK   RX   R@   Rn   (   R    R6   t   reqt   resRl   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt$   test_different_request_response_urls°   s    !c      	   C  s¯   |  j  d t ƒ — } |  j t | j |  j |  j ƒ | j |  j |  j |  j ƒ | j |  j |  j ƒ } t	 | t
 ƒ s} t ‚ |  j |  j | ƒ d | j k s¥ t ‚ Wd  QXd  S(   Nt   HTTPCACHE_IGNORE_MISSINGRl   (   R4   R-   t   assertRaisesR   Rm   R   R   RK   R   RX   R   RF   R@   Rn   (   R    R6   R   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_middleware_ignore_missing»   s    c         C  sá  t  d ƒ t d ƒ } } |  j ƒ  ™ } | j | |  j ƒ d  k sI t ‚ | j | | |  j ƒ | j | |  j ƒ } t | t ƒ s• t t	 | ƒ ƒ ‚ |  j
 | | ƒ d | j k sº t ‚ Wd  QXt  d ƒ t d ƒ } } |  j ƒ  ƒ } | j | |  j ƒ d  k s	t ‚ | j | | |  j ƒ | j j |  j | ƒ d  k sCt ‚ | j | |  j ƒ d  k sdt ‚ Wd  QXt  d ƒ t d ƒ } } |  j ƒ  ™ } | j | |  j ƒ d  k s³t ‚ | j | | |  j ƒ | j | |  j ƒ } t | t ƒ sÿt t	 | ƒ ƒ ‚ |  j
 | | ƒ d | j k s$t ‚ Wd  QXt  d ƒ t d ƒ } } |  j d d	 g ƒ ƒ } | j | |  j ƒ d  k s|t ‚ | j | | |  j ƒ | j j |  j | ƒ d  k s¶t ‚ | j | |  j ƒ d  k s×t ‚ Wd  QXd  S(
   Ns   http://test.com/Rl   s   file:///tmp/t.txts   s3://bucket/keys   http://bucket/keys   s3://bucket/key2s   http://bucket/key2t   HTTPCACHE_IGNORE_SCHEMESt   s3(   R   R   R4   Rm   R   RM   RF   RK   RX   t   typeR@   Rn   R5   RL   (   R    Rp   Rq   R6   Rl   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_middleware_ignore_schemesÄ   s8    !!!$'!!!$c      	   C  s<  |  j  d d g ƒ ’ } | j |  j |  j ƒ d  k s< t ‚ | j |  j |  j |  j ƒ | j j	 |  j |  j ƒ d  k s t ‚ | j |  j |  j ƒ d  k s£ t ‚ Wd  QX|  j  d d g ƒ x } | j |  j |  j |  j ƒ | j |  j |  j ƒ } t
 | t ƒ s
t ‚ |  j |  j | ƒ d | j k s2t ‚ Wd  QXd  S(   NR*   iÊ   iË   Rl   (   R4   Rm   R   R   RM   RF   RK   R   R5   RL   RX   R   R@   Rn   (   R    R6   R   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt!   test_middleware_ignore_http_codesí   s    $'*(   RS   RT   R.   Ro   Rr   Ru   Ry   Rz   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyRk   £   s   						)t   RFC2616PolicyTestc           B  s;   e  Z d  Z d „  Z d „  Z d „  Z d „  Z d „  Z RS(   s)   scrapy.extensions.httpcache.RFC2616Policyc         C  s±   yo | j  | |  j ƒ } | r= t | t t f ƒ s9 t ‚ | S| j | | |  j ƒ } t | t ƒ sj t ‚ | SWn; t k
 r¬ t d | ƒ t d | ƒ t d | ƒ ‚  n Xd  S(   NR   R   t   Result(	   Rm   R   RX   R   R   RF   RK   t	   Exceptiont   print(   R    R6   R   R   t   result(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   _process_requestresponse  s    c         C  s²  t  |  j j d d d i |  j d 6ƒ} t d ƒ } | j d i d d 6ƒ } | j d i d d 6ƒ } |  j ƒ  :} |  j | | | ƒ } |  j | | ƒ | j	 j
 |  j | ƒ d  k s¾ t ‚ |  j | | | ƒ } d	 | j k sè t ‚ | j | |  j ƒ } d	 | j k st ‚ |  j | | ƒ | j d
 d ƒ }	 |  j | | |	 ƒ }
 |  j |
 |	 ƒ d	 |
 j k snt ‚ |  j | | d  ƒ } |  j | |	 ƒ d	 | j k s¨t ‚ Wd  QXd  S(   NR   iÈ   R   t   Expiress   http://example.coms   no-stores   Cache-Controls   no-cacheRl   R   t   foo(   R   R   R=   R   R   t   replaceR4   R€   R@   R5   RL   R   RM   RF   Rn   Rm   (   R    t   res0t   req0t   req1t   req2R6   t   res1t   res2t   res3t   res0bt   res4t   res5(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_request_cacheability  s*    $c         C  sÊ  t  d i  f t  d i |  j d 6f t  d i |  j d 6f t  d i d d 6f t  d i d d 6f t  d i d	 d 6f t  d i d
 d 6f t  d i d	 d 6|  j d 6f t  d i  f t  d i  f t  d i  f t  d i  f t d i |  j d 6f t d i |  j d 6f t d i |  j d 6f t d i |  j d 6f t d i |  j d 6f t d i d d 6f t d i |  j d 6f t d i d d 6f g } |  j ƒ  7} x-t | ƒ D]\ } \ } } } t d | ƒ } t | j d | d | ƒ} |  j	 | | | ƒ }	 | j
 d d ƒ }
 |  j	 | | | r|
 n | ƒ } |  j |	 | ƒ |  j | | ƒ | j j |  j | ƒ } | rš|  j | |	 ƒ d | j k r‘| j d k s¼t ‚ q|  j | ƒ d | j k st ‚ qWWd  QXd  S(   Ni0  s   Last-ModifiedR   t   bart   Etags   max-age=3600s   Cache-ControliÈ   s   no-stores   no-store, max-age=300i.  i3  i”  iË   i,  i-  i‘  s   public, max-age=600R‚   s   http://example-%d.comR   R   Rl   (   RN   R   R   R-   R4   t	   enumerateR   R   R=   R€   Rƒ   R@   R5   RL   R   Rn   R   RF   t   assertFalse(   R    t	   responsesR6   t   idxt   shouldcacheR   R   R…   R„   Rˆ   t   res304R‰   t   resc(    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_response_cacheability.  sF    "!'c   
      C  s  d i |  j  d 6|  j d 6f d i |  j  d 6d d 6f d i d d 6d d 6f d i |  j d 6d	 d 6d
 t d ƒ d 6|  j  d 6|  j  d 6f d i |  j  d 6d d 6d d 6|  j d 6|  j  d 6f d i |  j d 6f d i |  j d 6|  j  d 6t d d d ƒ d 6f d i  f d i  f d i  f g
 } |  j ƒ  É } x¿ t | ƒ D]± \ } \ } } t d | ƒ } t | j d | d | ƒ} |  j	 | | | ƒ } |  j
 | | ƒ d | j k s»t ‚ |  j	 | | d  ƒ }	 |  j
 | |	 ƒ d |	 j k sDt ‚ qDWWd  QXd  S(   NiÈ   t   DateR   s   max-age=86405s   Cache-Controlt   299t   Ages   max-age=300t   86405s   max-age=i€Q i   s   Last-Modifiedt   86400t   publici
   i   i,  i-  i4  s   http://example-%d.comR   R   Rl   i€ô (   R   R   R   t   strR4   R‘   R   R   R=   R€   R@   Rn   RF   RM   (
   R    t
   sampledataR6   R”   R   R   R…   R„   Rˆ   R‰   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_cached_and_freshZ  s4    

2!c         C  sZ  d i |  j  d 6|  j d 6f d i |  j  d 6|  j d 6|  j d 6f d i |  j d 6f d i |  j d 6d d 6f d i |  j d 6|  j d 6f d i |  j d 6d d 6f d i d	 d
 6d d 6f d i d d
 6f d i d d
 6d d 6f d i d d
 6|  j d 6f g
 } |  j ƒ  B} x8t | ƒ D]*\ } \ } } t d | ƒ } t | j d | d | ƒ} |  j | | | ƒ } |  j	 | | ƒ d | j
 k s™t ‚ | j d d ƒ }	 |  j | | |	 ƒ }
 |  j	 |
 |	 ƒ d |
 j
 k såt ‚ d | k sýd | k r"|	 j d d ƒ } |  j | | | ƒ } |  j	 | |	 ƒ d | j
 k sLt ‚ q"q"WWd  QXd  S(   NiÈ   R™   R   s   Last-ModifiedR‚   t   ETagRœ   R›   s   max-age=86400s   Cache-Controls   no-caches   http://example-%d.comR   R   Rl   R   R   i0  (   R   R   R   R4   R‘   R   R   R=   R€   R@   Rn   RF   Rƒ   (   R    R    R6   R”   R   R   R…   t   res0aRˆ   R‹   R‰   t   res0cRŠ   (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   test_cached_and_stale  s4    ' (   RS   RT   R.   R€   RŽ   R˜   R¡   R¥   (    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyR{   ÿ   s   			,	't   __main__(%   t
   __future__R    R   R   R$   t   unittestt   email.utilsR   t
   contextlibR   Ri   t   scrapy.httpR   R   R   t   scrapy.spidersR   t   scrapy.settingsR   t   scrapy.exceptionsR   t   scrapy.utils.testR   t&   scrapy.downloadermiddlewares.httpcacheR	   t   TestCaseR
   RU   R\   R]   Rd   Re   Rg   Rk   R{   RS   t   main(    (    (    sM   /home/travis/build/scrapy/scrapy/tests/test_downloadermiddleware_httpcache.pyt   <module>   s0   W\¨