ó
ç'ÊUc           @  s
  d  Z  d d l m Z d d l Z d d l Z d d l Z d d l m Z d d l m	 Z	 m
 Z
 d d l m Z m Z m Z d d l m Z d d l m Z d d	 l m Z d d
 l m Z d d l m Z d d l m Z d d l m Z m Z d d l m Z d d l  m! Z! d d l" m# Z# d e f d „  ƒ  YZ$ d e f d „  ƒ  YZ% d e% f d „  ƒ  YZ& d e% f d „  ƒ  YZ' e( d „ Z) d e* f d „  ƒ  YZ+ d e j, f d „  ƒ  YZ- e. d k re/ e j0 ƒ d k re j0 d d  k re) d! e1 ƒ e	 j2 ƒ  qn  d S("   s  
Scrapy engine tests

This starts a testing web server (using twisted.server.Site) and then crawls it
with the Scrapy crawler.

To view the testing web server in a browser you can start it by running this
module with the ``runserver`` argument::

    python test_engine.py runserver
iÿÿÿÿ(   t   print_functionN(   t   urlparse(   t   reactort   defer(   t   servert   statict   util(   t   unittest(   t   signals(   t   get_crawler(   t
   dispatcher(   t   tests_datadir(   t   Spider(   t   Itemt   Field(   t   LinkExtractor(   t   Request(   t   disconnect_allt   TestItemc           B  s#   e  Z e ƒ  Z e ƒ  Z e ƒ  Z RS(    (   t   __name__t
   __module__R   t   namet   urlt   price(    (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR       s   		t
   TestSpiderc           B  sk   e  Z d  Z d  d g Z e j d ƒ Z e j d e j ƒ Z e j d e j ƒ Z	 e
 Z d „  Z d „  Z RS(   s   scrapytest.orgt	   localhosts   item\d+.htmls   <h1>(.*?)</h1>s   >Price: \$(.*?)<c         c  sh   t  ƒ  } t j |  j ƒ } xF | j | ƒ D]5 } | j | j ƒ r+ t d | j d |  j ƒ Vq+ q+ Wd  S(   NR   t   callback(	   R   t   ret   compilet
   itemurl_ret   extract_linkst   searchR   R   t
   parse_item(   t   selft   responset   xlinkt   itemret   link(    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   parse0   s
    	c         C  s   |  j  ƒ  } |  j j | j ƒ } | r= | j d ƒ | d <n  | j | d <|  j j | j ƒ } | r{ | j d ƒ | d <n  | S(   Ni   R   R   R   (   t   item_clst   name_reR   t   bodyt   groupR   t   price_re(   R!   R"   t   itemt   m(    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR    7   s    (   R   R   R   t   allowed_domainsR   R   R   t   MR(   R+   R   R'   R&   R    (    (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR   &   s   	t   TestDupeFilterSpiderc           B  s   e  Z d  „  Z RS(   c         C  s
   t  | ƒ S(   N(   R   (   R!   R   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   make_requests_from_urlD   s    (   R   R   R1   (    (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR0   C   s   t   DictItemsSpiderc           B  s   e  Z e Z RS(    (   R   R   t   dictR'   (    (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR2   H   s   c         C  sž   t  j j t d ƒ } t j | ƒ } | j d t j d ƒ ƒ | j d t j	 d d ƒ ƒ t
 j d t j | ƒ d d	 ƒ} |  rš t d
 | j ƒ  j ƒ n  | S(   Nt	   test_sitet   redirects   /redirectedt
   redirecteds   Redirected heres
   text/plaini    t	   interfaces	   127.0.0.1sC   Test server running at http://localhost:%d/ - hit Ctrl-C to finish.(   t   ost   patht   joinR   R   t   Filet   putChildR   t   Redirectt   DataR   t	   listenTCPR   t   Sitet   printt   getHostt   port(   t   debugt   root_dirt   rRC   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   start_test_siteL   s    !t
   CrawlerRunc           B  sh   e  Z d  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z	 d „  Z
 d	 „  Z d
 „  Z RS(   s<   A class to run the crawler and keep track of events occurredc         C  sC   d  |  _ g  |  _ g  |  _ g  |  _ g  |  _ i  |  _ | |  _ d  S(   N(   t   Nonet   spidert   respplugt   reqplugt
   reqdroppedt   itemrespt   signals_catchedt   spider_class(   R!   RP   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   __init__\   s    						c         C  sc  t  ƒ  |  _ |  j j ƒ  j |  _ |  j d ƒ |  j d ƒ |  j d ƒ g } xE t t ƒ j ƒ  D]1 \ } } | j d ƒ s^ t	 j
 |  j | ƒ q^ q^ Wt |  j ƒ |  _ |  j j j
 |  j t j ƒ |  j j j
 |  j t j ƒ |  j j j
 |  j t j ƒ |  j j j
 |  j t j ƒ |  j j d | ƒ |  j j |  _ t j ƒ  |  _ t	 j
 |  j t j ƒ |  j S(   Nt   /s	   /redirectt   _t
   start_urls(   RG   RC   RB   t   portnot   geturlt   varsR   t   itemst
   startswithR
   t   connectt   record_signalR	   RP   t   crawlert   item_scrapedt   request_scheduledt   request_droppedt   response_downloadedt   crawlRJ   R   t   Deferredt   deferredt   stopt   engine_stopped(   R!   RT   R   t   signal(    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   rune   s"    c         C  s`   |  j  j ƒ  x< t t ƒ j ƒ  D]( \ } } | j d ƒ s  t | ƒ q  q  W|  j j d  ƒ d  S(   NRS   (
   RC   t   stopListeningRW   R   RX   RY   R   Rc   R   RI   (   R!   R   Rf   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRd   |   s
    c         C  s   d |  j  | f S(   Ns   http://localhost:%s%s(   RU   (   R!   R9   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRV   ƒ   s    c         C  s   t  | ƒ } | j S(   N(   R   R9   (   R!   R   t   u(    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   getpath†   s    c         C  s   |  j  j | | f ƒ d  S(   N(   RN   t   append(   R!   R,   RJ   R"   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR]   Š   s    c         C  s   |  j  j | | f ƒ d  S(   N(   RL   Rk   (   R!   t   requestRJ   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR^      s    c         C  s   |  j  j | | f ƒ d  S(   N(   RM   Rk   (   R!   Rl   RJ   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR_      s    c         C  s   |  j  j | | f ƒ d  S(   N(   RK   Rk   (   R!   R"   RJ   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR`   “   s    c         O  s<   | j  ƒ  } | j d ƒ } | j d d ƒ | |  j | <d S(   s"   Record a signal and its parametersRf   t   senderN(   t   copyt   popRI   RO   (   R!   t   argst   kwargst
   signalargst   sig(    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR[   –   s    (   R   R   t   __doc__RQ   Rg   Rd   RV   Rj   R]   R^   R_   R`   R[   (    (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRH   Y   s   										t
   EngineTestc           B  sS   e  Z e j d  „  ƒ Z d „  Z d d „ Z d „  Z d „  Z	 d „  Z
 d „  Z RS(   c         c  s§   xi t  t f D][ } t | ƒ |  _ |  j j ƒ  V|  j ƒ  |  j d d ƒ |  j ƒ  |  j ƒ  |  j ƒ  q Wt t	 ƒ |  _ |  j j ƒ  V|  j d d ƒ |  j
 ƒ  d  S(   Nt   urls_to_visiti   i   (   R   R2   RH   Rg   t   _assert_visited_urlst   _assert_scheduled_requestst   _assert_downloaded_responsest   _assert_scraped_itemst   _assert_signals_catchedR0   t   _assert_dropped_requests(   R!   RJ   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   test_crawler    s    


c         C  s™   d d d d d d g } t  g  |  j j D] } | d j ^ q( ƒ } t  g  | D] } |  j j | ƒ ^ qN ƒ } | | k s• t d t | | ƒ ƒ ‚ d  S(	   NRR   s	   /redirects   /redirecteds   /item1.htmls   /item2.htmls   /item999.htmli    s   URLs not visited: %s(   t   setRg   RK   R   RV   t   AssertionErrort   list(   R!   t   must_be_visitedt   rpt   urls_visitedt   pt   urls_expected(    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRw   ±   s
    	,+c   
      C  sâ   |  j  | t |  j j ƒ ƒ d d d g } t g  |  j j D] } | d j ^ q; ƒ } t g  | D] } |  j j | ƒ ^ qa ƒ } | | k s” t ‚ t |  j j ƒ } t |  j j ƒ } t |  j j	 ƒ }	 |  j  | | |	 ƒ d  S(   Ns   /item999.htmls   /item2.htmls   /item1.htmli    (
   t   assertEqualt   lenRg   RL   R~   R   RV   R   RM   RK   (
   R!   Rv   t   paths_expectedt   rqt   urls_requestedR„   R…   t   scheduled_requests_countt   dropped_requests_countt   responses_count(    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRx   ¸   s    ,+	c         C  s    |  j  t |  j j ƒ d ƒ d  S(   Ni   (   R†   R‡   Rg   RM   (   R!   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR|   Æ   s    c         C  sŸ   |  j  d t |  j j ƒ ƒ x| |  j j D]n \ } } |  j j | j ƒ d k rf |  j  d | j ƒ n  |  j j | j ƒ d k r) |  j  d | j ƒ q) q) Wd  S(   Ni   s   /item999.htmli”  s	   /redirecti.  (   R†   R‡   Rg   RK   Rj   R   t   status(   R!   R"   RS   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRy   É   s    c         C  sÊ   |  j  d t |  j j ƒ ƒ x§ |  j j D]™ \ } } |  j  | d | j ƒ d | d k r‡ |  j  d | d ƒ |  j  d | d ƒ n  d | d k r) |  j  d	 | d ƒ |  j  d
 | d ƒ q) q) Wd  S(   Ni   R   s
   item1.htmls   Item 1 nameR   t   100R   s
   item2.htmls   Item 2 namet   200(   R†   R‡   Rg   RN   R   (   R!   R,   R"   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRz   Ó   s    c         C  s-  t  j |  j j k s t ‚ t  j |  j j k s6 t ‚ t  j |  j j k sQ t ‚ t  j |  j j k sl t ‚ t  j |  j j k s‡ t ‚ |  j	 i |  j j
 d 6|  j j t  j ƒ |  j	 i |  j j
 d 6|  j j t  j ƒ |  j j t  j j d d  ƒ |  j	 i |  j j
 d 6d d 6|  j j t  j ƒ d  S(   NRJ   t   spider_statst   finishedt   reason(   R   t   engine_startedRg   RO   R   Re   t   spider_openedt   spider_idlet   spider_closedR†   RJ   Ro   RI   (   R!   (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyR{   Þ   s    N(   R   R   R   t   inlineCallbacksR}   Rw   RI   Rx   R|   Ry   Rz   R{   (    (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyRu   ž   s   			
	t   __main__i   t	   runserverRD   (3   Rt   t
   __future__R    t   sysR8   R   t   six.moves.urllib.parseR   t   twisted.internetR   R   t   twisted.webR   R   R   t   twisted.trialR   t   scrapyR   t   scrapy.utils.testR	   t   scrapy.xlib.pydispatchR
   t   testsR   t   scrapy.spidersR   t   scrapy.itemR   R   t   scrapy.linkextractorsR   t   scrapy.httpR   t   scrapy.utils.signalR   R   R   R0   R2   t   FalseRG   t   objectRH   t   TestCaseRu   R   R‡   t   argvt   TrueRg   (    (    (    s5   /home/travis/build/scrapy/scrapy/tests/test_engine.pyt   <module>   s4   $EP(