ó
ç'ÊUc @ sè d d l Z d d l m Z d d l m Z m Z m Z m Z m Z m Z d d l
m Z d e j f d „ ƒ YZ
d e
f d „ ƒ YZ d e
f d
„ ƒ YZ d e f d „ ƒ YZ d
e f d „ ƒ YZ e d k rä e j ƒ n d S( iÿÿÿÿN( t resolve_encoding( t Requestt Responset TextResponset HtmlResponset XmlResponset Headers( t Selectort BaseResponseTestc B s_ e Z e Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z
d „ Z d „ Z RS( c
C sû | j t | j ƒ | j t | j d ƒ | j ƒ ƒ | j t | j d d d ƒ| j ƒ ƒ | j t | j d d d ƒ| j ƒ ƒ | j t | j d d i d d d d ƒ| j ƒ ƒ | j d ƒ } t | j t ƒ sà t ‚ | j | j d ƒ | j | j d ƒ t | j
t ƒ s t ‚ | j | j
i ƒ i d d 6} d
} | j d d | d | ƒ} | j
| k stt ‚ | j | j
d d ƒ | j d d d ƒ} | j | j d ƒ | j d d d ƒ} | j | j d ƒ | j t | j d
d d ƒd S( Ns http://example.com/t bodyt t headerst statusiÈ s http://www.example.comt cocot cacas a bodyi- t 301s http://example.comt lala200(
t assertRaisest Exceptiont response_classt
assertTruet
isinstancet urlt strt AssertionErrort assertEqualR R R t
ValueError( t selft rR R ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_init s* "((4
c C sÉ | j d d d ƒ} | j j d ƒ | j ƒ } | j | j | j ƒ | j | j | j ƒ | j | j k s{ t d ƒ ‚ | j | j | j ƒ | j | j k s¯ t d ƒ ‚ | j | j | j ƒ d S( s Test Response copys http://www.example.comR s Some bodyt cacheds+ flags must be a shallow copy, not identicals- headers must be a shallow copy, not identicalN( R t flagst appendt copyR R R R R ( R t r1t r2( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_copy+ s c C sP t d ƒ } d | j d <| j d d d d | ƒ} | j | j k sL t ‚ d S( Ns http://www.example.comt bart fooR s Some bodyt request( R t metaR R ( R t reqR" ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_copy_meta= s
c C sM d | j f d „ ƒ Y} | d ƒ } | j ƒ } t | ƒ | k sI t ‚ d S( s2 Test Response children copies preserve their classt CustomResponsec B s e Z RS( ( t __name__t
__module__( ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyR+ F s s http://www.example.comN( R R! t typeR ( R R+ R" R# ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_copy_inherited_classesC s c C s t i d d 6ƒ } | j d ƒ } | j d d d d d | ƒ } | j d k sU t ‚ | j | j | j ƒ | j | j | j f d ƒ | j | j | j f d ƒ | j | j | j f i | f ƒ | j d d d g ƒ} | j d d d g ƒ } | j | j d ƒ | j | j g ƒ d
S( s Test Response.replace() methodt valuet keys http://www.example.comR i- R s New bodyR R
iÈ R R N( iÈ i- ( R
s New body(
R R t replaceR R R R R R R ( R t hdrsR" R# t r3t r4( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_replaceN s "c C s‘ t | t ƒ r' | } | j | ƒ } n | j | ƒ } | } t | j t ƒ sT t ‚ | j | | ƒ | j | j | ƒ | j | j ƒ | ƒ d S( N(
R t unicodet encodet decodeR R R t _assert_response_encodingR t body_as_unicode( R t responset encodingR t body_unicodet body_str( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt _assert_response_values_ s c C s | j | j t | ƒ ƒ d S( N( R R= R ( R R< R= ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyR: l s c C sE | j d ƒ } | j t t | d d ƒ | j t t | d d ƒ d S( Ns http://example.comR s http://example2.comR t xxx( R R t AttributeErrort setattr( R R ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_immutable_attributeso s c C s2 | j d ƒ j d ƒ } d } | j | | ƒ d S( sI Test urljoin shortcut (only for existence, since behavior equals urljoin)s http://www.example.coms /tests http://www.example.com/testN( R t urljoinR ( R t joinedt absolute( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_urljoint s (
R, R- R R R R$ R* R/ R6 R@ R: RD RH ( ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyR s
t ResponseTextc B s e Z d „ Z RS( c C s | j t | j d ƒ d S( Nu http://www.example.com( R t TypeErrorR ( R ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_no_unicode_url} s ( R, R- RK ( ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyRI { s t TextResponseTestc B sz e Z e Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z d „ Z
d „ Z d „ Z d „ Z
d
„ Z d „ Z RS( c C sÀ t t | ƒ j ƒ | j d d d d d ƒ} | j d d ƒ } | j d d d d ƒ } t | | j ƒ sp t ‚ | j | j d ƒ | j | d ƒ | j | j d ƒ | j | j
ƒ d ƒ d S( Ns http://www.example.comR t helloR= t cp852R s http://www.example.com/othert latin1( t superRL R6 R R2 R R R R R: t _declared_encoding( R R" R# R4 ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyR6 … s c C s | j d ƒ } | j | | j j ƒ | j d d d d ƒ } t | j t ƒ sU t ‚ | j d d d d ƒ } | j | j d ƒ | j d d d d ƒ } | j | j d ƒ | j d d i d
g d 6ƒ} | j | j d ƒ | j d d i d g d 6ƒ} | j | j d ƒ d S(
Nu http://www.example.com/R R= s utf-8u http://www.example.com/price/£s http://www.example.com/price/£s latin-1s http://www.example.com/price/£R s text/html; charset=utf-8s Content-types text/html; charset=iso-8859-1( R R: t _DEFAULT_ENCODINGR R R R R ( R t resp( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_unicode_url‘ s c C s‚ d } | j t | j d d d ƒ| j d ƒ } | j d d | d d ƒ} | j t | j ƒ t ƒ ƒ | j | j ƒ | ƒ d S( Nu% кирилличеÑкий текÑÑ‚s http://www.example.comR u unicode bodyt cp1251R= ( R RJ R R8 R R R; R7 R ( R t unicode_stringt original_stringR" ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_unicode_body£ s c C sL | j d d i d g d 6d d ƒ} | j d d d d d ƒ} | j d d i d
g d 6d d ƒ} | j d d d ƒ} | j d d i d
g d 6d d ƒ} | j d d i d g d 6d d ƒ} | j d d i d g d 6d d ƒ} | j | j ƒ d ƒ | j | j ƒ d ƒ | j | j ƒ d ƒ | j | d ƒ | j | j ƒ d ƒ | j | j ƒ d ƒ | j | j ƒ d ƒ | j | j ƒ d ƒ | j | d ƒ | j ƒ d k rÇ| j ƒ d k sÍt ‚ | j | d d ƒ | j | d d ƒ | j | d d ƒ | j | d d ƒ | j | d d ƒ | j t
| j d d d ƒd S( Ns http://www.example.comR s text/html; charset=utf-8s Content-typeR s £R= s utf-8u £s text/html; charset=iso-8859-1s £s ¢£s text/html; charset=Nones text/html; charset=gb2312s ¨Ds text/html; charset=gbkt cp1252t asciis
iso-8859-1t gb18030u ―( R R t _headers_encodingt NoneRQ R: t _body_inferred_encodingR R@ R RJ ( R R" R# R4 R5 t r5t r6t r7( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt
test_encoding® s. %%%%%*c C sR | j d d i d g d 6d d ƒ} | j | j ƒ d ƒ | j | d d ƒ d S(
s1 Check that unknown declared encodings are ignoreds http://www.example.comR s text/html; charset=UKNOWNs Content-typeR s £s utf-8u £N( R R RQ R] R@ ( R R ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_declared_encoding_invalidÊ s
c C s2 | j d d d d d ƒ} | j | d d ƒ d S( s@ Test utf-16 because UnicodeDammit is known to have problems withs http://www.example.comR s ÿþh i R= s utf-16u hiN( R R@ ( R R ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt
test_utf16Ò s c C sR | j d d i d g d 6d d ƒ} | j | j d ƒ | j | j ƒ d ƒ d S( Ns http://www.example.comR s text/html; charset=utf-8s Content-typeR s WORDã«s utf-8u
WORD��( R R R= R; ( R R` ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt2 test_invalid_utf8_encoded_body_with_valid_utf8_BOM٠s
c C s$ d } d } i d g d 6} | j | d | ƒ} | j | j d ƒ | j | j ƒ d ƒ | j | d | ƒ} | j | j ƒ d ƒ | j | j d ƒ | j | d | d | ƒ} | j | j d ƒ | j | j ƒ d ƒ | j | d | d | ƒ} | j | j ƒ d ƒ | j | j d ƒ d S( Ns http://example.coms WORDs text/html; charset=utf-8s Content-typeR s utf-8u WORDR ( R R R= R; ( R R R R R< ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_bom_is_removed_from_bodyà s c C sâ | j d d d d d ƒ} d | j ƒ k sE t t | j ƒ ƒ ƒ ‚ d | j ƒ k so t t | j ƒ ƒ ƒ ‚ d | j ƒ k s™ t t | j ƒ ƒ ƒ ‚ | j d d d d d
ƒ} d | j ƒ k sÞ t t | j ƒ ƒ ƒ ‚ d S(
s( Test invalid chars are replaced properlys http://www.example.comR= s utf-8R s PREFIXã«SUFFIXu �u PREFIXu SUFFIXs http://example.coms ðvalueu valueN( R R; R t repr( R R ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_replace_wrong_encodingû s *** c C sÝ d } | j d d | ƒ} | j | j t ƒ | j | j j d ƒ | j | j | j ƒ | j | j j | ƒ | j | j j d ƒ j ƒ d g ƒ | j | j j
d ƒ j ƒ d g ƒ | j | j j d ƒ d g ƒ d S(
Ns8 Some pages http://www.example.comR t htmls //title/text()u Some pages title::texts Some (.*)u page( R t assertIsInstancet selectorR R R. t assertIsR< t xpatht extractt csst re( R R R< ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt
test_selector
s
c C s d } | j d d | ƒ} | j | j d ƒ j ƒ | j j d ƒ j ƒ ƒ | j | j d ƒ j ƒ | j j d ƒ j ƒ ƒ d S( Ns8 Some pages http://www.example.comR s //title/text()s title::text( R R Rm Rn Rk Ro ( R R R< ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_selector_shortcuts# s c C s² d } | j d d | ƒj d ƒ } d } | j | | ƒ d } | j d d | ƒj d ƒ } d } | j | | ƒ d } | j d d | ƒj d ƒ } d
} | j | | ƒ d S( sK Test urljoin shortcut which also evaluates base-url through get_base_url().s; s http://www.example.comR s /tests https://example.net/tests2 t tests http://www.example.com/tests3 s% http://www.example.com/elsewhere/testN( R RE R ( R R RF RG ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_urljoin_with_base_url0 s ( R, R- R R R6 RT RX Rb Rc Rd Re Rf Rh Rq Rr Rt ( ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyRL s
t HtmlResponseTestc B s e Z e Z d „ Z d „ Z RS( c C sÉ d } | j d d | ƒ} | j | d | ƒ d } | j d d | ƒ} | j | d | ƒ d } | j d d i d g d 6d | ƒ} | j | d | ƒ d
} | j d | ƒ } | j | d | ƒ d S( Ns¥ Some page
Price: £100'
s http://www.example.comR s
iso-8859-1s²
Price: £100
s Some page
Price: £100'
R s text/html; charset=iso-8859-1s Content-types
New body £( R R@ R2 ( R R R" R# R4 R5 ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_html_encodingF s %c C s2 d } | j d d | ƒ} | j | d | ƒ d S( NsQ Some pagebla blas http://www.example.comR t gb2312( R R@ ( R R R" ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_html5_meta_charseta s ( R, R- R R Rv Rx ( ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyRu B s t XmlResponseTestc B s2 e Z e Z d „ Z d „ Z d „ Z d „ Z RS( c C s² d } | j d d | ƒ} | j | | j j | ƒ d } | j d d | ƒ} | j | d | ƒ d } | j d d | d d ƒ} d } | j d | ƒ } | j | d | ƒ d S( Ns s http://www.example.comR s6 s
iso-8859-1R= s utf-8s New body( R R@ RR R2 ( R R R" R# R4 t body2R5 ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_xml_encodingk s c C sˆ d } d } | j d d | ƒ} | j d | ƒ } | j d | d d ƒ } | j | d | ƒ | j | d | ƒ | j | d | ƒ d S( Ns6 s1 s http://www.example.comR R= s utf-8s
iso-8859-1( R R2 R@ ( R R Rz R_ R` Ra ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt test_replace_encoding| s c C s™ d } | j d d | ƒ} | j | j t ƒ | j | j j d ƒ | j | j | j ƒ | j | j j | ƒ | j | j j d ƒ j ƒ d g ƒ d S( NsC values http://www.example.comR t xmls
//elem/text()u value(
R Rj Rk R R R. Rl R< Rm Rn ( R R R< ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyRq ‡ s c C sP d } | j d d | ƒ} | j | j d ƒ j ƒ | j j d ƒ j ƒ ƒ d S( NsC values http://www.example.comR s
//elem/text()( R R Rm Rn Rk ( R R R< ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyRr • s
( R, R- R R R{ R| Rq Rr ( ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyRy g s
t __main__( t unittestt w3lib.encodingR t scrapy.httpR R R R R R t scrapy.selectorR t TestCaseR RI RL Ru Ry R, t main( ( ( s< /home/travis/build/scrapy/scrapy/tests/test_http_response.pyt s .sÁ%9