ó
ç'ÊUc @ sù d d l Z d d l m Z d d l m Z m Z m Z m Z d d l m Z m
Z
m Z d d l m
Z
d e j d Z d e j f d „ ƒ YZ d
e f d „ ƒ YZ d e j f d
„ ƒ YZ d e j f d „ ƒ YZ e d k rõ e j ƒ n d S( iÿÿÿÿN( t unittest( t csvitert xmlitert _body_or_strt xmliter_lxml( t XmlResponset TextResponset Response( t get_testdatau foou bart XmliterTestCasec B sA e Z e e ƒ Z d „ Z d „ Z d „ Z d „ Z d „ Z RS( c C s¿ d } t d d d | ƒ } g } x] | j | d ƒ D]I } | j | j d ƒ j ƒ | j d ƒ j ƒ | j d ƒ j ƒ f ƒ q4 W| j | d g d
g d g f d g d
g d g f g ƒ d S( Ns´ Type 1 Name 1 Type 2 Name 2 t urls http://example.comt bodyt products @ids name/text()s
./type/text()t 001s Name 1s Type 1t 002s Name 2s Type 2( R R t appendt xpatht extractt assertEqual( t selfR t responset attrst x( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_xmliter s G c C sT d } | j g | j | d ƒ D] } | j d ƒ j ƒ ^ q d g d g g ƒ d S( Nug onetwoR s text()u oneu two( R R R R ( R R R ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_xmliter_text$ s 7c C sv d } t d d d | ƒ } | j | d ƒ } t | ƒ } | j d d ƒ | j | j d ƒ j ƒ d g ƒ | j | j d
ƒ j ƒ d g ƒ | j | j d ƒ j ƒ d
g ƒ | j | j d ƒ j ƒ d g ƒ | j | j d ƒ j ƒ d g ƒ | j | j d ƒ j ƒ d g ƒ | j | j d ƒ j ƒ g ƒ | j | j d ƒ j ƒ g ƒ | j | j d ƒ j ƒ g ƒ d S( Ns
My Dummy Company
http://www.mydummycompany.com
This is a dummy company. We do nothing.
-
Item 1
This is item 1
http://www.mydummycompany.com/items/1
http://www.mydummycompany.com/images/item1.jpg
ITEM_1
400
R
s http://mydummycompany.comR t itemt gs http://base.google.com/ns/1.0s title/text()s Item 1s description/text()s This is item 1s link/text()s% http://www.mydummycompany.com/items/1s g:image_link/text()s. http://www.mydummycompany.com/images/item1.jpgs g:id/text()t ITEM_1s g:price/text()t 400s image_link/text()s id/text()s price/text()( R R t nextt register_namespaceR R R ( R R R t my_itert node( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_xmliter_namespaces* s """""" c C sC d } | j | d ƒ } t | ƒ t | ƒ | j t t | ƒ d S( Nug onetwoR ( R R t assertRaisest
StopIteration( R R t iter( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_xmliter_exceptionL s
c C sD d } t d d | ƒ} | j | j | d ƒ j ƒ j ƒ d ƒ d S( Nsq
- Some Turkish Characters ÖÇÞÝÐÜ üðýþçö
s http://www.example.comR R u> - Some Turkish Characters ÖÇŞİĞÜ üğışçö
( R R R R R ( R R R ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_xmliter_encodingU s
( t __name__t
__module__t staticmethodR R R R! R% R& ( ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyR s " t LxmlXmliterTestCasec B s& e Z e e ƒ Z d „ Z d „ Z RS( c C s¾ d } t d d d | ƒ } | j | d ƒ } | j t t | ƒ ƒ d ƒ | j | d d ƒ } t | ƒ } | j | j d ƒ j ƒ d g ƒ t | ƒ } | j | j d ƒ j ƒ d
g ƒ d S( Ns
My Dummy Company
http://www.mydummycompany.com
This is a dummy company. We do nothing.
-
Item 1
This is item 1
http://www.mydummycompany.com/items/1
http://www.mydummycompany.com/images/item1.jpg
http://www.mydummycompany.com/images/item2.jpg
R
s http://mydummycompany.comR t
image_linki s http://base.google.com/ns/1.0s text()s. http://www.mydummycompany.com/images/item1.jpgs. http://www.mydummycompany.com/images/item2.jpg( R R R t lent listR R R ( R R R t no_namespace_itert namespace_iterR ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_xmliter_iterate_namespacea s " c C sò d } t d d d | ƒ } | j | d d d ƒ } t | ƒ } | j t | j d ƒ j ƒ ƒ d ƒ | j | j d
ƒ j ƒ d g ƒ | j | j d ƒ j ƒ d
g ƒ | j | d d d ƒ } t | ƒ } | j | j d ƒ j ƒ d g ƒ d S( Nsø
Apples
Bananas
African Coffee Table
80
120
R
s http://mydummycompany.comR t tables http://www.w3.org/TR/html4/t hs h:tr/h:tdi s h:tr/h:td[1]/text()t Appless h:tr/h:td[2]/text()t Bananass" http://www.w3schools.com/furnituret fs
f:name/text()s African Coffee Table( R R R R R, R R ( R R R R R ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_xmliter_namespaces_prefix~ s %"" ( R' R( R) R R R0 R6 ( ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyR* ^ s t UtilsCsvTestCasec B sÈ e Z e j j e j j e j j e ƒ ƒ d d ƒ Z e j j e d ƒ Z e j j e d ƒ Z
e j j e d ƒ Z d „ Z d „ Z
d „ Z d „ Z d „ Z d
„ Z d „ Z d „ Z d
„ Z RS( t sample_datat feedss feed-sample3.csvs feed-sample4.csvs feed-sample5.csvc C s t d d ƒ } t d d d | ƒ } t | ƒ } g | D] } | ^ q7 } | j | i d d 6d d 6d
d 6i d d 6d
d 6d d 6i d d 6d d 6t d 6i d d 6d d 6d d 6g ƒ xT | D]L } | j t d „ | j ƒ Dƒ ƒ ƒ | j t d „ | j ƒ Dƒ ƒ ƒ qÀ Wd S( NR9 s feed-sample3.csvR
s http://example.com/R u 1u idu alphau nameu foobaru valueu 2u unicodeu únÃcódé‽u 3u multiu 4u emptyu c s s | ] } t | t ƒ Vq d S( N( t
isinstancet unicode( t .0t k( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pys ³ s c s s | ] } t | t ƒ Vq d S( N( R: R; ( R< t v( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pys ´ s ( R R R R t FOOBAR_NLt assert_t allt keyst values( R R R t csvt rowt resultt
result_row( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_defaults¥ s
#c C sÉ t d d ƒ j d d ƒ } t d d d | ƒ } t | d d ƒ} | j g | D] } | ^ qO i d d
6d d 6d
d 6i d d
6d d 6d d 6i d d
6d d 6t d 6i d d
6d d 6d d 6g ƒ d S( NR9 s feed-sample3.csvt ,s R
s http://example.com/R t delimiteru 1u idu alphau nameu foobaru valueu 2u unicodeu únÃcódé‽u 3u multiu 4u emptyu ( R t replaceR R R R? ( R R R RD RE ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_delimiter¶ s c C sˆ t d d ƒ } t d d ƒ j d d ƒ } t d d d | ƒ } t | d d ƒ} | j g | D] } | ^ q^ i d
d 6d d
6d d 6i d d 6d d
6d d 6i d d 6d d
6t d 6i d d 6d d
6d d 6g ƒ t d d d | ƒ } t | d d d d ƒ} | j g | D] } | ^ qi d
d 6d d
6d d 6i d d 6d d
6d d 6i d d 6d d
6t d 6i d d 6d d
6d d 6g ƒ d S( NR9 s feed-sample6.csvRI t |R
s http://example.com/R t quotechart 'u 1u idu alphau nameu foobaru valueu 2u unicodeu únÃcódé‽u 3u multiu 4u emptyu RJ ( R RK R R R R? ( R t body1t body2t response1t csv1RE t response2t csv2( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_quotecharÁ s c C s· t d d ƒ } t d d d | ƒ } t | ƒ } | j g | D] } | ^ q= i d d 6d d 6d
d 6i d d 6d
d 6d d 6i d d 6d d 6d d 6i d d 6d d 6d d 6g ƒ d S( NR9 s feed-sample6.csvR
s http://example.com/R u 1u 'id'u 'alpha'u 'name'u 'foobar'u 'value'u 2u 'unicode'u 'únÃcódé‽'u '3'u 'multi'u 'foou 4u 'empty'u ( R R R R ( R R R RD RE ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_wrong_quotechar× s c C sÉ t d d ƒ j d d ƒ } t d d d | ƒ } t | d d ƒ} | j g | D] } | ^ qO i d d
6d d 6d
d 6i d d
6d d 6d d 6i d d
6d d 6t d 6i d d
6d d 6d d 6g ƒ d S( NR9 s feed-sample3.csvRI s R
s http://example.com/R RJ u 1u idu alphau nameu foobaru valueu 2u unicodeu únÃcódé‽u 3u multiu 4u emptyu ( R RK R R R R? ( R R R RD RE ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt; test_csviter_delimiter_binary_response_assume_utf8_encodingâ s c C sê t d d ƒ j ƒ } | d j d ƒ d j | d ƒ } } t d d d | ƒ } t | d
| ƒ} | j g | D] } | ^ qp i d d 6d
d 6d d 6i d d 6d d 6d d 6i d d 6d d 6d d 6i d d 6d d 6d d 6g ƒ d S( NR9 s feed-sample3.csvi RI s
i R
s http://example.com/R t headersu 1u idu alphau nameu foobaru valueu 2u unicodeu únÃcódé‽u 3u multiu foo
baru 4u emptyu ( R t
splitlinest splitt joinR R R ( R t sampleRY R R RD RE ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_headersí s ' c C sÏ t d d ƒ } d j | d d f ƒ } t d d d | ƒ } t | ƒ } | j g | D] } | ^ qU i d d
6d d 6d
d 6i d d
6d d 6d d 6i d d
6d d 6t d 6i d d
6d d 6d d 6g ƒ d S( NR9 s feed-sample3.csvs
s a,bs a,b,c,dR
s http://example.com/R u 1u idu alphau nameu foobaru valueu 2u unicodeu únÃcódé‽u 3u multiu 4u emptyu ( R R\ R R R R? ( R R R RD RE ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_falserowú s c C so t d d ƒ } t d d d | ƒ } t | ƒ } t | ƒ t | ƒ t | ƒ t | ƒ | j t t | ƒ d S( NR9 s feed-sample3.csvR
s http://example.com/R ( R R R R R" R# ( R R R R$ ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_exception s
c C s t d d ƒ } t d d ƒ } t d d d | d d ƒ } t | ƒ } | j g | D] } | ^ qR i d d
6d d 6d
d 6i d d
6d d 6d d 6g ƒ t d d d | d d ƒ } t | ƒ } | j g | D] } | ^ qÌ i d d
6d d 6d
d 6i d d
6d d 6d d 6g ƒ d S( NR9 s feed-sample4.csvs feed-sample5.csvR
s http://example.com/R t encodingt latin1u 1u idu latin1u nameu testu valueu 2u somethingu ñáéót cp852u cp852u â•šâ•©â•©â•©â•â•â•—( R R R R ( R RP RQ R RD RE ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_csviter_encoding s ( R' R( t ost pathR\ t abspatht dirnamet __file__t sample_feeds_dirt sample_feed_patht sample_feed2_patht sample_feed3_pathRH RL RV RW RX R^ R_ R` Rd ( ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyR7 Ÿ s 0
t
TestHelperc B s_ e Z d Z e j d ƒ Z e d d d e d d ƒ Z e d d d e ƒ Z d „ Z d „ Z
RS( s utf8-bodyt utf8R
s http://example.org/R Ra s utf-8c C sÝ xÖ | j | j | j | j f D]¶ } t | ƒ } | j | | j | ƒ t | d t ƒ} | j | | j | ƒ t | d t ƒ} | j | | j | ƒ | j t | ƒ t | ƒ k ƒ | j t | ƒ t | ƒ k ƒ q Wd S( NR; (
t bbodyt ubodyt txtresponseR R t _assert_type_and_valuet Truet Falset
assertTruet type( R t objt r1t r2t r3( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt test_body_or_str* s % c C sQ | j t | ƒ t | ƒ k d j t | ƒ t | ƒ | ƒ ƒ | j | | ƒ d S( Ns Got {}, expected {} for {!r}( Rv Rw t formatR ( R t at bRx ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyRs 6 s "( R' R( Rp t decodeRq R Rr R R R| Rs ( ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyRn $ s t __main__( Re t
twisted.trialR t scrapy.utils.iteratorsR R R R t scrapy.httpR R R t testsR t linesepR? t TestCaseR R* R7 Rn R' t main( ( ( s> /home/travis/build/scrapy/scrapy/tests/test_utils_iterators.pyt s " SA…