1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
|
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
# standard library dependencies
from petl.compat import pickle, next
# internal dependencies
from petl.util.base import Table
from petl.io.sources import read_source_from_arg, write_source_from_arg
def frompickle(source=None):
"""
Extract a table From data pickled in the given file. The rows in the
table should have been pickled to the file one at a time. E.g.::
>>> import petl as etl
>>> import pickle
>>> # set up a file to demonstrate with
... with open('example.p', 'wb') as f:
... pickle.dump(['foo', 'bar'], f)
... pickle.dump(['a', 1], f)
... pickle.dump(['b', 2], f)
... pickle.dump(['c', 2.5], f)
...
>>> # now demonstrate the use of frompickle()
... table1 = etl.frompickle('example.p')
>>> table1
+-----+-----+
| foo | bar |
+=====+=====+
| 'a' | 1 |
+-----+-----+
| 'b' | 2 |
+-----+-----+
| 'c' | 2.5 |
+-----+-----+
"""
source = read_source_from_arg(source)
return PickleView(source)
class PickleView(Table):
def __init__(self, source):
self.source = source
def __iter__(self):
with self.source.open('rb') as f:
try:
while True:
yield tuple(pickle.load(f))
except EOFError:
pass
def topickle(table, source=None, protocol=-1, write_header=True):
"""
Write the table to a pickle file. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar'],
... ['a', 1],
... ['b', 2],
... ['c', 2]]
>>> etl.topickle(table1, 'example.p')
>>> # look what it did
... table2 = etl.frompickle('example.p')
>>> table2
+-----+-----+
| foo | bar |
+=====+=====+
| 'a' | 1 |
+-----+-----+
| 'b' | 2 |
+-----+-----+
| 'c' | 2 |
+-----+-----+
Note that if a file already exists at the given location, it will be
overwritten.
The pickle file format preserves type information, i.e., reading and writing
is round-trippable for tables with non-string data values.
"""
_writepickle(table, source=source, mode='wb', protocol=protocol,
write_header=write_header)
Table.topickle = topickle
def appendpickle(table, source=None, protocol=-1, write_header=False):
"""
Append data to an existing pickle file. I.e.,
as :func:`petl.io.pickle.topickle` but the file is opened in append mode.
Note that no attempt is made to check that the fields or row lengths are
consistent with the existing data, the data rows from the table are simply
appended to the file.
"""
_writepickle(table, source=source, mode='ab', protocol=protocol,
write_header=write_header)
Table.appendpickle = appendpickle
def _writepickle(table, source, mode, protocol, write_header):
source = write_source_from_arg(source, mode)
with source.open(mode) as f:
it = iter(table)
try:
hdr = next(it)
except StopIteration:
return
if write_header:
pickle.dump(hdr, f, protocol)
for row in it:
pickle.dump(row, f, protocol)
def teepickle(table, source=None, protocol=-1, write_header=True):
"""
Return a table that writes rows to a pickle file as they are iterated
over.
"""
return TeePickleView(table, source=source, protocol=protocol,
write_header=write_header)
Table.teepickle = teepickle
class TeePickleView(Table):
def __init__(self, table, source=None, protocol=-1, write_header=True):
self.table = table
self.source = source
self.protocol = protocol
self.write_header = write_header
def __iter__(self):
protocol = self.protocol
source = write_source_from_arg(self.source)
with source.open('wb') as f:
it = iter(self.table)
try:
hdr = next(it)
except StopIteration:
return
if self.write_header:
pickle.dump(hdr, f, protocol)
yield tuple(hdr)
for row in it:
pickle.dump(row, f, protocol)
yield tuple(row)
|
