1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
from __future__ import absolute_import, print_function, division
import itertools
from petl.compat import next, text_type
from petl.errors import ArgumentError
from petl.util.base import Table
def unpack(table, field, newfields=None, include_original=False, missing=None):
"""
Unpack data values that are lists or tuples. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar'],
... [1, ['a', 'b']],
... [2, ['c', 'd']],
... [3, ['e', 'f']]]
>>> table2 = etl.unpack(table1, 'bar', ['baz', 'quux'])
>>> table2
+-----+-----+------+
| foo | baz | quux |
+=====+=====+======+
| 1 | 'a' | 'b' |
+-----+-----+------+
| 2 | 'c' | 'd' |
+-----+-----+------+
| 3 | 'e' | 'f' |
+-----+-----+------+
This function will attempt to unpack exactly the number of values as
given by the number of new fields specified. If there are more values
than new fields, remaining values will not be unpacked. If there are less
values than new fields, `missing` values will be added.
See also :func:`petl.transform.unpacks.unpackdict`.
"""
return UnpackView(table, field, newfields=newfields,
include_original=include_original, missing=missing)
Table.unpack = unpack
class UnpackView(Table):
def __init__(self, source, field, newfields=None, include_original=False,
missing=None):
self.source = source
self.field = field
self.newfields = newfields
self.include_original = include_original
self.missing = missing
def __iter__(self):
return iterunpack(self.source, self.field, self.newfields,
self.include_original, self.missing)
def iterunpack(source, field, newfields, include_original, missing):
it = iter(source)
try:
hdr = next(it)
except StopIteration:
hdr = []
flds = list(map(text_type, hdr))
if field in flds:
field_index = flds.index(field)
elif isinstance(field, int) and field < len(flds):
field_index = field
field = flds[field_index]
else:
raise ArgumentError('field invalid: must be either field name or index')
# determine output fields
outhdr = list(flds)
if not include_original:
outhdr.remove(field)
if isinstance(newfields, (list, tuple)):
outhdr.extend(newfields)
nunpack = len(newfields)
elif isinstance(newfields, int):
nunpack = newfields
newfields = [text_type(field) + text_type(i+1) for i in range(newfields)]
outhdr.extend(newfields)
elif newfields is None:
nunpack = 0
else:
raise ArgumentError('newfields argument must be list or tuple of field '
'names, or int (number of values to unpack)')
yield tuple(outhdr)
# construct the output data
for row in it:
value = row[field_index]
if include_original:
out_row = list(row)
else:
out_row = [v for i, v in enumerate(row) if i != field_index]
nvals = len(value)
if nunpack > 0:
if nvals >= nunpack:
newvals = value[:nunpack]
else:
newvals = list(value) + ([missing] * (nunpack - nvals))
out_row.extend(newvals)
yield tuple(out_row)
def unpackdict(table, field, keys=None, includeoriginal=False,
samplesize=1000, missing=None):
"""
Unpack dictionary values into separate fields. E.g.::
>>> import petl as etl
>>> table1 = [['foo', 'bar'],
... [1, {'baz': 'a', 'quux': 'b'}],
... [2, {'baz': 'c', 'quux': 'd'}],
... [3, {'baz': 'e', 'quux': 'f'}]]
>>> table2 = etl.unpackdict(table1, 'bar')
>>> table2
+-----+-----+------+
| foo | baz | quux |
+=====+=====+======+
| 1 | 'a' | 'b' |
+-----+-----+------+
| 2 | 'c' | 'd' |
+-----+-----+------+
| 3 | 'e' | 'f' |
+-----+-----+------+
See also :func:`petl.transform.unpacks.unpack`.
"""
return UnpackDictView(table, field, keys=keys,
includeoriginal=includeoriginal,
samplesize=samplesize, missing=missing)
Table.unpackdict = unpackdict
class UnpackDictView(Table):
def __init__(self, table, field, keys=None, includeoriginal=False,
samplesize=1000, missing=None):
self.table = table
self.field = field
self.keys = keys
self.includeoriginal = includeoriginal
self.samplesize = samplesize
self.missing = missing
def __iter__(self):
return iterunpackdict(self.table, self.field, self.keys,
self.includeoriginal, self.samplesize,
self.missing)
def iterunpackdict(table, field, keys, includeoriginal, samplesize, missing):
# set up
it = iter(table)
try:
hdr = next(it)
except StopIteration:
hdr = []
flds = list(map(text_type, hdr))
fidx = flds.index(field)
outhdr = list(flds)
if not includeoriginal:
del outhdr[fidx]
# are keys specified?
if not keys:
# need to sample to find keys
sample = list(itertools.islice(it, samplesize))
keys = set()
for row in sample:
try:
keys |= set(row[fidx].keys())
except AttributeError:
pass
it = itertools.chain(sample, it)
keys = sorted(keys)
outhdr.extend(keys)
yield tuple(outhdr)
# generate the data rows
for row in it:
outrow = list(row)
if not includeoriginal:
del outrow[fidx]
for key in keys:
try:
outrow.append(row[fidx][key])
except (IndexError, KeyError, TypeError):
outrow.append(missing)
yield tuple(outrow)
|
