1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
package Catmandu::Importer::CSV;
use Catmandu::Sane;
our $VERSION = '1.2024';
use Text::CSV;
use List::Util qw(reduce);
use Moo;
use namespace::clean;
with 'Catmandu::Importer';
has csv => (is => 'ro', lazy => 1, builder => '_build_csv');
has sep_char => (
is => 'ro',
default => sub {','},
coerce => sub {
my $sep_char = $_[0];
$sep_char =~ s/(\\[abefnrt])/"qq{$1}"/gee;
return $sep_char;
}
);
has quote_char => (is => 'ro', default => sub {'"'});
has escape_char => (is => 'ro', default => sub {'"'});
has allow_loose_quotes => (is => 'ro', default => sub {0});
has allow_loose_escapes => (is => 'ro', default => sub {0});
has header => (is => 'ro', default => sub {1});
has fields => (
is => 'rwp',
coerce => sub {
my $fields = $_[0];
if (ref $fields eq 'ARRAY') {return $fields}
if (ref $fields eq 'HASH') {return [sort keys %$fields]}
return [split ',', $fields];
},
);
sub _build_csv {
my ($self) = @_;
Text::CSV->new(
{
binary => 1,
sep_char => $self->sep_char,
quote_char => $self->quote_char ? $self->quote_char : undef,
escape_char => $self->escape_char ? $self->escape_char : undef,
allow_loose_quotes => $self->allow_loose_quotes,
allow_loose_escapes => $self->allow_loose_escapes,
}
);
}
sub generator {
my ($self) = @_;
sub {
state $line = 0;
state $fh = $self->fh;
state $csv = do {
if ($self->header) {
if ($self->fields) {
$self->csv->getline($fh);
$line++;
}
else {
$self->_set_fields($self->csv->getline($fh));
$line++;
}
}
if ($self->fields) {
$self->csv->column_names($self->fields);
}
$self->csv;
};
# generate field names if needed
unless ($self->fields) {
my $row = $csv->getline($fh) // return;
$line++;
my $fields = [0 .. (@$row - 1)];
$self->_set_fields($fields);
$csv->column_names($fields);
return reduce {
$a->{$b} = $row->[$b] if length $row->[$b];
$a;
} +{}, @$fields;
}
my $rec = $csv->getline_hr($fh);
$line++;
if (defined $rec || $csv->eof()) {
return $rec;
}
else {
my ($cde, $str, $pos) = $csv->error_diag();
die
"at line $line (byte $pos) found a Text::CSV parse error($cde) $str";
}
};
}
1;
__END__
=pod
=head1 NAME
Catmandu::Importer::CSV - Package that imports CSV data
=head1 SYNOPSIS
# From the command line
# convert a CSV file to JSON
catmandu convert CSV to JSON < journals.csv
# set column names if CSV file has no header line
echo '12157,"The Journal of Headache and Pain",2193-1801' | \
catmandu convert CSV --header 0 --fields 'id,title,issn' to YAML
# set field separator and quote character
echo '12157;$The Journal of Headache and Pain$;2193-1801' | \
catmandu convert CSV --header 0 --fields 'id,title,issn' --sep_char ';' --quote_char '$' to XLSX --file journal.xlsx
# In a Perl script
use Catmandu;
my $importer = Catmandu->importer('CSV', file => "/foo/bar.csv");
my $n = $importer->each(sub {
my $hashref = $_[0];
# ...
});
=head1 DESCRIPTION
The package imports comma-separated values (CSV). The object
fields are read from the CSV header line or given via the C<fields> parameter.
Strings in CSV are quoted by C<quote_char> and fields are separated by
C<sep_char>.
=head1 CONFIGURATION
=over
=item file
Read input from a local file given by its path. Alternatively a scalar
reference can be passed to read from a string.
=item fh
Read input from an L<IO::Handle>. If not specified, L<Catmandu::Util::io> is used to
create the input stream from the C<file> argument or by using STDIN.
=item encoding
Binmode of the input stream C<fh>. Set to C<:utf8> by default.
=item fix
An ARRAY of one or more fixes or file scripts to be applied to imported items.
=item fields
List of fields to be used as columns, given as array reference, comma-separated
string, or hash reference. If C<header> is C<0> and C<fields> is C<undef> the
fields will be named by column index ("0", "1", "2", ...).
=item header
Read fields from a header line with the column names, if set to C<1> (the
default).
=item sep_char
Column separator (C<,> by default)
=item quote_char
Quotation character (C<"> by default)
=item escape_char
Character for escaping inside quoted field (C<"> by default)
=item allow_loose_quotes
=item allow_loose_escapes
Allow common bad-practice in CSV escaping
=back
=head1 METHODS
Every L<Catmandu::Importer> is a L<Catmandu::Iterable> all its methods are
inherited. The methods are not idempotent: CSV streams can only be read once.
=head1 SEE ALSO
L<Catmandu::Exporter::CSV>, L<Catmandu::Importer::XLS>
=cut
|
