1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
package Catmandu::Importer::JSON;
use Catmandu::Sane;
our $VERSION = '1.2024';
use Cpanel::JSON::XS ();
use Moo;
use namespace::clean;
with 'Catmandu::Importer';
has line_delimited => (is => 'ro', default => sub {0});
has byte_buffer_size => (is => 'ro', default => sub {256});
has json => (is => 'lazy');
sub _build_json {
my ($self) = @_;
Cpanel::JSON::XS->new->utf8($self->encoding eq ':raw');
}
sub _build_encoding {':raw'}
sub generator {
my ($self) = @_;
if ($self->line_delimited) {
return sub {
state $json = $self->json;
state $fh = $self->fh;
if (defined(my $line = <$fh>)) {
return $json->decode($line);
}
return;
};
}
# switch to incremental parser
sub {
state $json = $self->json;
state $fh = $self->fh;
state $buf_size = $self->byte_buffer_size;
for (;;) {
my $res = sysread($fh, my $buf, $buf_size);
$res // Catmandu::Error->throw($!);
$json->incr_parse($buf); # void context, so no parsing
$json->incr_text =~ s/^[^{]+//;
return if $json->incr_text =~ /^$/;
last if $json->incr_text =~ /^{/;
}
# read data until we get a single json object
for (;;) {
if (my $data = $json->incr_parse) {
return $data;
}
my $res = sysread($fh, my $buf, $buf_size);
$res // Catmandu::Error->throw($!);
$res
|| Catmandu::Error->throw(
"JSON syntax error: unexpected end of object");
$json->incr_parse($buf);
}
return;
};
}
1;
__END__
=pod
=head1 NAME
Catmandu::Importer::JSON - Package that imports JSON data
=head1 SYNOPSIS
# From the command line
$ catmandu convert JSON to YAML < data.json
# or for faster newline delimited input
$ catmandu convert JSON --line_delimited 1 to YAML < data.json
# In a Perl script
use Catmandu;
my $importer = Catmandu->importer('JSON', file => "/foo/bar.json");
my $n = $importer->each(sub {
my $hashref = $_[0];
# ...
});
=head1 DESCRIPTION
This package imports JSON data. The parser is quite liberal in the input
it accepts. You can use the C<line_delimited> option to parse "newline
delimited JSON" faster:
{ "recordno": 1, "name": "Alpha" }
{ "recordno": 2, "name": "Beta" }
{ "recordno": 3, "name": "Gamma" }
=head1 CONFIGURATION
=over
=item file
Read input from a local file given by its path. Alternatively a scalar
reference can be passed to read from a string.
=item fh
Read input from an L<IO::Handle>. If not specified, L<Catmandu::Util::io> is used to
create the input stream from the C<file> argument or by using STDIN.
=item encoding
Binmode of the input stream C<fh>. Set to C<:utf8> by default.
=item fix
An ARRAY of one or more fixes or file scripts to be applied to imported items.
=item line_delimited
Read line-delimited JSON line-by-line with a non-incremental parser.
=item byte_buffer_size
Number of bytes that is read by each iteration of the incremental parser.
Ignored if C<line_delinmited> is C<1>. Default is C<256>.
=back
=head1 METHODS
Every L<Catmandu::Importer> is a L<Catmandu::Iterable> all its methods are
inherited. The methods are not idempotent: JSON streams can only be read once.
=head1 SEE ALSO
L<Catmandu::Exporter::JSON>
=cut
|
