1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
package Catmandu::Importer::Text;
use Catmandu::Sane;
our $VERSION = '1.2024';
use Moo;
use namespace::clean;
with 'Catmandu::Importer';
has pattern => (
is => 'ro',
coerce => sub {
$_[0] =~ /\n/m ? qr{$_[0]}x : qr{$_[0]};
},
);
has split => (
is => 'ro',
coerce => sub {
length $_[0] == 1 ? quotemeta($_[0]) : qr{$_[0]};
}
);
sub generator {
my ($self) = @_;
sub {
state $pattern = $self->pattern;
state $split = $self->split;
state $count = 0;
state $line;
while (defined($line = $self->fh->getline)) {
chomp $line;
next if $pattern and $line !~ $pattern;
my $data = {_id => ++$count};
if (@+ < 2) { # no capturing groups
$data->{text} = $line;
}
elsif (%+) { # named capturing groups
$data->{match} = {%+};
}
else { # numbered capturing groups
no strict 'refs';
$data->{match} = [map {$$_} 1 .. @+ - 1];
}
if ($split) {
$data->{text} = [split $split, $line];
}
return $data;
}
return;
};
}
1;
__END__
=pod
=head1 NAME
Catmandu::Importer::Text - Package that imports textual data
=head1 SYNOPSIS
# From the command line
# separate fields by whitespace sequences just like awk
catmandu convert Text --split '\s+'
# import all lines starting with '#', omitting this character
catmandu convert Text --pattern '^#(.*)'
# In a Perl script
use Catmandu;
my $importer = Catmandu->importer('Text' , file => "/foo/bar.txt" );
# print all lines with line number
$importer->each(sub {
my $item = $_[0];
printf "%d: %s" , $item->{_id} , $item->{text};
});
=head1 DESCRIPTION
This package reads textual input line by line. Each line is
imported as item with line number in field C<_id> and text content in field
C<text>. Line separators are not included. Lines can further be split by
character or pattern and a regular expression can be specified to only import
selected lines and to translate pattern groups to fields.
=head1 CONFIGURATION
=over
=item file
Read input from a local file given by its path. Alternatively a scalar
reference can be passed to read from a string.
=item fh
Read input from an L<IO::Handle>. If not specified, L<Catmandu::Util::io> is used to
create the input stream from the C<file> argument or by using STDIN.
=item encoding
Binmode of the input stream C<fh>. Set to C<:utf8> by default.
=item fix
An ARRAY of one or more fixes or file scripts to be applied to imported items.
=item split
Single Character or regular expression (as string with a least two characters),
to split each line. Resulting parts are imported in field C<text> as array.
=item pattern
Regular expression, given as string, to only import matching lines.
Whitespaces in patterns are ignored or must be escaped if patterns consists of
multiple lines. If the pattern contains capturing groups, captured values are
imported in field C<match> instead of C<text>.
For instance dates in C<YYYY-MM-DD> format can be imported as named fields with
(?<year>\d\d\d\d)-(?<month>\d\d)-(?<day>\d\d)
or as array with
(\d\d\d\d)- # year
(\d\d)- # month
(\d\d) # day
=back
=head1 METHODS
Every L<Catmandu::Importer> is a L<Catmandu::Iterable> with all its methods
inherited.
=head1 SEE ALSO
L<Catmandu::Exporter::Text>
L<Catmandu::Fix::parse_text>
Unix tools L<awk|https://en.wikipedia.org/wiki/AWK> and
L<sed|https://en.wikipedia.org/wiki/Sed>
=cut
|
