1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/dataset-write.R
\name{write_delim_dataset}
\alias{write_delim_dataset}
\alias{write_csv_dataset}
\alias{write_tsv_dataset}
\title{Write a dataset into partitioned flat files.}
\usage{
write_delim_dataset(
dataset,
path,
partitioning = dplyr::group_vars(dataset),
basename_template = "part-{i}.txt",
hive_style = TRUE,
existing_data_behavior = c("overwrite", "error", "delete_matching"),
max_partitions = 1024L,
max_open_files = 900L,
max_rows_per_file = 0L,
min_rows_per_group = 0L,
max_rows_per_group = bitwShiftL(1, 20),
col_names = TRUE,
batch_size = 1024L,
delim = ",",
na = "",
eol = "\\n",
quote = c("needed", "all", "none")
)
write_csv_dataset(
dataset,
path,
partitioning = dplyr::group_vars(dataset),
basename_template = "part-{i}.csv",
hive_style = TRUE,
existing_data_behavior = c("overwrite", "error", "delete_matching"),
max_partitions = 1024L,
max_open_files = 900L,
max_rows_per_file = 0L,
min_rows_per_group = 0L,
max_rows_per_group = bitwShiftL(1, 20),
col_names = TRUE,
batch_size = 1024L,
delim = ",",
na = "",
eol = "\\n",
quote = c("needed", "all", "none")
)
write_tsv_dataset(
dataset,
path,
partitioning = dplyr::group_vars(dataset),
basename_template = "part-{i}.tsv",
hive_style = TRUE,
existing_data_behavior = c("overwrite", "error", "delete_matching"),
max_partitions = 1024L,
max_open_files = 900L,
max_rows_per_file = 0L,
min_rows_per_group = 0L,
max_rows_per_group = bitwShiftL(1, 20),
col_names = TRUE,
batch_size = 1024L,
na = "",
eol = "\\n",
quote = c("needed", "all", "none")
)
}
\arguments{
\item{dataset}{\link{Dataset}, \link{RecordBatch}, \link{Table}, \code{arrow_dplyr_query}, or
\code{data.frame}. If an \code{arrow_dplyr_query}, the query will be evaluated and
the result will be written. This means that you can \code{select()}, \code{filter()}, \code{mutate()},
etc. to transform the data before it is written if you need to.}
\item{path}{string path, URI, or \code{SubTreeFileSystem} referencing a directory
to write to (directory will be created if it does not exist)}
\item{partitioning}{\code{Partitioning} or a character vector of columns to
use as partition keys (to be written as path segments). Default is to
use the current \code{group_by()} columns.}
\item{basename_template}{string template for the names of files to be written.
Must contain \code{"{i}"}, which will be replaced with an autoincremented
integer to generate basenames of datafiles. For example, \code{"part-{i}.arrow"}
will yield \verb{"part-0.arrow", ...}.
If not specified, it defaults to \code{"part-{i}.<default extension>"}.}
\item{hive_style}{logical: write partition segments as Hive-style
(\code{key1=value1/key2=value2/file.ext}) or as just bare values. Default is \code{TRUE}.}
\item{existing_data_behavior}{The behavior to use when there is already data
in the destination directory. Must be one of "overwrite", "error", or
"delete_matching".
\itemize{
\item "overwrite" (the default) then any new files created will overwrite
existing files
\item "error" then the operation will fail if the destination directory is not
empty
\item "delete_matching" then the writer will delete any existing partitions
if data is going to be written to those partitions and will leave alone
partitions which data is not written to.
}}
\item{max_partitions}{maximum number of partitions any batch may be
written into. Default is 1024L.}
\item{max_open_files}{maximum number of files that can be left opened
during a write operation. If greater than 0 then this will limit the
maximum number of files that can be left open. If an attempt is made to open
too many files then the least recently used file will be closed.
If this setting is set too low you may end up fragmenting your data
into many small files. The default is 900 which also allows some # of files to be
open by the scanner before hitting the default Linux limit of 1024.}
\item{max_rows_per_file}{maximum number of rows per file.
If greater than 0 then this will limit how many rows are placed in any single file.
Default is 0L.}
\item{min_rows_per_group}{write the row groups to the disk when this number of
rows have accumulated. Default is 0L.}
\item{max_rows_per_group}{maximum rows allowed in a single
group and when this number of rows is exceeded, it is split and the next set
of rows is written to the next group. This value must be set such that it is
greater than \code{min_rows_per_group}. Default is 1024 * 1024.}
\item{col_names}{Whether to write an initial header line with column names.}
\item{batch_size}{Maximum number of rows processed at a time. Default is 1024L.}
\item{delim}{Delimiter used to separate values. Defaults to \code{","} for \code{write_delim_dataset()} and
\code{write_csv_dataset()}, and \verb{"\\t} for \code{write_tsv_dataset()}. Cannot be changed for \code{write_tsv_dataset()}.}
\item{na}{a character vector of strings to interpret as missing values. Quotes are not allowed in this string.
The default is an empty string \code{""}.}
\item{eol}{the end of line character to use for ending rows. The default is \code{"\\n"}.}
\item{quote}{How to handle fields which contain characters that need to be quoted.
\itemize{
\item \code{needed} - Enclose all strings and binary values in quotes which need them, because their CSV rendering can
contain quotes itself (the default)
\item \code{all} - Enclose all valid values in quotes. Nulls are not quoted. May cause readers to
interpret all values as strings if schema is inferred.
\item \code{none} - Do not enclose any values in quotes. Prevents values from containing quotes ("),
cell delimiters (,) or line endings (\\r, \\n), (following RFC4180). If values
contain these characters, an error is caused when attempting to write.
}}
}
\value{
The input \code{dataset}, invisibly.
}
\description{
The \verb{write_*_dataset()} are a family of wrappers around \link{write_dataset} to allow for easy switching
between functions for writing datasets.
}
\seealso{
\code{\link[=write_dataset]{write_dataset()}}
}
|
