1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
|
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/options.R
\name{parallelly.options}
\alias{parallelly.options}
\alias{parallelly.debug}
\alias{parallelly.availableCores.custom}
\alias{parallelly.availableCores.methods}
\alias{parallelly.availableCores.min}
\alias{parallelly.availableCores.fallback}
\alias{parallelly.availableCores.omit}
\alias{parallelly.availableCores.system}
\alias{parallelly.availableWorkers.methods}
\alias{parallelly.availableWorkers.custom}
\alias{parallelly.fork.enable}
\alias{parallelly.maxWorkers.localhost}
\alias{parallelly.supportsMulticore.disableOn}
\alias{parallelly.supportsMulticore.unstable}
\alias{R_PARALLELLY_AVAILABLECORES_FALLBACK}
\alias{R_PARALLELLY_AVAILABLECORES_OMIT}
\alias{R_PARALLELLY_AVAILABLECORES_SYSTEM}
\alias{R_PARALLELLY_AVAILABLECORES_MIN}
\alias{R_PARALLELLY_FORK_ENABLE}
\alias{R_PARALLELLY_SUPPORTSMULTICORE_DISABLEON}
\alias{R_PARALLELLY_SUPPORTSMULTICORE_UNSTABLE}
\alias{future.availableCores.custom}
\alias{future.availableCores.methods}
\alias{future.availableCores.fallback}
\alias{future.availableCores.system}
\alias{future.availableWorkers.methods}
\alias{future.availableWorkers.custom}
\alias{future.fork.enable}
\alias{future.supportsMulticore.unstable}
\alias{R_FUTURE_AVAILABLECORES_FALLBACK}
\alias{R_FUTURE_AVAILABLECORES_SYSTEM}
\alias{R_FUTURE_FORK_ENABLE}
\alias{R_FUTURE_SUPPORTSMULTICORE_UNSTABLE}
\alias{parallelly.makeNodePSOCK.setup_strategy}
\alias{parallelly.makeNodePSOCK.validate}
\alias{parallelly.makeNodePSOCK.connectTimeout}
\alias{parallelly.makeNodePSOCK.timeout}
\alias{parallelly.makeNodePSOCK.useXDR}
\alias{parallelly.makeNodePSOCK.socketOptions}
\alias{parallelly.makeNodePSOCK.rshcmd}
\alias{parallelly.makeNodePSOCK.rshopts}
\alias{parallelly.makeNodePSOCK.tries}
\alias{parallelly.makeNodePSOCK.tries.delay}
\alias{R_PARALLELLY_MAKENODEPSOCK_SETUP_STRATEGY}
\alias{R_PARALLELLY_MAKENODEPSOCK_VALIDATE}
\alias{R_PARALLELLY_MAKENODEPSOCK_CONNECTTIMEOUT}
\alias{R_PARALLELLY_MAKENODEPSOCK_TIMEOUT}
\alias{R_PARALLELLY_MAKENODEPSOCK_USEXDR}
\alias{R_PARALLELLY_MAKENODEPSOCK_SOCKETOPTIONS}
\alias{R_PARALLELLY_MAKENODEPSOCK_RSHCMD}
\alias{R_PARALLELLY_MAKENODEPSOCK_RSHOPTS}
\alias{R_PARALLELLY_MAKENODEPSOCK_TRIES}
\alias{R_PARALLELLY_MAKENODEPSOCK_TRIES_DELAY}
\title{Options Used by the 'parallelly' Package}
\description{
Below are the \R options and environment variables that are used by the
\pkg{parallelly} package and packages enhancing it.\cr
\cr
\emph{WARNING: Note that the names and the default values of these options may
change in future versions of the package. Please use with care until
further notice.}
}
\section{Backward compatibility with the \pkg{future} package}{
The functions in the \pkg{parallelly} package originates from the
\pkg{future} package. Because they are widely used within the future
ecosystem, we need to keep them backward compatible for quite a long time,
in order for all existing packages and R scripts to have time to adjust.
This also goes for the \R options and the environment variables used to
configure these functions.
All options and environment variables used here have prefixes \code{parallelly.}
and \code{R_PARALLELLY_}, respectively. Because of the backward compatibility
with the \pkg{future} package, the same settings can also be controlled
by options and environment variables with prefixes \code{future.} and
\code{R_FUTURE_} until further notice, e.g. setting option
\code{future.availableCores.fallback=1} is the same as setting option
\code{parallelly.availableCores.fallback=1}, and setting environment
variable \env{R_FUTURE_AVAILABLECORES_FALLBACK=1} is the same as setting
\env{R_PARALLELLY_AVAILABLECORES_FALLBACK=1}.
}
\section{Configuring number of parallel workers}{
The below \R options and environment variables control the default results of \code{\link[=availableCores]{availableCores()}} and \code{\link[=availableWorkers]{availableWorkers()}}.
\describe{
\item{\code{parallelly.availableCores.logical}:}{(logical) The default value of argument \code{logical} as used by \code{availableCores()}, \code{availableWorkers()}, and \code{availableCores()} for querying \code{parallel::detectCores(logical = logical)}. The default is \code{TRUE} just like it is for \code{\link[parallel:detectCores]{parallel::detectCores()}}.}
\item{\code{parallelly.availableCores.methods}:}{(character vector) Default lookup methods for \code{\link[=availableCores]{availableCores()}}. (Default: \code{c("system", "cgroups.cpuset", "cgroups.cpuquota", "cgroups2.cpu.max", "nproc", "mc.cores", "BiocParallel", "_R_CHECK_LIMIT_CORES_", "Bioconductor", "LSF", "PJM", "PBS", "SGE", "Slurm", "fallback", "custom")})}
\item{\code{parallelly.availableCores.custom}:}{(function) If set and a function, then this function will be called (without arguments) by \code{\link[=availableCores]{availableCores()}} where its value, coerced to an integer, is interpreted as a number of cores.}
\item{\code{parallelly.availableCores.fallback}:}{(integer) Number of cores to use when no core-specifying settings are detected other than \code{"system"} and \code{"nproc"}. This options makes it possible to set the default number of cores returned by \code{availableCores()} / \code{availableWorkers()} yet allow users and schedulers to override it. In multi-tenant environment, such as HPC clusters, it is useful to set environment variable \env{R_PARALLELLY_AVAILABLECORES_FALLBACK} to \code{1}, which will set this option when the package is loaded.}
\item{\code{parallelly.availableCores.system}:}{(integer) Number of "system" cores used instead of what is reported by \code{\link{availableCores}(which = "system")}. This option allows you to effectively override what \code{parallel::detectCores()} reports the system has.}
\item{\code{parallelly.availableCores.min}:}{(integer) The minimum number of cores \code{\link[=availableCores]{availableCores()}} is allowed to return. This can be used to force multiple cores on a single-core environment. If this is limit is applied, the names of the returned value are appended with an asterisk (\code{*}). (Default: \code{1L})}
\item{\code{parallelly.availableCores.omit}:}{(integer) Number of cores to set aside, i.e. not to include.}
\item{\code{parallelly.availableWorkers.methods}:}{(character vector) Default lookup methods for \code{\link[=availableWorkers]{availableWorkers()}}. (Default: \code{c("mc.cores", "BiocParallel", "_R_CHECK_LIMIT_CORES_", "Bioconductor", "LSF", "PJM", "PBS", "SGE", "Slurm", "custom", "cgroups.cpuset", "cgroups.cpuquota", "cgroups2.cpu.max", "nproc", "system", "fallback")})}
\item{\code{parallelly.availableWorkers.custom}:}{(function) If set and a function, then this function will be called (without arguments) by \code{\link[=availableWorkers]{availableWorkers()}} where its value, coerced to a character vector, is interpreted as hostnames of available workers.}
}
}
\section{Configuring forked parallel processing}{
The below \R options and environment variables control the default result of \code{\link[=supportsMulticore]{supportsMulticore()}}.
\describe{
\item{\code{parallelly.fork.enable}:}{(logical) Enable or disable \emph{forked} processing. If \code{FALSE}, multicore futures becomes sequential futures. If \code{NA}, or not set (the default), the a set of best-practices rules decide whether should be supported or not.}
\item{\code{parallelly.supportsMulticore.disableOn}:}{(character vector)
because the environment in which R runs is considered unstable for
forked processing.
If this vector contains \code{"rstudio_console"}, it is disabled when
running R in the RStudio Console.
If this vector contains \code{"rstudio_terminal"}, it is disabled when
running R in the RStudio Terminal.
(Default: \code{c("rstudio_console", "rstudio_terminal")})
}
\item{\code{parallelly.supportsMulticore.unstable}:}{(character) Controls whether a warning should be produced or not whenever multicore processing is automatically disabled per settings in option \code{parallelly.supportsMulticore.disableOn}. If \code{"warn"} (default), then an informative warning is produces the first time 'multicore' futures are used. If \code{"quiet"}, no warning is produced.}
}
}
\section{Configuring setup of parallel PSOCK clusters}{
The below \R options and environment variables control the default results of \code{\link[=makeClusterPSOCK]{makeClusterPSOCK()}} and its helper function \code{\link[=makeNodePSOCK]{makeNodePSOCK()}} that creates the individual cluster nodes.
\describe{
\item{\code{parallelly.maxWorkers.localhost}:}{(two numerics) Maximum number of localhost workers, relative to \code{availableCores()}, accepted and allowed. The first element corresponds to the threshold where a warning is produced, the second where an error is produced. Thresholds may be \code{+Inf}. If only the first exist, no error is produced (defaults to \code{c(1.0, 3.0)} corresponding to a maximum 100\% and 300\% use).}
\item{\code{parallelly.makeNodePSOCK.setup_strategy}:}{(character) If \code{"parallel"} (default), the PSOCK cluster nodes are set up concurrently. If \code{"sequential"}, they are set up sequentially.}
\item{\code{parallelly.makeNodePSOCK.validate}:}{(logical) If TRUE (default), after the nodes have been created, they are all validated that they work by inquiring about their session information, which is saved in attribute \code{session_info} of each node.}
\item{\code{parallelly.makeNodePSOCK.connectTimeout}:}{(numeric) The maximum time (in seconds) allowed for each socket connection between the master and a worker to be established (defaults to 2*60 seconds = 2 minutes).}
\item{\code{parallelly.makeNodePSOCK.timeout}:}{(numeric) The maximum time (in seconds) allowed to pass without the master and a worker communicate with each other (defaults to 30\emph{24}60*60 seconds = 30 days).}
\item{\code{parallelly.makeNodePSOCK.useXDR}:}{(logical) If FALSE (default), the communication between master and workers, which is binary, will use small-endian (faster), otherwise big-endian ("XDR"; slower).}
\item{\code{parallelly.makeNodePSOCK.socketOptions}:}{(character string) If set to another value than \code{"NULL"}, then option \code{socketOptions} is set to this value on the workers during startup. See \code{\link[base:connections]{base::socketConnection()}} for details. (defaults to \code{"no-delay"})}
\item{\code{parallelly.makeNodePSOCK.rshcmd}:}{(character vector) The command to be run on the master to launch a process on another host.}
\item{\code{parallelly.makeNodePSOCK.rshopts}:}{(character vector) Addition command-line options appended to \code{rshcmd}. These arguments are only applied when connecting to non-localhost machines.}
\item{\code{parallelly.makeNodePSOCK.tries}:}{(integer) The maximum number of attempts done to launch each node. Only used when setting up cluster nodes using the sequential strategy.}
\item{\code{parallelly.makeNodePSOCK.tries.delay}:}{(numeric) The number of seconds to wait before trying to launch a cluster node that failed to launch previously. Only used when setting up cluster nodes using the sequential strategy.}
}
}
\section{Options for debugging}{
\describe{
\item{\code{parallelly.debug}:}{(logical) If \code{TRUE}, extensive debug messages are generated. (Default: \code{FALSE})}
}
}
\section{Environment variables that set R options}{
All of the above \R \verb{parallelly.*} options can be set by
corresponding environment variables \env{R_PARALLELLY_*} \emph{when the
\pkg{parallelly} package is loaded}.
For example, if \code{R_PARALLELLY_MAKENODEPSOCK_SETUP_STRATEGY="sequential"},
then option \code{parallelly.makeNodePSOCK.setup_strategy} is set to
\code{"sequential"} (character).
Similarly, if \code{R_PARALLELLY_AVAILABLECORES_FALLBACK="1"}, then option
\code{parallelly.availableCores.fallback} is set to \code{1} (integer).
}
\examples{
# Set an R option:
options(parallelly.availableCores.fallback = 1L)
}
\seealso{
To set \R options when \R starts (even before the \pkg{parallelly} package is loaded), see the \link[base]{Startup} help page. The \href{https://cran.r-project.org/package=startup}{\pkg{startup}} package provides a friendly mechanism for configuring \R's startup process.
}
|
