% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/loo.R
\name{loo}
\alias{loo}
\alias{loo.array}
\alias{loo.matrix}
\alias{loo.function}
\alias{loo_i}
\alias{is.loo}
\alias{is.psis_loo}
\title{Efficient approximate leave-one-out cross-validation (LOO)}
\usage{
loo(x, ...)

\method{loo}{array}(
  x,
  ...,
  r_eff = 1,
  save_psis = FALSE,
  cores = getOption("mc.cores", 1),
  is_method = c("psis", "tis", "sis")
)

\method{loo}{matrix}(
  x,
  ...,
  r_eff = 1,
  save_psis = FALSE,
  cores = getOption("mc.cores", 1),
  is_method = c("psis", "tis", "sis")
)

\method{loo}{`function`}(
  x,
  ...,
  data = NULL,
  draws = NULL,
  r_eff = 1,
  save_psis = FALSE,
  cores = getOption("mc.cores", 1),
  is_method = c("psis", "tis", "sis")
)

loo_i(i, llfun, ..., data = NULL, draws = NULL, r_eff = 1, is_method = "psis")

is.loo(x)

is.psis_loo(x)
}
\arguments{
\item{x}{A log-likelihood array, matrix, or function. The \strong{Methods (by class)}
section, below, has detailed descriptions of how to specify the inputs for
each method.}

\item{r_eff}{Vector of relative effective sample size estimates for the
likelihood (\code{exp(log_lik)}) of each observation. This is related to
the relative efficiency of estimating the normalizing term in
self-normalized importance sampling when using posterior draws obtained
with MCMC. If MCMC draws are used and \code{r_eff} is not provided then
the reported PSIS effective sample sizes and Monte Carlo error estimates
can be over-optimistic. If the posterior draws are (near) independent then
\code{r_eff=1} can be used. \code{r_eff} has to be a scalar (same value is used
for all observations) or a vector with length equal to the number of
observations. The default value is 1. See the \code{\link[=relative_eff]{relative_eff()}} helper
functions for help computing \code{r_eff}.}

\item{save_psis}{Should the \code{psis} object created internally by \code{loo()} be
saved in the returned object? The \code{loo()} function calls \code{\link[=psis]{psis()}}
internally but by default discards the (potentially large) \code{psis} object
after using it to compute the LOO-CV summaries. Setting \code{save_psis=TRUE}
will add a \code{psis_object} component to the list returned by \code{loo}.
This is useful if you plan to use the \code{\link[=E_loo]{E_loo()}} function to compute
weighted expectations after running \code{loo}. Several functions in the
\pkg{bayesplot} package also accept \code{psis} objects.}

\item{cores}{The number of cores to use for parallelization. This defaults to
the option \code{mc.cores} which can be set for an entire R session by
\code{options(mc.cores = NUMBER)}. The old option \code{loo.cores} is now
deprecated but will be given precedence over \code{mc.cores} until
\code{loo.cores} is removed in a future release. \strong{As of version
2.0.0 the default is now 1 core if \code{mc.cores} is not set}, but we
recommend using as many (or close to as many) cores as possible.
\itemize{
\item Note for Windows 10 users: it is \strong{strongly}
\href{https://github.com/stan-dev/loo/issues/94}{recommended} to avoid using
the \code{.Rprofile} file to set \code{mc.cores} (using the \code{cores} argument or
setting \code{mc.cores} interactively or in a script is fine).
}}

\item{is_method}{The importance sampling method to use. The following methods
are implemented:
\itemize{
\item \code{\link[=psis]{"psis"}}: Pareto-Smoothed Importance Sampling (PSIS). Default method.
\item \code{\link[=tis]{"tis"}}: Truncated Importance Sampling (TIS) with truncation at
\code{sqrt(S)}, where \code{S} is the number of posterior draws.
\item \code{\link[=sis]{"sis"}}: Standard Importance Sampling (SIS).
}}

\item{data, draws, ...}{For the \code{loo.function()} method and the \code{loo_i()}
function, these are the data, posterior draws, and other arguments to pass
to the log-likelihood function. See the \strong{Methods (by class)} section
below for details on how to specify these arguments.}

\item{i}{For \code{loo_i()}, an integer in \code{1:N}.}

\item{llfun}{For \code{loo_i()}, the same as \code{x} for the
\code{loo.function()} method. A log-likelihood function as described in the
\strong{Methods (by class)} section.}
}
\value{
The \code{loo()} methods return a named list with class
\code{c("psis_loo", "loo")} and components:
\describe{
\item{\code{estimates}}{
A matrix with two columns (\code{Estimate}, \code{SE}) and three rows (\code{elpd_loo},
\code{p_loo}, \code{looic}). This contains point estimates and standard errors of the
expected log pointwise predictive density (\code{\link[=loo-glossary]{elpd_loo}}), the
effective number of parameters (\code{\link[=loo-glossary]{p_loo}}) and the LOO
information criterion \code{looic} (which is just \code{-2 * elpd_loo}, i.e.,
converted to deviance scale).
}

\item{\code{pointwise}}{
A matrix with five columns (and number of rows equal to the number of
observations) containing the pointwise contributions of the measures
(\code{elpd_loo}, \code{mcse_elpd_loo}, \code{p_loo}, \code{looic}, \code{influence_pareto_k}).
in addition to the three measures in \code{estimates}, we also report
pointwise values of the Monte Carlo standard error of \code{\link[=loo-glossary]{elpd_loo}}
(\code{\link[=loo-glossary]{mcse_elpd_loo}}), and statistics describing the influence of
each observation on the posterior distribution (\code{influence_pareto_k}).
These are the estimates of the shape parameter \eqn{k} of the
generalized Pareto fit to the importance ratios for each leave-one-out
distribution (see the \link{pareto-k-diagnostic} page for details).
}

\item{\code{diagnostics}}{
A named list containing two vectors:
\itemize{
\item \code{pareto_k}: Importance sampling reliability diagnostics. By default,
these are equal to the \code{influence_pareto_k} in \code{pointwise}.
Some algorithms can improve importance sampling reliability and
modify these diagnostics. See the \link{pareto-k-diagnostic} page for details.
\item \code{n_eff}: PSIS effective sample size estimates.
}
}

\item{\code{psis_object}}{
This component will be \code{NULL} unless the \code{save_psis} argument is set to
\code{TRUE} when calling \code{loo()}. In that case \code{psis_object} will be the object
of class \code{"psis"} that is created when the \code{loo()} function calls \code{\link[=psis]{psis()}}
internally to do the PSIS procedure.
}
}

The \code{loo_i()} function returns a named list with components
\code{pointwise} and \code{diagnostics}. These components have the same
structure as the \code{pointwise} and \code{diagnostics} components of the
object returned by \code{loo()} except they contain results for only a single
observation.
}
\description{
The \code{loo()} methods for arrays, matrices, and functions compute PSIS-LOO
CV, efficient approximate leave-one-out (LOO) cross-validation for Bayesian
models using Pareto smoothed importance sampling (\link[=psis]{PSIS}). This is
an implementation of the methods described in Vehtari, Gelman, and Gabry
(2017) and Vehtari, Simpson, Gelman, Yao, and Gabry (2024).

The \code{loo_i()} function enables testing log-likelihood
functions for use with the \code{loo.function()} method.
}
\details{
The \code{loo()} function is an S3 generic and methods are provided for
3-D pointwise log-likelihood arrays, pointwise log-likelihood matrices, and
log-likelihood functions. The array and matrix methods are the most
convenient, but for models fit to very large datasets the \code{loo.function()}
method is more memory efficient and may be preferable.
}
\section{Methods (by class)}{
\itemize{
\item \code{loo(array)}: An \eqn{I} by \eqn{C} by \eqn{N} array, where \eqn{I}
is the number of MCMC iterations per chain, \eqn{C} is the number of
chains, and \eqn{N} is the number of data points.

\item \code{loo(matrix)}: An \eqn{S} by \eqn{N} matrix, where \eqn{S} is the size
of the posterior sample (with all chains merged) and \eqn{N} is the number
of data points.

\item \code{loo(`function`)}: A function \code{f()} that takes arguments \code{data_i} and \code{draws} and returns a
vector containing the log-likelihood for a single observation \code{i} evaluated
at each posterior draw. The function should be written such that, for each
observation \code{i} in \code{1:N}, evaluating

\if{html}{\out{<div class="sourceCode">}}\preformatted{f(data_i = data[i,, drop=FALSE], draws = draws)
}\if{html}{\out{</div>}}

results in a vector of length \code{S} (size of posterior sample). The
log-likelihood function can also have additional arguments but \code{data_i} and
\code{draws} are required.

If using the function method then the arguments \code{data} and \code{draws} must also
be specified in the call to \code{loo()}:
\itemize{
\item \code{data}: A data frame or matrix containing the data (e.g.
observed outcome and predictors) needed to compute the pointwise
log-likelihood. For each observation \code{i}, the \code{i}th row of
\code{data} will be passed to the \code{data_i} argument of the
log-likelihood function.
\item \code{draws}: An object containing the posterior draws for any
parameters needed to compute the pointwise log-likelihood. Unlike
\code{data}, which is indexed by observation, for each observation the
entire object \code{draws} will be passed to the \code{draws} argument of
the log-likelihood function.
\item The \code{...} can be used if your log-likelihood function takes additional
arguments. These arguments are used like the \code{draws} argument in that they
are recycled for each observation.
}

}}
\section{Defining \code{loo()} methods in a package}{
 Package developers can define
\code{loo()} methods for fitted models objects. See the example \code{loo.stanfit()}
method in the \strong{Examples} section below for an example of defining a
method that calls \code{loo.array()}. The \code{loo.stanreg()} method in the
\strong{rstanarm} package is an example of defining a method that calls
\code{loo.function()}.
}

\examples{
### Array and matrix methods (using example objects included with loo package)
# Array method
LLarr <- example_loglik_array()
rel_n_eff <- relative_eff(exp(LLarr))
loo(LLarr, r_eff = rel_n_eff, cores = 2)

# Matrix method
LLmat <- example_loglik_matrix()
rel_n_eff <- relative_eff(exp(LLmat), chain_id = rep(1:2, each = 500))
loo(LLmat, r_eff = rel_n_eff, cores = 2)


### Using log-likelihood function instead of array or matrix
set.seed(124)

# Simulate data and draw from posterior
N <- 50; K <- 10; S <- 100; a0 <- 3; b0 <- 2
p <- rbeta(1, a0, b0)
y <- rbinom(N, size = K, prob = p)
a <- a0 + sum(y); b <- b0 + N * K - sum(y)
fake_posterior <- as.matrix(rbeta(S, a, b))
dim(fake_posterior) # S x 1
fake_data <- data.frame(y,K)
dim(fake_data) # N x 2

llfun <- function(data_i, draws) {
  # each time called internally within loo the arguments will be equal to:
  # data_i: ith row of fake_data (fake_data[i,, drop=FALSE])
  # draws: entire fake_posterior matrix
  dbinom(data_i$y, size = data_i$K, prob = draws, log = TRUE)
}

# Use the loo_i function to check that llfun works on a single observation
# before running on all obs. For example, using the 3rd obs in the data:
loo_3 <- loo_i(i = 3, llfun = llfun, data = fake_data, draws = fake_posterior)
print(loo_3$pointwise[, "elpd_loo"])

# Use loo.function method (default r_eff=1 is used as this posterior not obtained via MCMC)
loo_with_fn <- loo(llfun, draws = fake_posterior, data = fake_data)

# If we look at the elpd_loo contribution from the 3rd obs it should be the
# same as what we got above with the loo_i function and i=3:
print(loo_with_fn$pointwise[3, "elpd_loo"])
print(loo_3$pointwise[, "elpd_loo"])

# Check that the loo.matrix method gives same answer as loo.function method
log_lik_matrix <- sapply(1:N, function(i) {
  llfun(data_i = fake_data[i,, drop=FALSE], draws = fake_posterior)
})
loo_with_mat <- loo(log_lik_matrix)
all.equal(loo_with_mat$estimates, loo_with_fn$estimates) # should be TRUE!


\dontrun{
### For package developers: defining loo methods

# An example of a possible loo method for 'stanfit' objects (rstan package).
# A similar method is included in the rstan package.
# In order for users to be able to call loo(stanfit) instead of
# loo.stanfit(stanfit) the NAMESPACE needs to be handled appropriately
# (roxygen2 and devtools packages are good for that).
#
loo.stanfit <-
 function(x,
         pars = "log_lik",
         ...,
         save_psis = FALSE,
         cores = getOption("mc.cores", 1)) {
  stopifnot(length(pars) == 1L)
  LLarray <- loo::extract_log_lik(stanfit = x,
                                  parameter_name = pars,
                                  merge_chains = FALSE)
  r_eff <- loo::relative_eff(x = exp(LLarray), cores = cores)
  loo::loo.array(LLarray,
                 r_eff = r_eff,
                 cores = cores,
                 save_psis = save_psis)
}
}


}
\references{
Vehtari, A., Gelman, A., and Gabry, J. (2017). Practical Bayesian model
evaluation using leave-one-out cross-validation and WAIC.
\emph{Statistics and Computing}. 27(5), 1413--1432. doi:10.1007/s11222-016-9696-4
(\href{https://link.springer.com/article/10.1007/s11222-016-9696-4}{journal version},
\href{https://arxiv.org/abs/1507.04544}{preprint arXiv:1507.04544}).

Vehtari, A., Simpson, D., Gelman, A., Yao, Y., and Gabry, J. (2024).
Pareto smoothed importance sampling. \emph{Journal of Machine Learning Research},
25(72):1-58.
\href{https://jmlr.org/papers/v25/19-556.html}{PDF}
}
\seealso{
\itemize{
\item The \strong{loo} package \href{https://mc-stan.org/loo/articles/index.html}{vignettes}
for demonstrations.
\item The \href{https://mc-stan.org/loo/articles/online-only/faq.html}{FAQ page} on
the \strong{loo} website for answers to frequently asked questions.
\item \code{\link[=psis]{psis()}} for the underlying Pareto Smoothed Importance Sampling (PSIS)
procedure used in the LOO-CV approximation.
\item \link{pareto-k-diagnostic} for convenience functions for looking at diagnostics.
\item \code{\link[=loo_compare]{loo_compare()}} for model comparison.
}
}