1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/resamples.R
\name{resamples}
\alias{resamples}
\alias{resamples.default}
\alias{summary.resamples}
\alias{sort.resamples}
\alias{as.matrix.resamples}
\alias{as.data.frame.resamples}
\alias{modelCor}
\alias{print.resamples}
\title{Collation and Visualization of Resampling Results}
\usage{
resamples(x, ...)
\method{resamples}{default}(x, modelNames = names(x), ...)
\method{sort}{resamples}(x, decreasing = FALSE, metric = x$metric[1], FUN = mean, ...)
\method{summary}{resamples}(object, metric = object$metrics, ...)
\method{as.matrix}{resamples}(x, metric = x$metric[1], ...)
\method{as.data.frame}{resamples}(x, row.names = NULL, optional = FALSE, metric = x$metric[1], ...)
modelCor(x, metric = x$metric[1], ...)
\method{print}{resamples}(x, ...)
}
\arguments{
\item{x}{a list of two or more objects of class \code{\link{train}},
\code{\link{sbf}} or \code{\link{rfe}} with a common set of resampling
indices in the \code{control} object. For \code{sort.resamples}, it is an
object generated by \code{resamples}.}
\item{\dots}{only used for \code{sort} and \code{modelCor} and captures
arguments to pass to \code{sort} or \code{FUN}.}
\item{modelNames}{an optional set of names to give to the resampling results}
\item{decreasing}{logical. Should the sort be increasing or decreasing?}
\item{metric}{a character string for the performance measure used to sort or
computing the between-model correlations}
\item{FUN}{a function whose first argument is a vector and returns a scalar,
to be applied to each model's performance measure.}
\item{object}{an object generated by \code{resamples}}
\item{row.names, optional}{not currently used but included for consistency
with \code{as.data.frame}}
}
\value{
For \code{resamples}: an object with class \code{"resamples"} with
elements \item{call }{the call} \item{values }{a data frame of results where
rows correspond to resampled data sets and columns indicate the model and
metric} \item{models }{a character string of model labels} \item{metrics }{a
character string of performance metrics} \item{methods }{a character string
of the \code{\link{train}} \code{method} argument values for each model }
For \code{sort.resamples} a character string in the sorted order is
generated. \code{modelCor} returns a correlation matrix.
}
\description{
These functions provide methods for collection, analyzing and visualizing a
set of resampling results from a common data set.
}
\details{
The ideas and methods here are based on Hothorn et al. (2005) and Eugster et
al. (2008).
The results from \code{\link{train}} can have more than one performance
metric per resample. Each metric in the input object is saved.
\code{resamples} checks that the resampling results match; that is, the
indices in the object \code{trainObject$control$index} are the same. Also,
the argument \code{\link{trainControl}} \code{returnResamp} should have a
value of \code{"final"} for each model.
The summary function computes summary statistics across each model/metric
combination.
}
\examples{
data(BloodBrain)
set.seed(1)
## tmp <- createDataPartition(logBBB,
## p = .8,
## times = 100)
## rpartFit <- train(bbbDescr, logBBB,
## "rpart",
## tuneLength = 16,
## trControl = trainControl(
## method = "LGOCV", index = tmp))
## ctreeFit <- train(bbbDescr, logBBB,
## "ctree",
## trControl = trainControl(
## method = "LGOCV", index = tmp))
## earthFit <- train(bbbDescr, logBBB,
## "earth",
## tuneLength = 20,
## trControl = trainControl(
## method = "LGOCV", index = tmp))
## or load pre-calculated results using:
## load(url("http://caret.r-forge.r-project.org/exampleModels.RData"))
## resamps <- resamples(list(CART = rpartFit,
## CondInfTree = ctreeFit,
## MARS = earthFit))
## resamps
## summary(resamps)
}
\references{
Hothorn et al. The design and analysis of benchmark experiments.
Journal of Computational and Graphical Statistics (2005) vol. 14 (3) pp.
675-699
Eugster et al. Exploratory and inferential analysis of benchmark
experiments. Ludwigs-Maximilians-Universitat Munchen, Department of
Statistics, Tech. Rep (2008) vol. 30
}
\seealso{
\code{\link{train}}, \code{\link{trainControl}},
\code{\link{diff.resamples}}, \code{\link{xyplot.resamples}},
\code{\link{densityplot.resamples}}, \code{\link{bwplot.resamples}},
\code{\link{splom.resamples}}
}
\author{
Max Kuhn
}
\keyword{models}
|
