1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/resamples.R
\name{diff.resamples}
\alias{diff.resamples}
\alias{summary.diff.resamples}
\alias{compare_models}
\title{Inferential Assessments About Model Performance}
\usage{
\method{diff}{resamples}(
x,
models = x$models,
metric = x$metrics,
test = t.test,
confLevel = 0.95,
adjustment = "bonferroni",
...
)
\method{summary}{diff.resamples}(object, digits = max(3, getOption("digits") - 3), ...)
compare_models(a, b, metric = a$metric[1])
}
\arguments{
\item{x}{an object generated by \code{resamples}}
\item{models}{a character string for which models to compare}
\item{metric}{a character string for which metrics to compare}
\item{test}{a function to compute differences. The output of this function
should have scalar outputs called \code{estimate} and \code{p.value}}
\item{confLevel}{confidence level to use for
\code{\link{dotplot.diff.resamples}}. See Details below.}
\item{adjustment}{any p-value adjustment method to pass to
\code{\link[stats]{p.adjust}}.}
\item{\dots}{further arguments to pass to \code{test}}
\item{object}{a object generated by \code{diff.resamples}}
\item{digits}{the number of significant differences to display when printing}
\item{a, b}{two objects of class \code{\link{train}}, \code{\link{sbf}} or
\code{\link{rfe}} with a common set of resampling indices in the
\code{control} object.}
}
\value{
An object of class \code{"diff.resamples"} with elements: \item{call
}{the call} \item{difs }{a list for each metric being compared. Each list
contains a matrix with differences in columns and resamples in rows }
\item{statistics }{a list of results generated by \code{test}}
\item{adjustment}{the p-value adjustment used} \item{models}{a character
string for which models were compared.} \item{metrics }{a character string
of performance metrics that were used}
or...
An object of class \code{"summary.diff.resamples"} with elements: \item{call
}{the call} \item{table }{a list of tables that show the differences and
p-values }
...or (for \code{compare_models}) an object of class \code{htest} resulting
from \code{\link[stats]{t.test}}.
}
\description{
Methods for making inferences about differences between models
}
\details{
The ideas and methods here are based on Hothorn et al. (2005) and Eugster et
al. (2008).
For each metric, all pair-wise differences are computed and tested to assess
if the difference is equal to zero.
When a Bonferroni correction is used, the confidence level is changed from
\code{confLevel} to \code{1-((1-confLevel)/p)} here \code{p} is the number
of pair-wise comparisons are being made. For other correction methods, no
such change is used.
\code{compare_models} is a shorthand function to compare two models using a
single metric. It returns the results of \code{\link[stats]{t.test}} on the
differences.
}
\examples{
\dontrun{
#load(url("http://topepo.github.io/caret/exampleModels.RData"))
resamps <- resamples(list(CART = rpartFit,
CondInfTree = ctreeFit,
MARS = earthFit))
difs <- diff(resamps)
difs
summary(difs)
compare_models(rpartFit, ctreeFit)
}
}
\references{
Hothorn et al. The design and analysis of benchmark experiments.
Journal of Computational and Graphical Statistics (2005) vol. 14 (3) pp.
675-699
Eugster et al. Exploratory and inferential analysis of benchmark
experiments. Ludwigs-Maximilians-Universitat Munchen, Department of
Statistics, Tech. Rep (2008) vol. 30
}
\seealso{
\code{\link{resamples}}, \code{\link{dotplot.diff.resamples}},
\code{\link{densityplot.diff.resamples}},
\code{\link{bwplot.diff.resamples}}, \code{\link{levelplot.diff.resamples}}
}
\author{
Max Kuhn
}
\keyword{models}
|
