1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/relativeOverfitting.R
\name{estimateRelativeOverfitting}
\alias{estimateRelativeOverfitting}
\title{Estimate relative overfitting.}
\usage{
estimateRelativeOverfitting(
predish,
measures,
task,
learner = NULL,
pred.train = NULL,
iter = 1
)
}
\arguments{
\item{predish}{(\link{ResampleDesc} | \link{ResamplePrediction} | \link{Prediction})\cr
Resampling strategy or resampling prediction or test predictions.}
\item{measures}{(\link{Measure} | list of \link{Measure})\cr
Performance measure(s) to evaluate.
Default is the default measure for the task, see here \link{getDefaultMeasure}.}
\item{task}{(\link{Task})\cr
The task.}
\item{learner}{(\link{Learner} | \code{character(1)})\cr
The learner.
If you pass a string the learner will be created via \link{makeLearner}.}
\item{pred.train}{(\link{Prediction})\cr
Training predictions. Only needed if test predictions are passed.}
\item{iter}{(\link{integer})\cr
Iteration number. Default 1, usually you don't need to specify this. Only needed if test predictions are passed.}
}
\value{
(\link{data.frame}). Relative overfitting estimate(s), named by measure(s), for each resampling iteration.
}
\description{
Estimates the relative overfitting of a model as the ratio of the difference in test and train performance to the difference of test performance in the no-information case and train performance.
In the no-information case the features carry no information with respect to the prediction. This is simulated by permuting features and predictions.
}
\details{
Currently only support for classification and regression tasks is implemented.
}
\examples{
\dontshow{ if (requireNamespace("class")) \{ }
task = makeClassifTask(data = iris, target = "Species")
rdesc = makeResampleDesc("CV", iters = 2)
estimateRelativeOverfitting(rdesc, acc, task, makeLearner("classif.knn"))
estimateRelativeOverfitting(rdesc, acc, task, makeLearner("classif.lda"))
rpred = resample("classif.knn", task, rdesc)$pred
estimateRelativeOverfitting(rpred, acc, task)
\dontshow{ \} }
}
\references{
Bradley Efron and Robert Tibshirani; Improvements on Cross-Validation: The .632+ Bootstrap Method, Journal of the American Statistical Association, Vol. 92, No. 438. (Jun., 1997), pp. 548-560.
}
\seealso{
Other performance:
\code{\link{ConfusionMatrix}},
\code{\link{calculateConfusionMatrix}()},
\code{\link{calculateROCMeasures}()},
\code{\link{makeCostMeasure}()},
\code{\link{makeCustomResampledMeasure}()},
\code{\link{makeMeasure}()},
\code{\link{measures}},
\code{\link{performance}()},
\code{\link{setAggregation}()},
\code{\link{setMeasurePars}()}
}
\concept{performance}
|
