File: performance.Rd

package info (click to toggle)
r-cran-rocr 1.0-11-2
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 676 kB
  • sloc: sh: 13; makefile: 2
file content (220 lines) | stat: -rw-r--r-- 11,915 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/performance.R
\name{performance}
\alias{performance}
\title{Function to create performance objects}
\usage{
performance(prediction.obj, measure, x.measure = "cutoff", ...)
}
\arguments{
\item{prediction.obj}{An object of class \code{prediction}.}

\item{measure}{Performance measure to use for the evaluation. A complete list
of the performance measures that are available for \code{measure} and
\code{x.measure} is given in the 'Details' section.}

\item{x.measure}{A second performance measure. If different from the default,
a two-dimensional curve, with \code{x.measure} taken to be the unit in
direction of the x axis, and \code{measure} to be the unit in direction of
the y axis, is created. This curve is parametrized with the cutoff.}

\item{...}{Optional arguments (specific to individual performance measures).}
}
\value{
An S4 object of class \code{performance}.
}
\description{
All kinds of predictor evaluations are performed using this function.
}
\details{
Here is the list of available performance measures. Let Y and
\eqn{\hat{Y}}{Yhat} be random variables representing the class and the prediction for
a randomly drawn sample, respectively. We denote by
\eqn{\oplus}{+} and \eqn{\ominus}{-} the positive and
negative class, respectively. Further, we use the following
abbreviations for empirical quantities: P (\# positive
samples), N (\# negative samples), TP (\# true positives), TN (\# true
negatives), FP (\# false positives), FN (\# false negatives).
\describe{
 \item{\code{acc}:}{Accuracy. \eqn{P(\hat{Y}=Y)}{P(Yhat = Y)}. Estimated
   as: \eqn{\frac{TP+TN}{P+N}}{(TP+TN)/(P+N)}.}
 \item{\code{err}:}{Error rate. \eqn{P(\hat{Y}\ne Y)}{P(Yhat !=
                                                          Y)}. Estimated as: \eqn{\frac{FP+FN}{P+N}}{(FP+FN)/(P+N)}.}
 \item{\code{fpr}:}{False positive rate. \eqn{P(\hat{Y}=\oplus | Y =
                                                   \ominus)}{P(Yhat = + | Y = -)}. Estimated as:
     \eqn{\frac{FP}{N}}{FP/N}.}
 \item{\code{fall}:}{Fallout. Same as \code{fpr}.}
 \item{\code{tpr}:}{True positive
   rate. \eqn{P(\hat{Y}=\oplus|Y=\oplus)}{P(Yhat = + | Y = +)}. Estimated
   as: \eqn{\frac{TP}{P}}{TP/P}.}
 \item{\code{rec}:}{Recall. Same as \code{tpr}.}
 \item{\code{sens}:}{Sensitivity. Same as \code{tpr}.}
 \item{\code{fnr}:}{False negative
   rate. \eqn{P(\hat{Y}=\ominus|Y=\oplus)}{P(Yhat = - | Y =
                                               +)}. Estimated as: \eqn{\frac{FN}{P}}{FN/P}.}
 \item{\code{miss}:}{Miss. Same as \code{fnr}.}
 \item{\code{tnr}:}{True negative rate. \eqn{P(\hat{Y} =
                                                  \ominus|Y=\ominus)}{P(Yhat = - | Y = -)}.}
 \item{\code{spec}:}{Specificity. Same as \code{tnr}.}
 \item{\code{ppv}:}{Positive predictive
   value. \eqn{P(Y=\oplus|\hat{Y}=\oplus)}{P(Y = + | Yhat =
                                               +)}. Estimated as: \eqn{\frac{TP}{TP+FP}}{TP/(TP+FP)}.}
 \item{\code{prec}:}{Precision. Same as \code{ppv}.}
 \item{\code{npv}:}{Negative predictive
   value. \eqn{P(Y=\ominus|\hat{Y}=\ominus)}{P(Y = - | Yhat =
                                                 -)}. Estimated as: \eqn{\frac{TN}{TN+FN}}{TN/(TN+FN)}.}
 \item{\code{pcfall}:}{Prediction-conditioned
   fallout. \eqn{P(Y=\ominus|\hat{Y}=\oplus)}{P(Y = - | Yhat =
                                                  +)}. Estimated as: \eqn{\frac{FP}{TP+FP}}{FP/(TP+FP)}.}
 \item{\code{pcmiss}:}{Prediction-conditioned
   miss. \eqn{P(Y=\oplus|\hat{Y}=\ominus)}{P(Y = + | Yhat =
                                               -)}. Estimated as: \eqn{\frac{FN}{TN+FN}}{FN/(TN+FN)}.}
 \item{\code{rpp}:}{Rate of positive predictions. \eqn{P( \hat{Y} =
                                                           \oplus)}{P(Yhat = +)}. Estimated as: (TP+FP)/(TP+FP+TN+FN).}
 \item{\code{rnp}:}{Rate of negative predictions. \eqn{P( \hat{Y} =
                                                           \ominus)}{P(Yhat = -)}. Estimated as: (TN+FN)/(TP+FP+TN+FN).}
 \item{\code{phi}:}{Phi correlation coefficient. \eqn{\frac{TP \cdot
   TN - FP \cdot FN}{\sqrt{ (TP+FN) \cdot (TN+FP) \cdot (TP+FP)
     \cdot (TN+FN)}}}{(TP*TN -
                         FP*FN)/(sqrt((TP+FN)*(TN+FP)*(TP+FP)*(TN+FN)))}. Yields a
   number between -1 and 1, with 1 indicating a perfect
   prediction, 0 indicating a random prediction. Values below 0
   indicate a worse than random prediction.}
 \item{\code{mat}:}{Matthews correlation coefficient. Same as \code{phi}.}
 \item{\code{mi}:}{Mutual information. \eqn{I(\hat{Y},Y) := H(Y) -
     H(Y|\hat{Y})}{I(Yhat, Y) := H(Y) - H(Y | Yhat)}, where H is the
   (conditional) entropy. Entropies are estimated naively (no bias
                                                           correction).}
 \item{\code{chisq}:}{Chi square test statistic. \code{?chisq.test}
   for details. Note that R might raise a warning if the sample size
   is too small.}
 \item{\code{odds}:}{Odds ratio. \eqn{\frac{TP \cdot TN}{FN \cdot
   FP}}{(TP*TN)/(FN*FP)}. Note that odds ratio produces
   Inf or NA values for all cutoffs corresponding to FN=0 or
   FP=0. This can substantially decrease the plotted cutoff region.}
 \item{\code{lift}:}{Lift
   value. \eqn{\frac{P(\hat{Y}=\oplus|Y=\oplus)}{P(\hat{Y}=\oplus)}}{P(Yhat = + |
                                                                         Y = +)/P(Yhat = +)}.}
 \item{\code{f}:}{Precision-recall F measure (van Rijsbergen, 1979). Weighted
   harmonic mean of precision (P) and recall (R). \eqn{F =
     \frac{1}{\alpha \frac{1}{P} + (1-\alpha)\frac{1}{R}}}{F = 1/
       (alpha*1/P + (1-alpha)*1/R)}. If
   \eqn{\alpha=\frac{1}{2}}{alpha=1/2}, the mean is balanced. A
   frequent equivalent formulation is
   \eqn{F = \frac{(\beta^2+1) \cdot P \cdot R}{R + \beta^2 \cdot
     P}}{F = (beta^2+1) * P * R / (R + beta^2 * P)}. In this formulation, the
     mean is balanced if \eqn{\beta=1}{beta=1}. Currently, ROCR only accepts
     the alpha version as input (e.g. \eqn{\alpha=0.5}{alpha=0.5}). If no 
     value for alpha is given, the mean will be balanced by default.}
 \item{\code{rch}:}{ROC convex hull. A ROC (=\code{tpr} vs \code{fpr}) curve 
   with concavities (which represent suboptimal choices of cutoff) removed 
   (Fawcett 2001). Since the result is already a parametric performance 
   curve, it cannot be used in combination with other measures.}
 \item{\code{auc}:}{Area under the ROC curve. This is equal to the value of the
   Wilcoxon-Mann-Whitney test statistic and also the probability that the
   classifier will score are randomly drawn positive sample higher than a
   randomly drawn negative sample. Since the output of
   \code{auc} is cutoff-independent, this
   measure cannot be combined with other measures into a parametric
   curve. The partial area under the ROC curve up to a given false
   positive rate can be calculated by passing the optional parameter
   \code{fpr.stop=0.5} (or any other value between 0 and 1) to 
   \code{performance}.}
 \item{\code{aucpr}:}{Area under the Precision/Recall curve. Since the output
   of \code{aucpr} is cutoff-independent, this measure cannot be combined 
   with other measures into a parametric curve.}
 \item{\code{prbe}:}{Precision-recall break-even point. The cutoff(s) where
   precision and recall are equal. At this point, positive and negative
   predictions are made at the same rate as their prevalence in the
   data. Since the output of
   \code{prbe} is just a cutoff-independent scalar, this
   measure cannot be combined with other measures into a parametric curve.}
 \item{\code{cal}:}{Calibration error. The calibration error is the
   absolute difference between predicted confidence and actual reliability. This
   error is estimated at all cutoffs by sliding a window across the
   range of possible cutoffs. The default window size of 100 can be
   adjusted by passing the optional parameter \code{window.size=200}
   to \code{performance}. E.g., if for several
   positive samples the output of the classifier is around 0.75, you might
   expect from a well-calibrated classifier that the fraction of them
   which is correctly predicted as positive is also around 0.75. In a
   well-calibrated classifier, the probabilistic confidence estimates
   are realistic. Only for use with
   probabilistic output (i.e. scores between 0 and 1).}
 \item{\code{mxe}:}{Mean cross-entropy. Only for use with
   probabilistic output. \eqn{MXE :=-\frac{1}{P+N}( \sum_{y_i=\oplus}
                                                   ln(\hat{y}_i) + \sum_{y_i=\ominus} ln(1-\hat{y}_i))}{MXE := - 1/(P+N) \sum_{y_i=+}
                                                     ln(yhat_i) + \sum_{y_i=-} ln(1-yhat_i)}. Since the output of
   \code{mxe} is just a cutoff-independent scalar, this
   measure cannot be combined with other measures into a parametric curve.}
 \item{\code{rmse}:}{Root-mean-squared error. Only for use with
   numerical class labels. \eqn{RMSE:=\sqrt{\frac{1}{P+N}\sum_i (y_i
                                                                 - \hat{y}_i)^2}}{RMSE := sqrt(1/(P+N) \sum_i (y_i -
                                                                                                                 yhat_i)^2)}. Since the output of
   \code{rmse} is just a cutoff-independent scalar, this
   measure cannot be combined with other measures into a parametric curve.}
 \item{\code{sar}:}{Score combinining performance measures of different
   characteristics, in the attempt of creating a more "robust"
   measure (cf. Caruana R., ROCAI2004):
     SAR = 1/3 * ( Accuracy + Area under the ROC curve + Root
                   mean-squared error ).}
 \item{\code{ecost}:}{Expected cost. For details on cost curves,
   cf. Drummond&Holte 2000,2004. \code{ecost} has an obligatory x
   axis, the so-called 'probability-cost function'; thus it cannot be
   combined with other measures. While using \code{ecost} one is
   interested in the lower envelope of a set of lines, it might be
   instructive to plot the whole set of lines in addition to the lower
   envelope. An example is given in \code{demo(ROCR)}.}
 \item{\code{cost}:}{Cost of a classifier when
   class-conditional misclassification costs are explicitly given.
   Accepts the optional parameters \code{cost.fp} and
   \code{cost.fn}, by which the costs for false positives and
   negatives can be adjusted, respectively. By default, both are set
   to 1.}
}
}
\note{
Here is how to call \code{performance()} to create some standard
evaluation plots:
\describe{
  \item{ROC curves:}{measure="tpr", x.measure="fpr".}
  \item{Precision/recall graphs:}{measure="prec", x.measure="rec".}
  \item{Sensitivity/specificity plots:}{measure="sens", x.measure="spec".}
  \item{Lift charts:}{measure="lift", x.measure="rpp".}
}
}
\examples{
# computing a simple ROC curve (x-axis: fpr, y-axis: tpr)
library(ROCR)
data(ROCR.simple)
pred <- prediction( ROCR.simple$predictions, ROCR.simple$labels)
pred
perf <- performance(pred,"tpr","fpr")
perf
plot(perf)

# precision/recall curve (x-axis: recall, y-axis: precision)
perf <- performance(pred, "prec", "rec")
perf
plot(perf)

# sensitivity/specificity curve (x-axis: specificity,
# y-axis: sensitivity)
perf <- performance(pred, "sens", "spec")
perf
plot(perf)
}
\references{
A detailed list of references can be found on the ROCR homepage at
\url{http://rocr.bioinf.mpi-sb.mpg.de}.
}
\seealso{
\code{\link{prediction}},
\code{\link{prediction-class}},
\code{\link{performance-class}},
\code{\link{plot.performance}}
}
\author{
Tobias Sing \email{tobias.sing@gmail.com}, Oliver Sander
\email{osander@gmail.com}
}