1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
|
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/plotCritDifferences.R
\name{generateCritDifferencesData}
\alias{generateCritDifferencesData}
\title{Generate data for critical-differences plot.}
\usage{
generateCritDifferencesData(
bmr,
measure = NULL,
p.value = 0.05,
baseline = NULL,
test = "bd"
)
}
\arguments{
\item{bmr}{(\link{BenchmarkResult})\cr
Benchmark result.}
\item{measure}{(\link{Measure})\cr
Performance measure.
Default is the first measure used in the benchmark experiment.}
\item{p.value}{(\code{numeric(1)})\cr
P-value for the critical difference. Default: 0.05}
\item{baseline}{(\code{character(1)}): (\code{learner.id}) \cr
Select a \code{learner.id} as baseline for the \code{test = "bd"}
("Bonferroni-Dunn") critical differences
diagram. The critical difference interval will then be positioned arround this learner.
Defaults to best performing algorithm. \cr
For \code{test = "nemenyi"}, no baseline is needed as it performs \emph{all pairwise
comparisons}.}
\item{test}{(\code{character(1)}) \cr
Test for which the critical differences are computed. \cr
\dQuote{bd} for the Bonferroni-Dunn Test, which is comparing all
classifiers to a \code{baseline}, thus performing a comparison
of one classifier to all others. \cr
Algorithms not connected by a single line are statistically different
from the baseline. \cr
\dQuote{nemenyi} for the \link[PMCMRplus:frdAllPairsNemenyiTest]{PMCMRplus::frdAllPairsNemenyiTest}
which is comparing all classifiers to each other. The null hypothesis that
there is a difference between the classifiers can not be rejected for all
classifiers that have a single grey bar connecting them.}
}
\value{
(\code{critDifferencesData}). List containing:
\item{data}{(data.frame) containing the info for the descriptive
part of the plot}
\item{friedman.nemenyi.test}{(list) of class \code{pairwise.htest} \cr
contains the calculated
\link[PMCMRplus:frdAllPairsNemenyiTest]{PMCMRplus::frdAllPairsNemenyiTest}}
\item{cd.info}{(list) containing info on the critical difference
and its positioning}
\item{baseline}{\code{baseline} chosen for plotting}
\item{p.value}{p.value used for the \link[PMCMRplus:frdAllPairsNemenyiTest]{PMCMRplus::frdAllPairsNemenyiTest}
and for computation of the critical difference}
}
\description{
Generates data that can be used to plot a
critical differences plot. Computes the critical differences according
to either the
\code{"Bonferroni-Dunn"} test or the \code{"Nemenyi"} test.\cr
\code{"Bonferroni-Dunn"} usually yields higher power as it does not
compare all algorithms to each other, but all algorithms to a
\code{baseline} instead. \cr
Learners are drawn on the y-axis according to their average rank. \cr
For \code{test = "nemenyi"} a bar is drawn, connecting all groups of not
significantly different learners.\cr
For \code{test = "bd"} an interval is drawn arround the algorithm selected
as a baseline. All learners within this interval are not signifcantly different
from the baseline. \cr
Calculation:
\deqn{CD = q_{\alpha} \sqrt{\left(\frac{k(k+1)}{6N}\right)}}{CD = q_alpha sqrt(k(k+1)/(6N))} \cr
Where \eqn{q_\alpha} is based on the studentized range statistic.
See references for details.
}
\seealso{
Other generate_plot_data:
\code{\link{generateCalibrationData}()},
\code{\link{generateFeatureImportanceData}()},
\code{\link{generateFilterValuesData}()},
\code{\link{generateLearningCurveData}()},
\code{\link{generatePartialDependenceData}()},
\code{\link{generateThreshVsPerfData}()},
\code{\link{plotFilterValues}()}
Other benchmark:
\code{\link{BenchmarkResult}},
\code{\link{batchmark}()},
\code{\link{benchmark}()},
\code{\link{convertBMRToRankMatrix}()},
\code{\link{friedmanPostHocTestBMR}()},
\code{\link{friedmanTestBMR}()},
\code{\link{getBMRAggrPerformances}()},
\code{\link{getBMRFeatSelResults}()},
\code{\link{getBMRFilteredFeatures}()},
\code{\link{getBMRLearnerIds}()},
\code{\link{getBMRLearnerShortNames}()},
\code{\link{getBMRLearners}()},
\code{\link{getBMRMeasureIds}()},
\code{\link{getBMRMeasures}()},
\code{\link{getBMRModels}()},
\code{\link{getBMRPerformances}()},
\code{\link{getBMRPredictions}()},
\code{\link{getBMRTaskDescs}()},
\code{\link{getBMRTaskIds}()},
\code{\link{getBMRTuneResults}()},
\code{\link{plotBMRBoxplots}()},
\code{\link{plotBMRRanksAsBarChart}()},
\code{\link{plotBMRSummary}()},
\code{\link{plotCritDifferences}()},
\code{\link{reduceBatchmarkResults}()}
}
\concept{benchmark}
\concept{generate_plot_data}
|