1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
test_that("test_CritDifferences", {
lrns = list(
makeLearner("classif.rpart"),
makeLearner("classif.nnet"))
tasks = list(multiclass.task, binaryclass.task)
rdesc = makeResampleDesc("Holdout")
meas = list(acc, ber)
res = benchmark(lrns, tasks, rdesc, meas)
# Case: Make sure rpart is better then nnet in regards to
# ber. Minimal p.value for only 2 learners ~.157
res$results$binary$classif.nnet$aggr[2] = 1
res$results$multiclass$classif.nnet$aggr[2] = 1
expect_warning({
r1 = generateCritDifferencesData(res)
})
expect_s3_class(r1, "CritDifferencesData")
expect_warning({
r2 = generateCritDifferencesData(res, ber, test = "nemenyi")
})
expect_s3_class(r2, "CritDifferencesData")
r3 = generateCritDifferencesData(res, ber, p.value = 0.5, test = "bd")
expect_s3_class(r3, "CritDifferencesData")
# Test Issue #554 (equally performing learners)
lrns2 = list(
makeLearner("classif.rpart", "rpart1"),
makeLearner("classif.rpart", "rpart2"))
res2 = benchmark(lrns2, tasks, rdesc, meas)
expect_warning(
{
r4 = generateCritDifferencesData(res2, acc, p.value = 0.3, test = "bd")
},
"Learner performances might be exactly equal.")
expect_s3_class(r4, "CritDifferencesData")
plotCritDifferences(r1)
suppressMessages(ggsave(tempfile(fileext = ".png")))
plotCritDifferences(r2)
suppressMessages(ggsave(tempfile(fileext = ".png")))
plotCritDifferences(r3, baseline = "classif.rpart")
suppressMessages(ggsave(tempfile(fileext = ".png")))
plotCritDifferences(r4)
suppressMessages(ggsave(tempfile(fileext = ".png")))
})
|
