1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
test_that("relativeOverfitting works with ResampleDesc", {
rdesc = makeResampleDesc("CV", iters = 2)
ro = estimateRelativeOverfitting(rdesc, acc, multiclass.small.task, makeLearner("classif.knn"))
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.acc))
expect_equal(nrow(ro), 2)
})
test_that("relativeOverfitting works with multiple measures with ResampleDesc", {
rdesc = makeResampleDesc("CV", iters = 2)
ro = estimateRelativeOverfitting(rdesc, list(acc, mmce), multiclass.small.task, makeLearner("classif.knn"))
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.acc))
expect_true(is.numeric(ro$relative.overfit.mmce))
expect_equal(nrow(ro), 2)
})
test_that("relativeOverfitting works for regression with ResampleDesc", {
rdesc = makeResampleDesc("CV", iters = 2)
ro = estimateRelativeOverfitting(rdesc, mse, regr.small.task, makeLearner("regr.rpart"))
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.mse))
expect_equal(nrow(ro), 2)
})
test_that("relativeOverfitting works with ResamplePrediction", {
rdesc = makeResampleDesc("CV", iters = 2)
rpred = resample("classif.knn", multiclass.small.task, rdesc)$pred
ro = estimateRelativeOverfitting(rpred, acc, multiclass.small.task)
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.acc))
expect_equal(nrow(ro), 2)
})
test_that("relativeOverfitting works with multiple measures with ResamplePrediction", {
rdesc = makeResampleDesc("CV", iters = 2)
rpred = resample("classif.knn", multiclass.small.task, rdesc)$pred
ro = estimateRelativeOverfitting(rpred, list(acc, mmce), multiclass.small.task)
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.acc))
expect_true(is.numeric(ro$relative.overfit.mmce))
expect_equal(nrow(ro), 2)
})
test_that("relativeOverfitting works for regression with ResamplePrediction", {
rdesc = makeResampleDesc("CV", iters = 2)
rpred = resample("regr.rpart", regr.small.task, rdesc)$pred
ro = estimateRelativeOverfitting(rpred, mse, regr.small.task)
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.mse))
expect_equal(nrow(ro), 2)
})
test_that("relativeOverfitting works with train/test", {
mod = train(makeLearner("classif.knn"), multiclass.small.task, subset = multiclass.small.train.inds)
pred.train = predict(mod, task = multiclass.small.task, subset = multiclass.small.train.inds)
pred.test = predict(mod, task = multiclass.small.task, subset = multiclass.small.test.inds)
ro = estimateRelativeOverfitting(pred.test, acc, multiclass.small.task, pred.train = pred.train)
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.acc))
expect_equal(nrow(ro), 1)
})
test_that("relativeOverfitting works with multiple measures with train/test", {
mod = train(makeLearner("classif.knn"), multiclass.small.task, subset = multiclass.small.train.inds)
pred.train = predict(mod, task = multiclass.small.task, subset = multiclass.small.train.inds)
pred.test = predict(mod, task = multiclass.small.task, subset = multiclass.small.test.inds)
ro = estimateRelativeOverfitting(pred.test, list(acc, mmce), multiclass.small.task, pred.train = pred.train)
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.acc))
expect_true(is.numeric(ro$relative.overfit.mmce))
expect_equal(nrow(ro), 1)
})
test_that("relativeOverfitting works for regression with train/test", {
mod = train(makeLearner("regr.rpart"), regr.small.task, subset = regr.small.train.inds)
pred.train = predict(mod, task = regr.small.task, subset = regr.small.train.inds)
pred.test = predict(mod, task = regr.small.task, subset = regr.small.test.inds)
ro = estimateRelativeOverfitting(pred.test, mse, regr.small.task, pred.train = pred.train)
expect_true(is.data.frame(ro))
expect_true(is.numeric(ro$relative.overfit.mse))
expect_equal(nrow(ro), 1)
})
|
