1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
"""
Demo for training continuation
==============================
"""
import os
import pickle
import tempfile
from sklearn.datasets import load_breast_cancer
import xgboost
def training_continuation(tmpdir: str, use_pickle: bool) -> None:
"""Basic training continuation."""
# Train 128 iterations in 1 session
X, y = load_breast_cancer(return_X_y=True)
clf = xgboost.XGBClassifier(n_estimators=128, eval_metric="logloss")
clf.fit(X, y, eval_set=[(X, y)])
print("Total boosted rounds:", clf.get_booster().num_boosted_rounds())
# Train 128 iterations in 2 sessions, with the first one runs for 32 iterations and
# the second one runs for 96 iterations
clf = xgboost.XGBClassifier(n_estimators=32, eval_metric="logloss")
clf.fit(X, y, eval_set=[(X, y)])
assert clf.get_booster().num_boosted_rounds() == 32
# load back the model, this could be a checkpoint
if use_pickle:
path = os.path.join(tmpdir, "model-first-32.pkl")
with open(path, "wb") as fd:
pickle.dump(clf, fd)
with open(path, "rb") as fd:
loaded = pickle.load(fd)
else:
path = os.path.join(tmpdir, "model-first-32.json")
clf.save_model(path)
loaded = xgboost.XGBClassifier()
loaded.load_model(path)
clf = xgboost.XGBClassifier(n_estimators=128 - 32, eval_metric="logloss")
clf.fit(X, y, eval_set=[(X, y)], xgb_model=loaded)
print("Total boosted rounds:", clf.get_booster().num_boosted_rounds())
assert clf.get_booster().num_boosted_rounds() == 128
def training_continuation_early_stop(tmpdir: str, use_pickle: bool) -> None:
"""Training continuation with early stopping."""
early_stopping_rounds = 5
early_stop = xgboost.callback.EarlyStopping(
rounds=early_stopping_rounds, save_best=True
)
n_estimators = 512
X, y = load_breast_cancer(return_X_y=True)
clf = xgboost.XGBClassifier(
n_estimators=n_estimators, eval_metric="logloss", callbacks=[early_stop]
)
clf.fit(X, y, eval_set=[(X, y)])
print("Total boosted rounds:", clf.get_booster().num_boosted_rounds())
best = clf.best_iteration
# Train 512 iterations in 2 sessions, with the first one runs for 128 iterations and
# the second one runs until early stop.
clf = xgboost.XGBClassifier(
n_estimators=128, eval_metric="logloss", callbacks=[early_stop]
)
# Reinitialize the early stop callback
early_stop = xgboost.callback.EarlyStopping(
rounds=early_stopping_rounds, save_best=True
)
clf.set_params(callbacks=[early_stop])
clf.fit(X, y, eval_set=[(X, y)])
assert clf.get_booster().num_boosted_rounds() == 128
# load back the model, this could be a checkpoint
if use_pickle:
path = os.path.join(tmpdir, "model-first-128.pkl")
with open(path, "wb") as fd:
pickle.dump(clf, fd)
with open(path, "rb") as fd:
loaded = pickle.load(fd)
else:
path = os.path.join(tmpdir, "model-first-128.json")
clf.save_model(path)
loaded = xgboost.XGBClassifier()
loaded.load_model(path)
early_stop = xgboost.callback.EarlyStopping(
rounds=early_stopping_rounds, save_best=True
)
clf = xgboost.XGBClassifier(
n_estimators=n_estimators - 128, eval_metric="logloss", callbacks=[early_stop]
)
clf.fit(
X,
y,
eval_set=[(X, y)],
xgb_model=loaded,
)
print("Total boosted rounds:", clf.get_booster().num_boosted_rounds())
assert clf.best_iteration == best
if __name__ == "__main__":
with tempfile.TemporaryDirectory() as tmpdir:
training_continuation_early_stop(tmpdir, False)
training_continuation_early_stop(tmpdir, True)
training_continuation(tmpdir, True)
training_continuation(tmpdir, False)
|
