1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
"""A basic kernel monitor with autorestarting.
This watches a kernel's state using KernelManager.is_alive and auto
restarts the kernel if it dies.
It is an incomplete base class, and must be subclassed.
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2013 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from IPython.config.configurable import LoggingConfigurable
from IPython.utils.traitlets import (
Instance, Float, Dict, Bool, Integer,
)
#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------
class KernelRestarter(LoggingConfigurable):
"""Monitor and autorestart a kernel."""
kernel_manager = Instance('IPython.kernel.KernelManager')
time_to_dead = Float(3.0, config=True,
help="""Kernel heartbeat interval in seconds."""
)
restart_limit = Integer(5, config=True,
help="""The number of consecutive autorestarts before the kernel is presumed dead."""
)
_restarting = Bool(False)
_restart_count = Integer(0)
callbacks = Dict()
def _callbacks_default(self):
return dict(restart=[], dead=[])
def start(self):
"""Start the polling of the kernel."""
raise NotImplementedError("Must be implemented in a subclass")
def stop(self):
"""Stop the kernel polling."""
raise NotImplementedError("Must be implemented in a subclass")
def add_callback(self, f, event='restart'):
"""register a callback to fire on a particular event
Possible values for event:
'restart' (default): kernel has died, and will be restarted.
'dead': restart has failed, kernel will be left dead.
"""
self.callbacks[event].append(f)
def remove_callback(self, f, event='restart'):
"""unregister a callback to fire on a particular event
Possible values for event:
'restart' (default): kernel has died, and will be restarted.
'dead': restart has failed, kernel will be left dead.
"""
try:
self.callbacks[event].remove(f)
except ValueError:
pass
def _fire_callbacks(self, event):
"""fire our callbacks for a particular event"""
for callback in self.callbacks[event]:
try:
callback()
except Exception as e:
self.log.error("KernelRestarter: %s callback %r failed", event, callback, exc_info=True)
def poll(self):
self.log.debug('Polling kernel...')
if not self.kernel_manager.is_alive():
if self._restarting:
self._restart_count += 1
else:
self._restart_count = 1
if self._restart_count >= self.restart_limit:
self.log.warn("KernelRestarter: restart failed")
self._fire_callbacks('dead')
self._restarting = False
self._restart_count = 0
self.stop()
else:
self.log.info('KernelRestarter: restarting kernel (%i/%i)',
self._restart_count,
self.restart_limit
)
self._fire_callbacks('restart')
self.kernel_manager.restart_kernel(now=True)
self._restarting = True
else:
if self._restarting:
self.log.debug("KernelRestarter: restart apparently succeeded")
self._restarting = False
|
