1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
|
from __future__ import annotations
from processview.utils.singleton import singleton
from processview.core.dataset import Dataset
from processview.core.dataset import DatasetIdentifier
from collections import OrderedDict
from silx.utils.enum import Enum as _Enum
import threading
from typing import Optional, Type, Union
from datetime import datetime
import weakref
import logging
_logger = logging.getLogger(__name__)
class DatasetState(_Enum):
"""possible dataset status relatif to a Process"""
ON_GOING = "on going"
SUCCEED = "succeed"
FAILED = "failed"
PENDING = "pending"
SKIPPED = "skipped"
WAIT_USER_VALIDATION = "waiting user validation"
CANCELLED = "cancelled"
@singleton
class ProcessManager:
"""
Manager to register and observe `SuperviseProcess`.
"""
def __init__(self):
self._processes = {}
self._dataset_process_states: dict[int, dict[str, tuple]] = {}
"""key is processID, Value is a tuple of
(dataset state, details)"""
self._processID = 0
self._updateCallback = set()
"""list of callback to trigger when an update is generated"""
self._newProcessCallback = set()
"""list of callback to trigger when a process is added"""
self._processRemovedCallback = set()
"""list of callback to trigger when a process is removed"""
self._process_renamed_callback = set()
"""list of callback to trigger when a process is renamed"""
self.lock = threading.Lock()
def met(self, process, dataset) -> bool:
"""check if a dataset has already met a dataset"""
if isinstance(dataset, DatasetIdentifier):
dataset_id = dataset
elif dataset is None:
return False
else:
dataset_id = dataset.get_identifier()
if process.process_id in self._dataset_process_states:
return dataset_id in self._dataset_process_states[process.process_id]
def process_renamed(self, process):
"""
the process name only matter for the GUI side
"""
for callback in self._process_renamed_callback:
callback(process)
def register(self, process) -> int:
"""
Register a process to the manager
:param SuperviseProcess process:
"""
from processview.core.superviseprocess import (
SuperviseProcess,
) # avoid cyclic import
if not isinstance(process, SuperviseProcess):
raise TypeError(
f"{process} is expected to be an isntance of {SuperviseProcess}"
)
process_id = self._processID
self._processID += 1
self._processes[process_id] = weakref.ref(process)
process._set_process_id(process_id)
with self.lock:
for callback in self._newProcessCallback:
callback()
return process_id
def _find_dataset_id(self, data) -> Dataset:
if isinstance(data, Dataset):
return data.get_identifier()
elif isinstance(data, DatasetIdentifier):
return data
else:
raise TypeError(
"dataset should be an instance of Dataset or "
"DatasetIdentifier. Get {} instead".format(type(data))
)
def _find_process(self, process):
from processview.core.superviseprocess import (
SuperviseProcess,
) # avoid cyclic import
if isinstance(process, SuperviseProcess):
return process
elif isinstance(process, str):
for p in self.get_processes():
if p.name == process:
return p
else:
raise TypeError(
"process should be an instance of SuperviseProcess or"
" str. Get {} instead".format(type(process))
)
def unregister(self, process):
"""
Unregister a process to the manager
:param BaseProcess process:
"""
with self.lock:
if process.process_id in self._processes and process.is_master_process:
del self._processes[process.process_id]
for callback in self._processRemovedCallback:
callback()
def get_processes(self) -> tuple:
"""
:return: tuple of processes currently registered
"""
processes = []
for _, p in self._processes.items():
if p() is not None:
processes.append(p())
return tuple(processes)
def get_process(self, process_id: int):
if process_id in self._processes:
return self._processes[process_id]()
def get_datasets(self) -> tuple:
"""
:return: tuple of datasets
"""
res = set()
for _, dataset_states in self._dataset_process_states.items():
[res.add(dataset) for dataset in dataset_states]
return tuple(res)
def remove_dataset(self, dataset: Dataset):
"""
Remove a dataset from the list of registered dataset
"""
dataset_id = self._find_dataset_id(dataset)
for process_id in self._dataset_process_states:
self._dataset_process_states[process_id].pop(dataset_id, None)
def notify_dataset_state(
self,
dataset: Dataset | DatasetIdentifier,
process,
state: DatasetState,
details=None,
) -> None:
"""
Update dataset state
:param Dataset dataset: dataset which state is updated
:param BaseProcess process: Process concern by the new state
:param DatasetState state: current State
:param str info: details about the error or success
:return:
"""
from processview.core.superviseprocess import (
SuperviseProcess,
) # avoid cyclic import
if not isinstance(process, SuperviseProcess):
_logger.warning(
f"Unable to update dataset {dataset} state. Process does not inherit from SuperviseProcess"
)
if process.process_id not in self._processes:
self.register(process)
if process.process_id not in self._dataset_process_states:
self._dataset_process_states[process.process_id] = OrderedDict()
if details is None:
details = ""
if isinstance(dataset, Dataset):
dataset_id = dataset.get_identifier()
else:
dataset_id = dataset
if not isinstance(dataset_id, DatasetIdentifier):
raise TypeError(
f"dataset should be an instance of {Dataset} or {DatasetIdentifier}. Got {type(dataset)}"
)
# as we store the identifier and not a string anymore we might need to remove another instanciation
process_states = self._dataset_process_states[process.process_id]
to_remove = set()
for id in process_states:
if id == dataset_id:
to_remove.add(id)
for id in to_remove:
del process_states[id]
self._dataset_process_states[process.process_id] = process_states
self._dataset_process_states[process.process_id][dataset_id] = (
state,
datetime.now(),
details,
)
self.updated()
def get_dataset_state(self, dataset_id, process) -> Union[None, DatasetState]:
"""
:param Dataset dataset_id:
:param BaseProcess process:
:return: DatasetState relative to provided process if know
:rtype: Union[None, DatasetState]
"""
dataset_id = self._find_dataset_id(dataset_id)
assert isinstance(dataset_id, DatasetIdentifier)
process = self._find_process(process)
if process is None:
_logger.warning("process {} is no more supervised".format(process))
return
if dataset_id is None:
_logger.warning("dataset {} is no more supervised".format(dataset_id))
elif process.process_id in self._dataset_process_states:
if dataset_id in self._dataset_process_states[process.process_id]:
return self._dataset_process_states[process.process_id][dataset_id][0]
return None
def get_dataset_details(self, dataset_id, process) -> Union[None, str]:
"""
:param Dataset dataset_id:
:param BaseProcess process:
:return: DatasetState relative to provided process if know
"""
dataset_id = self._find_dataset_id(dataset_id)
process = self._find_process(process)
if process is None:
_logger.warning("process {} is no more supervised".format(process))
return
if dataset_id is None:
_logger.warning("dataset {} is no more supervised".format(dataset_id))
if process.process_id in self._dataset_process_states:
if dataset_id in self._dataset_process_states[process.process_id]:
return self._dataset_process_states[process.process_id][dataset_id][2]
return None
def get_dataset_time_stamp(self, dataset_id, process) -> Union[None, str]:
"""
:param Dataset dataset_id:
:param BaseProcess process:
:return: DatasetState relative to provided process if know
"""
dataset_id = self._find_dataset_id(dataset_id)
process = self._find_process(process)
if process is None:
_logger.warning("process {} is no more supervised".format(process))
return
if dataset_id is None:
_logger.warning("dataset {} is no more supervised".format(dataset_id))
if process.process_id in self._dataset_process_states:
if dataset_id in self._dataset_process_states[process.process_id]:
return self._dataset_process_states[process.process_id][dataset_id][1]
return None
def get_dataset_stream(
self, dataset, time_stamp=False
) -> tuple[int, DatasetState, list]:
"""
:param Dataset dataset: dataset the stream is focus on
:param bool time_stamp: if True then return timestamp in the list of
elements
:return: stream of (process ID, DatasetState) for a given dataset
"""
stream = []
for process_id, dataset_states in self._dataset_process_states.items():
dst_id = dataset.get_identifier()
if dst_id in dataset_states:
state, _timestamp, _ = dataset_states[dst_id]
stream.append((process_id, state, _timestamp))
# order the stream
stream = sorted(stream, key=lambda elmt: elmt[2])
if not time_stamp:
stream = [s[:-1] for s in stream]
return tuple(stream)
def get_process_history(self, process, time_stamp=False) -> tuple:
"""
Return the know history of the process.
:param BaseProcess process:
:param bool time_stamp: if True then return timestamp in the list of
elements
:return: tuple of (DatasetIdentifier, state, [timestamp])
"""
history = []
if process.process_id in self._dataset_process_states:
dataset_states = self._dataset_process_states[process.process_id]
for dataset_id, info in dataset_states.items():
state, _timestamp, details = info
history.append((dataset_id, state, _timestamp))
history = sorted(history, key=lambda elmt: elmt[2])
if not time_stamp:
history = [s[:-1] for s in history]
return tuple(history)
def updated(self):
"""Function 'open' for monckey patch"""
with self.lock:
for callback in self._updateCallback:
callback()
def add_update_callback(self, callback) -> None:
"""
:param callback: add a callback to be trigger when dataset state change
"""
with self.lock:
self._updateCallback.add(callback)
def remove_update_callback(self, callback) -> None:
"""
:param callback: remove a callback from the stack of callback to be
call when dataset state change
"""
with self.lock:
if callback in self._updateCallback:
self._updateCallback.remove(callback)
def add_new_process_callback(self, callback):
self._newProcessCallback.add(callback)
def remove_new_process_callback(self, callback):
with self.lock:
if callback in self._newProcessCallback:
self._newProcessCallback.remove(callback)
def add_process_removed_callback(self, callback):
self._processRemovedCallback.add(callback)
def remove_process_removed_callback(self, callback):
with self.lock:
if callback in self._processRemovedCallback:
self._processRemovedCallback.remove(callback)
def add_process_name_changed_callback(self, callback):
self._process_renamed_callback.add(callback)
def clear(self):
"""
clear registered processes and dataset states
:return:
"""
self._processes = {}
self._dataset_process_states = {}
|