# Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """ Utility class for VM related operations on Hyper-V Clusters. """ import ctypes import re import sys import threading import time from eventlet import patcher from eventlet import tpool from oslo_log import log as logging from oslo_utils import excutils from six.moves import queue from os_win._i18n import _ from os_win import _utils import os_win.conf from os_win import constants from os_win import exceptions from os_win.utils import baseutils from os_win.utils.compute import _clusapi_utils from os_win.utils.winapi import constants as w_const from os_win.utils.winapi.libs import clusapi as clusapi_def from os_win.utils.winapi import wintypes CONF = os_win.conf.CONF LOG = logging.getLogger(__name__) class ClusterUtils(baseutils.BaseUtils): _MSCLUSTER_NODE = 'MSCluster_Node' _MSCLUSTER_RES = 'MSCluster_Resource' _VM_BASE_NAME = 'Virtual Machine %s' _VM_TYPE = 'Virtual Machine' _MS_CLUSTER_NAMESPACE = '//%s/root/MSCluster' _LIVE_MIGRATION_TYPE = 4 _IGNORE_LOCKED = 1 _DESTROY_GROUP = 1 _FAILBACK_WINDOW_MIN = 0 _FAILBACK_WINDOW_MAX = 23 _WMI_EVENT_TIMEOUT_MS = 100 _WMI_EVENT_CHECK_INTERVAL = 2 def __init__(self, host='.', timeout=CONF.os_win.connect_cluster_timeout): self._instance_name_regex = re.compile('Virtual Machine (.*)') self._clusapi_utils = _clusapi_utils.ClusApiUtils() self._cmgr = _clusapi_utils.ClusterContextManager() if sys.platform == 'win32': self._init_hyperv_conn(host, timeout) def _init_hyperv_conn(self, host, timeout): # The Failover Cluster WMI provider may be unavailable after a reboot. # Let's wait for it. @_utils.wmi_retry_decorator( error_codes=(w_const.ERROR_SHARING_PAUSED, w_const.EPT_S_NOT_REGISTERED), max_sleep_time=5, max_retry_count=None, timeout=timeout) def init(): try: self._conn_cluster = self._get_wmi_conn( self._MS_CLUSTER_NAMESPACE % host) self._cluster = self._conn_cluster.MSCluster_Cluster()[0] # extract this node name from cluster's path path = self._cluster.path_() self._this_node = re.search(r'\\\\(.*)\\root', path, re.IGNORECASE).group(1) except AttributeError: raise exceptions.HyperVClusterException( _("Could not initialize cluster wmi connection.")) init() def _get_failover_watcher(self): raw_query = ("SELECT * FROM __InstanceModificationEvent " "WITHIN %(wmi_check_interv)s WHERE TargetInstance ISA " "'%(cluster_res)s' AND " "TargetInstance.Type='%(cluster_res_type)s' AND " "TargetInstance.OwnerNode != PreviousInstance.OwnerNode" % {'wmi_check_interv': self._WMI_EVENT_CHECK_INTERVAL, 'cluster_res': self._MSCLUSTER_RES, 'cluster_res_type': self._VM_TYPE}) return self._conn_cluster.watch_for(raw_wql=raw_query) def check_cluster_state(self): if len(list(self._get_cluster_nodes())) < 1: raise exceptions.HyperVClusterException( _("Not enough cluster nodes.")) def get_node_name(self): return self._this_node def _get_cluster_nodes(self): return self.cluster_enum(w_const.CLUSTER_ENUM_NODE) def _get_vm_groups(self): for r in self.cluster_enum(w_const.CLUSTER_ENUM_GROUP): group_type = self.get_cluster_group_type(r['name']) if group_type == w_const.ClusGroupTypeVirtualMachine: yield r def _lookup_vm_group_check(self, vm_name): vm = self._lookup_vm_group(vm_name) if not vm: raise exceptions.HyperVVMNotFoundException(vm_name=vm_name) return vm def _lookup_vm_group(self, vm_name): return self._lookup_res(self._conn_cluster.MSCluster_ResourceGroup, vm_name) def _lookup_res(self, resource_source, res_name): res = resource_source(Name=res_name) n = len(res) if n == 0: return None elif n > 1: raise exceptions.HyperVClusterException( _('Duplicate resource name %s found.') % res_name) else: return res[0] def get_cluster_node_names(self): nodes = self._get_cluster_nodes() return [n['name'] for n in nodes] def get_vm_host(self, vm_name): with self._cmgr.open_cluster_group(vm_name) as group_handle: state_info = self._get_cluster_group_state(group_handle) return state_info['owner_node'] def list_instances(self): return [r['name'] for r in self._get_vm_groups()] def list_instance_uuids(self): return [r['id'] for r in self._get_vm_groups()] def add_vm_to_cluster(self, vm_name, max_failover_count=1, failover_period=6, auto_failback=True): """Adds the VM to the Hyper-V Cluster. :param vm_name: The name of the VM to be added to the Hyper-V Cluster :param max_failover_count: The number of times the Hyper-V Cluster will try to failover the VM within the given failover period. If the VM will try to failover more than this number of the given failover_period, the VM will end up in a failed state. :param failover_period: The period (hours) over which the given max_failover_count failovers can occur. After this period expired, the failover count for the given VM is reset. :param auto_failback: boolean, whether the VM will be allowed to move back to its original host when it is available again. """ LOG.debug("Add vm to cluster called for vm %s" % vm_name) self._cluster.AddVirtualMachine(vm_name) vm_group = self._lookup_vm_group_check(vm_name) vm_group.FailoverThreshold = max_failover_count vm_group.FailoverPeriod = failover_period vm_group.PersistentState = True vm_group.AutoFailbackType = int(bool(auto_failback)) # set the earliest and latest time that the group can be moved # back to its preferred node. The unit is in hours. vm_group.FailbackWindowStart = self._FAILBACK_WINDOW_MIN vm_group.FailbackWindowEnd = self._FAILBACK_WINDOW_MAX vm_group.put() def bring_online(self, vm_name): with self._cmgr.open_cluster_group(vm_name) as group_handle: self._clusapi_utils.online_cluster_group(group_handle) def take_offline(self, vm_name): with self._cmgr.open_cluster_group(vm_name) as group_handle: self._clusapi_utils.offline_cluster_group(group_handle) def delete(self, vm_name): # We're sticking with WMI, for now. Destroying VM cluster groups using # clusapi's DestroyClusterGroup function acts strange. VMs get # recreated asyncronuously and put in suspended state, # breaking everything. vm = self._lookup_vm_group_check(vm_name) vm.DestroyGroup(self._DESTROY_GROUP) def cluster_enum(self, object_type): with self._cmgr.open_cluster_enum(object_type) as enum_handle: object_count = self._clusapi_utils.cluster_get_enum_count( enum_handle) for idx in range(object_count): item = self._clusapi_utils.cluster_enum(enum_handle, idx) item_dict = dict(version=item.dwVersion, type=item.dwType, id=item.lpszId, name=item.lpszName) yield item_dict def vm_exists(self, vm_name): res_name = self._VM_BASE_NAME % vm_name try: with self._cmgr.open_cluster_resource(res_name): return True except exceptions.ClusterObjectNotFound: return False def live_migrate_vm(self, vm_name, new_host, timeout=None): self._migrate_vm(vm_name, new_host, self._LIVE_MIGRATION_TYPE, constants.CLUSTER_GROUP_ONLINE, timeout) def _migrate_vm(self, vm_name, new_host, migration_type, exp_state_after_migr, timeout): syntax = w_const.CLUSPROP_SYNTAX_LIST_VALUE_DWORD migr_type = wintypes.DWORD(migration_type) prop_entries = [ self._clusapi_utils.get_property_list_entry( w_const.CLUS_RESTYPE_NAME_VM, syntax, migr_type), self._clusapi_utils.get_property_list_entry( w_const.CLUS_RESTYPE_NAME_VM_CONFIG, syntax, migr_type) ] prop_list = self._clusapi_utils.get_property_list(prop_entries) flags = ( w_const.CLUSAPI_GROUP_MOVE_RETURN_TO_SOURCE_NODE_ON_ERROR | w_const.CLUSAPI_GROUP_MOVE_QUEUE_ENABLED | w_const.CLUSAPI_GROUP_MOVE_HIGH_PRIORITY_START) with self._cmgr.open_cluster() as cluster_handle, \ self._cmgr.open_cluster_group( vm_name, cluster_handle=cluster_handle) as group_handle, \ self._cmgr.open_cluster_node( new_host, cluster_handle=cluster_handle) as dest_node_handle, \ _ClusterGroupStateChangeListener(cluster_handle, vm_name) as listener: self._clusapi_utils.move_cluster_group(group_handle, dest_node_handle, flags, prop_list) try: self._wait_for_cluster_group_migration( listener, vm_name, group_handle, exp_state_after_migr, timeout) except exceptions.ClusterGroupMigrationTimeOut: with excutils.save_and_reraise_exception() as ctxt: self._cancel_cluster_group_migration( listener, vm_name, group_handle, exp_state_after_migr, timeout) # This is rather unlikely to happen but we're # covering it out. try: self._validate_migration(group_handle, vm_name, exp_state_after_migr, new_host) LOG.warning( 'Cluster group migration completed ' 'successfuly after cancel attempt. ' 'Suppressing timeout exception.') ctxt.reraise = False except exceptions.ClusterGroupMigrationFailed: pass else: self._validate_migration(group_handle, vm_name, exp_state_after_migr, new_host) def _validate_migration(self, group_handle, group_name, expected_state, expected_node): state_info = self._clusapi_utils.get_cluster_group_state(group_handle) owner_node = state_info['owner_node'] group_state = state_info['state'] if (expected_state != group_state or expected_node.lower() != owner_node.lower()): raise exceptions.ClusterGroupMigrationFailed( group_name=group_name, expected_state=expected_state, expected_node=expected_node, group_state=group_state, owner_node=owner_node) def cancel_cluster_group_migration(self, group_name, expected_state, timeout=None): with self._cmgr.open_cluster() as cluster_handle, \ self._cmgr.open_cluster_group( group_name, cluster_handle=cluster_handle) as group_handle, \ _ClusterGroupStateChangeListener(cluster_handle, group_name) as listener: self._cancel_cluster_group_migration( listener, group_name, group_handle, expected_state, timeout) def _cancel_cluster_group_migration(self, event_listener, group_name, group_handle, expected_state, timeout=None): LOG.info("Canceling cluster group '%s' migration", group_name) try: cancel_finished = ( self._clusapi_utils.cancel_cluster_group_operation( group_handle)) except exceptions.Win32Exception as ex: group_state_info = self._get_cluster_group_state(group_handle) migration_pending = self._is_migration_pending( group_state_info['state'], group_state_info['status_info'], expected_state) if (ex.error_code == w_const.ERROR_INVALID_STATE and not migration_pending): LOG.debug('Ignoring group migration cancel error. ' 'No migration is pending.') cancel_finished = True else: raise if not cancel_finished: LOG.debug("Waiting for group migration to be canceled.") try: self._wait_for_cluster_group_migration( event_listener, group_name, group_handle, expected_state, timeout=timeout) except Exception: LOG.exception("Failed to cancel cluster group migration.") raise exceptions.JobTerminateFailed() LOG.info("Cluster group migration canceled.") def _is_migration_queued(self, group_status_info): return bool( group_status_info & w_const.CLUSGRP_STATUS_WAITING_IN_QUEUE_FOR_MOVE) def _is_migration_pending(self, group_state, group_status_info, expected_state): migration_pending = ( group_state != expected_state or self._is_migration_queued(group_status_info)) return migration_pending def _wait_for_cluster_group_migration(self, event_listener, group_name, group_handle, expected_state, timeout=None): time_start = time.time() time_left = timeout if timeout else 'undefined' group_state_info = self._get_cluster_group_state(group_handle) group_state = group_state_info['state'] group_status_info = group_state_info['status_info'] migration_pending = self._is_migration_pending( group_state, group_status_info, expected_state) if not migration_pending: return while not timeout or time_left > 0: time_elapsed = time.time() - time_start time_left = timeout - time_elapsed if timeout else 'undefined' LOG.debug("Waiting for cluster group '%(group_name)s' " "migration to finish. " "Time left: %(time_left)s.", dict(group_name=group_name, time_left=time_left)) try: event = event_listener.get(time_left if timeout else None) except queue.Empty: break group_state = event.get('state', group_state) group_status_info = event.get('status_info', group_status_info) migration_pending = self._is_migration_pending(group_state, group_status_info, expected_state) if not migration_pending: return LOG.error("Cluster group migration timed out.") raise exceptions.ClusterGroupMigrationTimeOut( group_name=group_name, time_elapsed=time.time() - time_start) def get_cluster_node_name(self, node_id): for node in self._get_cluster_nodes(): if node['id'] == node_id: return node['name'] err_msg = _("Could not find any cluster node with id: %s.") raise exceptions.NotFound(err_msg % node_id) def get_cluster_group_type(self, group_name): with self._cmgr.open_cluster_group(group_name) as group_handle: buff, buff_sz = self._clusapi_utils.cluster_group_control( group_handle, w_const.CLUSCTL_GROUP_GET_RO_COMMON_PROPERTIES) return self._clusapi_utils.get_cluster_group_type( ctypes.byref(buff), buff_sz) def get_cluster_group_state_info(self, group_name): """Gets cluster group state info. :return: a dict containing the following keys: ['state', 'migration_queued', 'owner_node'] """ with self._cmgr.open_cluster_group(group_name) as group_handle: state_info = self._get_cluster_group_state(group_handle) migration_queued = self._is_migration_queued( state_info['status_info']) return dict(owner_node=state_info['owner_node'], state=state_info['state'], migration_queued=migration_queued) def _get_cluster_group_state(self, group_handle): state_info = self._clusapi_utils.get_cluster_group_state(group_handle) buff, buff_sz = self._clusapi_utils.cluster_group_control( group_handle, w_const.CLUSCTL_GROUP_GET_RO_COMMON_PROPERTIES) status_info = self._clusapi_utils.get_cluster_group_status_info( ctypes.byref(buff), buff_sz) state_info['status_info'] = status_info return state_info def _monitor_vm_failover(self, watcher, callback, event_timeout_ms=_WMI_EVENT_TIMEOUT_MS): """Creates a monitor to check for new WMI MSCluster_Resource events. This method will poll the last _WMI_EVENT_CHECK_INTERVAL + 1 seconds for new events and listens for _WMI_EVENT_TIMEOUT_MS milliseconds, since listening is a thread blocking action. Any event object caught will then be processed. """ vm_name = None new_host = None try: # wait for new event for _WMI_EVENT_TIMEOUT_MS milliseconds. if patcher.is_monkey_patched('thread'): wmi_object = tpool.execute(watcher, event_timeout_ms) else: wmi_object = watcher(event_timeout_ms) old_host = wmi_object.previous.OwnerNode new_host = wmi_object.OwnerNode # wmi_object.Name field is of the form: # 'Virtual Machine nova-instance-template' # wmi_object.Name filed is a key and as such is not affected # by locale, so it will always be 'Virtual Machine' match = self._instance_name_regex.search(wmi_object.Name) if match: vm_name = match.group(1) if vm_name: try: callback(vm_name, old_host, new_host) except Exception: LOG.exception( "Exception during failover callback.") except exceptions.x_wmi_timed_out: pass def get_vm_owner_change_listener(self): def listener(callback): watcher = self._get_failover_watcher() while True: # We avoid setting an infinite timeout in order to let # the process gracefully stop. Note that the os-win WMI # event listeners are meant to be used as long running # daemons, so no stop API is provided ATM. try: self._monitor_vm_failover( watcher, callback, constants.DEFAULT_WMI_EVENT_TIMEOUT_MS) except Exception: LOG.exception("The VM cluster group owner change " "event listener encountered an " "unexpected exception.") time.sleep(constants.DEFAULT_WMI_EVENT_TIMEOUT_MS / 1000) return listener def get_vm_owner_change_listener_v2(self): def listener(callback): cluster_handle = self._clusapi_utils.open_cluster() _listener = _ClusterGroupOwnerChangeListener(cluster_handle) while True: try: event = _listener.get() group_name = event['cluster_object_name'] group_type = self.get_cluster_group_type(group_name) if group_type != w_const.ClusGroupTypeVirtualMachine: continue new_node_id = event['parent_id'] new_node_name = self.get_cluster_node_name(new_node_id) callback(group_name, new_node_name) except Exception: LOG.exception("The VM cluster group owner change " "event listener encountered an " "unexpected exception.") time.sleep(constants.DEFAULT_WMI_EVENT_TIMEOUT_MS / 1000) return listener # At the moment, those event listeners are not meant to be used outside # os-win, mostly because of the underlying API limitations. class _ClusterEventListener(object): _notif_keys = {} _notif_port_h = None _cluster_handle = None _running = False _stop_on_error = True _error_sleep_interval = 2 def __init__(self, cluster_handle, stop_on_error=True): self._cluster_handle = cluster_handle self._stop_on_error = stop_on_error self._clusapi_utils = _clusapi_utils.ClusApiUtils() self._event_queue = queue.Queue() self._setup() def __enter__(self): self._ensure_listener_running() return self def _get_notif_key_dw(self, notif_key): notif_key_dw = self._notif_keys.get(notif_key) if notif_key_dw is None: notif_key_dw = wintypes.DWORD(notif_key) # We have to make sure those addresses are preserved. self._notif_keys[notif_key] = notif_key_dw return notif_key_dw def _add_filter(self, notif_filter, notif_key=0): notif_key_dw = self._get_notif_key_dw(notif_key) # We'll get a notification handle if not already existing. self._notif_port_h = self._clusapi_utils.create_cluster_notify_port_v2( self._cluster_handle, notif_filter, self._notif_port_h, notif_key_dw) def _setup_notif_port(self): for notif_filter in self._notif_filters_list: filter_struct = clusapi_def.NOTIFY_FILTER_AND_TYPE( dwObjectType=notif_filter['object_type'], FilterFlags=notif_filter['filter_flags']) notif_key = notif_filter.get('notif_key', 0) self._add_filter(filter_struct, notif_key) def _setup(self): self._setup_notif_port() # If eventlet monkey patching is used, this will actually be a # greenthread. We just don't want to enforce eventlet usage. worker = threading.Thread(target=self._listen) worker.daemon = True self._running = True worker.start() def __exit__(self, exc_type, exc_value, traceback): self.stop() def _signal_stopped(self): self._running = False self._event_queue.put(None) def stop(self): self._signal_stopped() if self._notif_port_h: self._clusapi_utils.close_cluster_notify_port(self._notif_port_h) def _listen(self): while self._running: try: # We're using an indefinite timeout here. When the listener is # closed, this will raise an 'invalid handle value' error, # which we're going to ignore. event = _utils.avoid_blocking_call( self._clusapi_utils.get_cluster_notify_v2, self._notif_port_h, timeout_ms=-1) processed_event = self._process_event(event) if processed_event: self._event_queue.put(processed_event) except Exception: if self._running: LOG.exception( "Unexpected exception in event listener loop.") if self._stop_on_error: LOG.warning( "The cluster event listener will now close.") self._signal_stopped() else: time.sleep(self._error_sleep_interval) def _process_event(self, event): return event def get(self, timeout=None): self._ensure_listener_running() event = self._event_queue.get(timeout=timeout) self._ensure_listener_running() return event def _ensure_listener_running(self): if not self._running: raise exceptions.OSWinException( _("Cluster event listener is not running.")) class _ClusterGroupStateChangeListener(_ClusterEventListener): _NOTIF_KEY_GROUP_STATE = 0 _NOTIF_KEY_GROUP_COMMON_PROP = 1 _notif_filters_list = [ dict(object_type=w_const.CLUSTER_OBJECT_TYPE_GROUP, filter_flags=w_const.CLUSTER_CHANGE_GROUP_STATE_V2, notif_key=_NOTIF_KEY_GROUP_STATE), dict(object_type=w_const.CLUSTER_OBJECT_TYPE_GROUP, filter_flags=w_const.CLUSTER_CHANGE_GROUP_COMMON_PROPERTY_V2, notif_key=_NOTIF_KEY_GROUP_COMMON_PROP)] def __init__(self, cluster_handle, group_name=None, **kwargs): self._group_name = group_name super(_ClusterGroupStateChangeListener, self).__init__( cluster_handle, **kwargs) def _process_event(self, event): group_name = event['cluster_object_name'] if self._group_name and self._group_name.lower() != group_name.lower(): return preserved_keys = ['cluster_object_name', 'object_type', 'filter_flags', 'notif_key'] processed_event = {key: event[key] for key in preserved_keys} notif_key = event['notif_key'] if notif_key == self._NOTIF_KEY_GROUP_STATE: if event['buff_sz'] != ctypes.sizeof(wintypes.DWORD): raise exceptions.ClusterPropertyRetrieveFailed() state_p = ctypes.cast(event['buff'], wintypes.PDWORD) state = state_p.contents.value processed_event['state'] = state return processed_event elif notif_key == self._NOTIF_KEY_GROUP_COMMON_PROP: try: status_info = ( self._clusapi_utils.get_cluster_group_status_info( ctypes.byref(event['buff']), event['buff_sz'])) processed_event['status_info'] = status_info return processed_event except exceptions.ClusterPropertyListEntryNotFound: # At the moment, we only care about the 'StatusInformation' # common property. pass class _ClusterGroupOwnerChangeListener(_ClusterEventListener): _notif_filters_list = [ dict(object_type=w_const.CLUSTER_OBJECT_TYPE_GROUP, filter_flags=w_const.CLUSTER_CHANGE_GROUP_OWNER_NODE_V2) ]