# Copyright 2013 Hewlett-Packard Development Company, L.P.
# Copyright 2013 International Business Machines Corporation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Conduct all activity related to bare-metal deployments.
A single instance of :py:class:`ironic.conductor.manager.ConductorManager` is
created within the *ironic-conductor* process, and is responsible for
performing all actions on bare metal resources (Chassis, Nodes, and Ports).
Commands are received via RPCs. The conductor service also performs periodic
tasks, eg. to monitor the status of active deployments.
Drivers are loaded via entrypoints by the
:py:class:`ironic.common.driver_factory` class. Each driver is instantiated
only once, when the ConductorManager service starts. In this way, a single
ConductorManager may use multiple drivers, and manage heterogeneous hardware.
When multiple :py:class:`ConductorManager` are run on different hosts, they are
all active and cooperatively manage all nodes in the deployment. Nodes are
locked by each conductor when performing actions which change the state of that
node; these locks are represented by the
:py:class:`ironic.conductor.task_manager.TaskManager` class.
A `tooz.hashring.HashRing
<https://git.openstack.org/cgit/openstack/tooz/tree/tooz/hashring.py>`_
is used to distribute nodes across the set of active conductors which support
each node's driver. Rebalancing this ring can trigger various actions by each
conductor, such as building or tearing down the TFTP environment for a node,
notifying Neutron of a change, etc.
"""
import collections
import datetime
import tempfile
import eventlet
from futurist import periodics
from futurist import waiters
from ironic_lib import metrics_utils
from oslo_db import exception as db_exception
from oslo_log import log
import oslo_messaging as messaging
from oslo_utils import excutils
from oslo_utils import uuidutils
from oslo_utils import versionutils
from six.moves import queue
from ironic.common import driver_factory
from ironic.common import exception
from ironic.common import faults
from ironic.common.glance_service import service_utils as glance_utils
from ironic.common.i18n import _
from ironic.common import images
from ironic.common import network
from ironic.common import release_mappings as versions
from ironic.common import states
from ironic.common import swift
from ironic.conductor import base_manager
from ironic.conductor import notification_utils as notify_utils
from ironic.conductor import task_manager
from ironic.conductor import utils
from ironic.conf import CONF
from ironic.drivers import base as drivers_base
from ironic import objects
from ironic.objects import base as objects_base
from ironic.objects import fields
MANAGER_TOPIC = 'ironic.conductor_manager'
LOG = log.getLogger(__name__)
METRICS = metrics_utils.get_metrics_logger(__name__)
SYNC_EXCLUDED_STATES = (states.DEPLOYWAIT, states.CLEANWAIT, states.ENROLL)
# NOTE(sambetts) This list is used to keep track of deprecation warnings that
# have already been issued for deploy drivers that do not accept the
# agent_version parameter and need updating.
_SEEN_AGENT_VERSION_DEPRECATIONS = []
# NOTE(rloo) This is used to keep track of deprecation warnings that have
# already been issued for deploy drivers that do not use deploy steps.
_SEEN_NO_DEPLOY_STEP_DEPRECATIONS = set()
class ConductorManager(base_manager.BaseConductorManager):
"""Ironic Conductor manager main class."""
# NOTE(rloo): This must be in sync with rpcapi.ConductorAPI's.
# NOTE(pas-ha): This also must be in sync with
# ironic.common.release_mappings.RELEASE_MAPPING['master']
RPC_API_VERSION = '1.47'
target = messaging.Target(version=RPC_API_VERSION)
def __init__(self, host, topic):
super(ConductorManager, self).__init__(host, topic)
self.power_state_sync_count = collections.defaultdict(int)
@METRICS.timer('ConductorManager.create_node')
# No need to add these since they are subclasses of InvalidParameterValue:
# InterfaceNotFoundInEntrypoint
# IncompatibleInterface,
# NoValidDefaultForInterface
@messaging.expected_exceptions(exception.InvalidParameterValue,
exception.DriverNotFound)
def create_node(self, context, node_obj):
"""Create a node in database.
:param context: an admin context
:param node_obj: a created (but not saved to the database) node object.
:returns: created node object.
:raises: InterfaceNotFoundInEntrypoint if validation fails for any
dynamic interfaces (e.g. network_interface).
:raises: IncompatibleInterface if one or more of the requested
interfaces are not compatible with the hardware type.
:raises: NoValidDefaultForInterface if no default can be calculated
for some interfaces, and explicit values must be provided.
:raises: InvalidParameterValue if some fields fail validation.
:raises: DriverNotFound if the driver or hardware type is not found.
"""
LOG.debug("RPC create_node called for node %s.", node_obj.uuid)
driver_factory.check_and_update_node_interfaces(node_obj)
node_obj.create()
return node_obj
@METRICS.timer('ConductorManager.update_node')
# No need to add these since they are subclasses of InvalidParameterValue:
# InterfaceNotFoundInEntrypoint
# IncompatibleInterface,
# NoValidDefaultForInterface
@messaging.expected_exceptions(exception.InvalidParameterValue,
exception.NodeLocked,
exception.InvalidState,
exception.DriverNotFound)
def update_node(self, context, node_obj, reset_interfaces=False):
"""Update a node with the supplied data.
This method is the main "hub" for PUT and PATCH requests in the API.
It ensures that the requested change is safe to perform,
validates the parameters with the node's driver, if necessary.
:param context: an admin context
:param node_obj: a changed (but not saved) node object.
:param reset_interfaces: whether to reset hardware interfaces to their
defaults.
:raises: NoValidDefaultForInterface if no default can be calculated
for some interfaces, and explicit values must be provided.
"""
node_id = node_obj.uuid
LOG.debug("RPC update_node called for node %s.", node_id)
# NOTE(jroll) clear maintenance_reason if node.update sets
# maintenance to False for backwards compatibility, for tools
# not using the maintenance endpoint.
# NOTE(kaifeng) also clear fault when out of maintenance.
delta = node_obj.obj_what_changed()
if 'maintenance' in delta and not node_obj.maintenance:
node_obj.maintenance_reason = None
node_obj.fault = None
# TODO(dtantsur): reconsider allowing changing some (but not all)
# interfaces for active nodes in the future.
# NOTE(kaifeng): INSPECTING is allowed to keep backwards
# compatibility, starting from API 1.39 node update is disallowed
# in this state.
allowed_update_states = [states.ENROLL, states.INSPECTING,
states.INSPECTWAIT, states.MANAGEABLE,
states.AVAILABLE]
action = _("Node %(node)s can not have %(field)s "
"updated unless it is in one of allowed "
"(%(allowed)s) states or in maintenance mode.")
updating_driver = 'driver' in delta
for iface in drivers_base.ALL_INTERFACES:
interface_field = '%s_interface' % iface
if interface_field not in delta:
if updating_driver and reset_interfaces:
setattr(node_obj, interface_field, None)
continue
if not (node_obj.provision_state in allowed_update_states
or node_obj.maintenance):
raise exception.InvalidState(
action % {'node': node_obj.uuid,
'allowed': ', '.join(allowed_update_states),
'field': interface_field})
driver_factory.check_and_update_node_interfaces(node_obj)
with task_manager.acquire(context, node_id, shared=False,
purpose='node update') as task:
# Prevent instance_uuid overwriting
if ('instance_uuid' in delta and node_obj.instance_uuid
and task.node.instance_uuid):
raise exception.NodeAssociated(
node=node_id, instance=task.node.instance_uuid)
# NOTE(dtantsur): if the resource class is changed for an active
# instance, nova will not update its internal record. That will
# result in the new resource class exposed on the node as available
# for consumption, and nova may try to schedule on this node again.
if ('resource_class' in delta and task.node.resource_class
and task.node.provision_state
not in allowed_update_states):
raise exception.InvalidState(
action % {'node': node_obj.uuid,
'allowed': ', '.join(allowed_update_states),
'field': 'resource_class'})
node_obj.save()
return node_obj
@METRICS.timer('ConductorManager.change_node_power_state')
@messaging.expected_exceptions(exception.InvalidParameterValue,
exception.NoFreeConductorWorker,
exception.NodeLocked)
def change_node_power_state(self, context, node_id, new_state,
timeout=None):
"""RPC method to encapsulate changes to a node's state.
Perform actions such as power on, power off. The validation is
performed synchronously, and if successful, the power action is
updated in the background (asynchronously). Once the power action
is finished and successful, it updates the power_state for the
node with the new power state.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:param new_state: the desired power state of the node.
:param timeout: timeout (in seconds) positive integer (> 0) for any
power state. ``None`` indicates to use default timeout.
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
:raises: InvalidParameterValue
:raises: MissingParameterValue
"""
LOG.debug("RPC change_node_power_state called for node %(node)s. "
"The desired new state is %(state)s.",
{'node': node_id, 'state': new_state})
with task_manager.acquire(context, node_id, shared=False,
purpose='changing node power state') as task:
task.driver.power.validate(task)
if (new_state not in
task.driver.power.get_supported_power_states(task)):
# FIXME(naohirot):
# After driver composition, we should print power interface
# name here instead of driver.
raise exception.InvalidParameterValue(
_('The driver %(driver)s does not support the power state,'
' %(state)s') %
{'driver': task.node.driver, 'state': new_state})
if new_state in (states.SOFT_REBOOT, states.SOFT_POWER_OFF):
power_timeout = (timeout
or CONF.conductor.soft_power_off_timeout)
else:
power_timeout = timeout
# Set the target_power_state and clear any last_error, since we're
# starting a new operation. This will expose to other processes
# and clients that work is in progress.
if new_state in (states.POWER_ON, states.REBOOT,
states.SOFT_REBOOT):
task.node.target_power_state = states.POWER_ON
else:
task.node.target_power_state = states.POWER_OFF
task.node.last_error = None
task.node.save()
task.set_spawn_error_hook(utils.power_state_error_handler,
task.node, task.node.power_state)
task.spawn_after(self._spawn_worker, utils.node_power_action,
task, new_state, timeout=power_timeout)
@METRICS.timer('ConductorManager.vendor_passthru')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeLocked,
exception.InvalidParameterValue,
exception.UnsupportedDriverExtension)
def vendor_passthru(self, context, node_id, driver_method,
http_method, info):
"""RPC method to encapsulate vendor action.
Synchronously validate driver specific info, and if successful invoke
the vendor method. If the method mode is 'async' the conductor will
start background worker to perform vendor action.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:param driver_method: the name of the vendor method.
:param http_method: the HTTP method used for the request.
:param info: vendor method args.
:raises: InvalidParameterValue if supplied info is not valid.
:raises: MissingParameterValue if missing supplied info
:raises: UnsupportedDriverExtension if current driver does not have
vendor interface or method is unsupported.
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
:raises: NodeLocked if the vendor passthru method requires an exclusive
lock but the node is locked by another conductor
:returns: A dictionary containing:
:return: The response of the invoked vendor method
:async: Boolean value. Whether the method was invoked
asynchronously (True) or synchronously (False). When invoked
asynchronously the response will be always None.
:attach: Boolean value. Whether to attach the response of
the invoked vendor method to the HTTP response object (True)
or return it in the response body (False).
"""
LOG.debug("RPC vendor_passthru called for node %s.", node_id)
# NOTE(mariojv): Not all vendor passthru methods require an exclusive
# lock on a node, so we acquire a shared lock initially. If a method
# requires an exclusive lock, we'll acquire one after checking
# vendor_opts before starting validation.
with task_manager.acquire(context, node_id, shared=True,
purpose='calling vendor passthru') as task:
vendor_iface = task.driver.vendor
try:
vendor_opts = vendor_iface.vendor_routes[driver_method]
vendor_func = vendor_opts['func']
except KeyError:
raise exception.InvalidParameterValue(
_('No handler for method %s') % driver_method)
http_method = http_method.upper()
if http_method not in vendor_opts['http_methods']:
raise exception.InvalidParameterValue(
_('The method %(method)s does not support HTTP %(http)s') %
{'method': driver_method, 'http': http_method})
# Change shared lock to exclusive if a vendor method requires
# it. Vendor methods default to requiring an exclusive lock.
if vendor_opts['require_exclusive_lock']:
task.upgrade_lock()
vendor_iface.validate(task, method=driver_method,
http_method=http_method, **info)
# Inform the vendor method which HTTP method it was invoked with
info['http_method'] = http_method
# Invoke the vendor method accordingly with the mode
is_async = vendor_opts['async']
ret = None
if is_async:
task.spawn_after(self._spawn_worker, vendor_func, task, **info)
else:
ret = vendor_func(task, **info)
return {'return': ret,
'async': is_async,
'attach': vendor_opts['attach']}
@METRICS.timer('ConductorManager.driver_vendor_passthru')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.InvalidParameterValue,
exception.UnsupportedDriverExtension,
exception.DriverNotFound,
exception.NoValidDefaultForInterface,
exception.InterfaceNotFoundInEntrypoint)
def driver_vendor_passthru(self, context, driver_name, driver_method,
http_method, info):
"""Handle top-level vendor actions.
RPC method which handles driver-level vendor passthru calls. These
calls don't require a node UUID and are executed on a random
conductor with the specified driver. If the method mode is
async the conductor will start background worker to perform
vendor action.
For dynamic drivers, the calculated default vendor interface is used.
:param context: an admin context.
:param driver_name: name of the driver or hardware type on which to
call the method.
:param driver_method: name of the vendor method, for use by the driver.
:param http_method: the HTTP method used for the request.
:param info: user-supplied data to pass through to the driver.
:raises: MissingParameterValue if missing supplied info
:raises: InvalidParameterValue if supplied info is not valid.
:raises: UnsupportedDriverExtension if current driver does not have
vendor interface, if the vendor interface does not implement
driver-level vendor passthru or if the passthru method is
unsupported.
:raises: DriverNotFound if the supplied driver is not loaded.
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
:raises: NoValidDefaultForInterface if no default interface
implementation can be found for this driver's vendor
interface.
:raises: InterfaceNotFoundInEntrypoint if the default interface for a
hardware type is invalid.
:returns: A dictionary containing:
:return: The response of the invoked vendor method
:async: Boolean value. Whether the method was invoked
asynchronously (True) or synchronously (False). When invoked
asynchronously the response will be always None.
:attach: Boolean value. Whether to attach the response of
the invoked vendor method to the HTTP response object (True)
or return it in the response body (False).
"""
# Any locking in a top-level vendor action will need to be done by the
# implementation, as there is little we could reasonably lock on here.
LOG.debug("RPC driver_vendor_passthru for driver %s.", driver_name)
driver = driver_factory.get_hardware_type(driver_name)
vendor = None
vendor_name = driver_factory.default_interface(
driver, 'vendor', driver_name=driver_name)
vendor = driver_factory.get_interface(driver, 'vendor',
vendor_name)
try:
vendor_opts = vendor.driver_routes[driver_method]
vendor_func = vendor_opts['func']
except KeyError:
raise exception.InvalidParameterValue(
_('No handler for method %s') % driver_method)
http_method = http_method.upper()
if http_method not in vendor_opts['http_methods']:
raise exception.InvalidParameterValue(
_('The method %(method)s does not support HTTP %(http)s') %
{'method': driver_method, 'http': http_method})
# Inform the vendor method which HTTP method it was invoked with
info['http_method'] = http_method
# Invoke the vendor method accordingly with the mode
is_async = vendor_opts['async']
ret = None
vendor.driver_validate(method=driver_method, **info)
if is_async:
self._spawn_worker(vendor_func, context, **info)
else:
ret = vendor_func(context, **info)
return {'return': ret,
'async': is_async,
'attach': vendor_opts['attach']}
@METRICS.timer('ConductorManager.get_node_vendor_passthru_methods')
@messaging.expected_exceptions(exception.UnsupportedDriverExtension)
def get_node_vendor_passthru_methods(self, context, node_id):
"""Retrieve information about vendor methods of the given node.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:returns: dictionary of <method name>:<method metadata> entries.
"""
LOG.debug("RPC get_node_vendor_passthru_methods called for node %s",
node_id)
lock_purpose = 'listing vendor passthru methods'
with task_manager.acquire(context, node_id, shared=True,
purpose=lock_purpose) as task:
return get_vendor_passthru_metadata(
task.driver.vendor.vendor_routes)
@METRICS.timer('ConductorManager.get_driver_vendor_passthru_methods')
@messaging.expected_exceptions(exception.UnsupportedDriverExtension,
exception.DriverNotFound,
exception.NoValidDefaultForInterface,
exception.InterfaceNotFoundInEntrypoint)
def get_driver_vendor_passthru_methods(self, context, driver_name):
"""Retrieve information about vendor methods of the given driver.
For dynamic drivers, the default vendor interface is used.
:param context: an admin context.
:param driver_name: name of the driver or hardware_type
:raises: UnsupportedDriverExtension if current driver does not have
vendor interface.
:raises: DriverNotFound if the supplied driver is not loaded.
:raises: NoValidDefaultForInterface if no default interface
implementation can be found for this driver's vendor
interface.
:raises: InterfaceNotFoundInEntrypoint if the default interface for a
hardware type is invalid.
:returns: dictionary of <method name>:<method metadata> entries.
"""
# Any locking in a top-level vendor action will need to be done by the
# implementation, as there is little we could reasonably lock on here.
LOG.debug("RPC get_driver_vendor_passthru_methods for driver %s",
driver_name)
driver = driver_factory.get_hardware_type(driver_name)
vendor = None
vendor_name = driver_factory.default_interface(
driver, 'vendor', driver_name=driver_name)
vendor = driver_factory.get_interface(driver, 'vendor',
vendor_name)
return get_vendor_passthru_metadata(vendor.driver_routes)
@METRICS.timer('ConductorManager.do_node_rescue')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeInMaintenance,
exception.NodeLocked,
exception.InstanceRescueFailure,
exception.InvalidStateRequested,
exception.UnsupportedDriverExtension
)
def do_node_rescue(self, context, node_id, rescue_password):
"""RPC method to rescue an existing node deployment.
Validate driver specific information synchronously, and then
spawn a background worker to rescue the node asynchronously.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:param rescue_password: string to be set as the password inside the
rescue environment.
:raises: InstanceRescueFailure if the node cannot be placed into
rescue mode.
:raises: InvalidStateRequested if the state transition is not supported
or allowed.
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
:raises: NodeLocked if the node is locked by another conductor.
:raises: NodeInMaintenance if the node is in maintenance mode.
:raises: UnsupportedDriverExtension if rescue interface is not
supported by the driver.
"""
LOG.debug("RPC do_node_rescue called for node %s.", node_id)
with task_manager.acquire(context,
node_id, purpose='node rescue') as task:
node = task.node
if node.maintenance:
raise exception.NodeInMaintenance(op=_('rescuing'),
node=node.uuid)
# driver validation may check rescue_password, so save it on the
# node early
instance_info = node.instance_info
instance_info['rescue_password'] = rescue_password
node.instance_info = instance_info
node.save()
try:
task.driver.power.validate(task)
task.driver.rescue.validate(task)
task.driver.network.validate(task)
except (exception.InvalidParameterValue,
exception.UnsupportedDriverExtension,
exception.MissingParameterValue) as e:
utils.remove_node_rescue_password(node, save=True)
raise exception.InstanceRescueFailure(
instance=node.instance_uuid,
node=node.uuid,
reason=_("Validation failed. Error: %s") % e)
try:
task.process_event(
'rescue',
callback=self._spawn_worker,
call_args=(self._do_node_rescue, task),
err_handler=utils.spawn_rescue_error_handler)
except exception.InvalidState:
utils.remove_node_rescue_password(node, save=True)
raise exception.InvalidStateRequested(
action='rescue', node=node.uuid,
state=node.provision_state)
def _do_node_rescue(self, task):
"""Internal RPC method to rescue an existing node deployment."""
node = task.node
def handle_failure(e, errmsg, log_func=LOG.error):
utils.remove_node_rescue_password(node, save=False)
node.last_error = errmsg % e
task.process_event('fail')
log_func('Error while performing rescue operation for node '
'%(node)s with instance %(instance)s: %(err)s',
{'node': node.uuid, 'instance': node.instance_uuid,
'err': e})
try:
next_state = task.driver.rescue.rescue(task)
except exception.IronicException as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to rescue: %s'))
except Exception as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to rescue. Exception: %s'),
log_func=LOG.exception)
if next_state == states.RESCUEWAIT:
task.process_event('wait')
elif next_state == states.RESCUE:
task.process_event('done')
else:
error = (_("Driver returned unexpected state %s") % next_state)
handle_failure(error,
_('Failed to rescue: %s'))
@METRICS.timer('ConductorManager.do_node_unrescue')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeInMaintenance,
exception.NodeLocked,
exception.InstanceUnrescueFailure,
exception.InvalidStateRequested,
exception.UnsupportedDriverExtension
)
def do_node_unrescue(self, context, node_id):
"""RPC method to unrescue a node in rescue mode.
Validate driver specific information synchronously, and then
spawn a background worker to unrescue the node asynchronously.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:raises: InstanceUnrescueFailure if the node fails to be unrescued
:raises: InvalidStateRequested if the state transition is not supported
or allowed.
:raises: NoFreeConductorWorker when there is no free worker to start
async task
:raises: NodeLocked if the node is locked by another conductor.
:raises: NodeInMaintenance if the node is in maintenance mode.
:raises: UnsupportedDriverExtension if rescue interface is not
supported by the driver.
"""
LOG.debug("RPC do_node_unrescue called for node %s.", node_id)
with task_manager.acquire(context, node_id,
purpose='node unrescue') as task:
node = task.node
if node.maintenance:
raise exception.NodeInMaintenance(op=_('unrescuing'),
node=node.uuid)
try:
task.driver.power.validate(task)
except (exception.InvalidParameterValue,
exception.MissingParameterValue) as e:
raise exception.InstanceUnrescueFailure(
instance=node.instance_uuid,
node=node.uuid,
reason=_("Validation failed. Error: %s") % e)
try:
task.process_event(
'unrescue',
callback=self._spawn_worker,
call_args=(self._do_node_unrescue, task),
err_handler=utils.provisioning_error_handler)
except exception.InvalidState:
raise exception.InvalidStateRequested(
action='unrescue', node=node.uuid,
state=node.provision_state)
def _do_node_unrescue(self, task):
"""Internal RPC method to unrescue a node in rescue mode."""
node = task.node
def handle_failure(e, errmsg, log_func=LOG.error):
node.last_error = errmsg % e
task.process_event('fail')
log_func('Error while performing unrescue operation for node '
'%(node)s with instance %(instance)s: %(err)s',
{'node': node.uuid, 'instance': node.instance_uuid,
'err': e})
try:
next_state = task.driver.rescue.unrescue(task)
except exception.IronicException as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to unrescue: %s'))
except Exception as e:
with excutils.save_and_reraise_exception():
handle_failure(e,
_('Failed to unrescue. Exception: %s'),
log_func=LOG.exception)
if next_state == states.ACTIVE:
task.process_event('done')
else:
error = (_("Driver returned unexpected state %s") % next_state)
handle_failure(error,
_('Failed to unrescue: %s'))
@task_manager.require_exclusive_lock
def _do_node_rescue_abort(self, task):
"""Internal method to abort an ongoing rescue operation.
:param task: a TaskManager instance with an exclusive lock
"""
node = task.node
try:
task.driver.rescue.clean_up(task)
except Exception as e:
LOG.exception('Failed to clean up rescue for node %(node)s '
'after aborting the operation. Error: %(err)s',
{'node': node.uuid, 'err': e})
error_msg = _('Failed to clean up rescue after aborting '
'the operation')
node.refresh()
node.last_error = error_msg
node.maintenance = True
node.maintenance_reason = error_msg
node.fault = faults.RESCUE_ABORT_FAILURE
node.save()
return
info_message = _('Rescue operation aborted for node %s.') % node.uuid
last_error = _('By request, the rescue operation was aborted.')
node.refresh()
node.last_error = last_error
node.save()
LOG.info(info_message)
@METRICS.timer('ConductorManager.do_node_deploy')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeLocked,
exception.NodeInMaintenance,
exception.InstanceDeployFailure,
exception.InvalidStateRequested)
def do_node_deploy(self, context, node_id, rebuild=False,
configdrive=None):
"""RPC method to initiate deployment to a node.
Initiate the deployment of a node. Validations are done
synchronously and the actual deploy work is performed in
background (asynchronously).
:param context: an admin context.
:param node_id: the id or uuid of a node.
:param rebuild: True if this is a rebuild request. A rebuild will
recreate the instance on the same node, overwriting
all disk. The ephemeral partition, if it exists, can
optionally be preserved.
:param configdrive: Optional. A gzipped and base64 encoded configdrive.
:raises: InstanceDeployFailure
:raises: NodeInMaintenance if the node is in maintenance mode.
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
:raises: InvalidStateRequested when the requested state is not a valid
target from the current state.
"""
LOG.debug("RPC do_node_deploy called for node %s.", node_id)
# NOTE(comstud): If the _sync_power_states() periodic task happens
# to have locked this node, we'll fail to acquire the lock. The
# client should perhaps retry in this case unless we decide we
# want to add retries or extra synchronization here.
with task_manager.acquire(context, node_id, shared=False,
purpose='node deployment') as task:
node = task.node
if node.maintenance:
raise exception.NodeInMaintenance(op=_('provisioning'),
node=node.uuid)
if rebuild:
event = 'rebuild'
# Note(gilliard) Clear these to force the driver to
# check whether they have been changed in glance
# NOTE(vdrok): If image_source is not from Glance we should
# not clear kernel and ramdisk as they're input manually
if glance_utils.is_glance_image(
node.instance_info.get('image_source')):
instance_info = node.instance_info
instance_info.pop('kernel', None)
instance_info.pop('ramdisk', None)
node.instance_info = instance_info
else:
event = 'deploy'
driver_internal_info = node.driver_internal_info
# Infer the image type to make sure the deploy driver
# validates only the necessary variables for different
# image types.
# NOTE(sirushtim): The iwdi variable can be None. It's up to
# the deploy driver to validate this.
iwdi = images.is_whole_disk_image(context, node.instance_info)
driver_internal_info['is_whole_disk_image'] = iwdi
node.driver_internal_info = driver_internal_info
node.save()
try:
task.driver.power.validate(task)
task.driver.deploy.validate(task)
utils.validate_instance_info_traits(task.node)
except exception.InvalidParameterValue as e:
raise exception.InstanceDeployFailure(
_("Failed to validate deploy or power info for node "
"%(node_uuid)s. Error: %(msg)s") %
{'node_uuid': node.uuid, 'msg': e}, code=e.code)
LOG.debug("do_node_deploy Calling event: %(event)s for node: "
"%(node)s", {'event': event, 'node': node.uuid})
try:
task.process_event(
event,
callback=self._spawn_worker,
call_args=(do_node_deploy, task, self.conductor.id,
configdrive),
err_handler=utils.provisioning_error_handler)
except exception.InvalidState:
raise exception.InvalidStateRequested(
action=event, node=task.node.uuid,
state=task.node.provision_state)
def _get_node_next_deploy_steps(self, task):
return self._get_node_next_steps(task, 'deploy')
[docs] @METRICS.timer('ConductorManager.continue_node_deploy')
def continue_node_deploy(self, context, node_id):
"""RPC method to continue deploying a node.
This is useful for deploying tasks that are async. When they complete,
they call back via RPC, a new worker and lock are set up, and deploying
continues. This can also be used to resume deploying on take_over.
:param context: an admin context.
:param node_id: the ID or UUID of a node.
:raises: InvalidStateRequested if the node is not in DEPLOYWAIT state
:raises: NoFreeConductorWorker when there is no free worker to start
async task
:raises: NodeLocked if node is locked by another conductor.
:raises: NodeNotFound if the node no longer appears in the database
"""
LOG.debug("RPC continue_node_deploy called for node %s.", node_id)
with task_manager.acquire(context, node_id, shared=False,
purpose='continue node deploying') as task:
node = task.node
# FIXME(rloo): This should be states.DEPLOYWAIT, but we're using
# this temporarily to get control back to the conductor, to finish
# the deployment. Once we split up the deployment into separate
# deploy steps and after we've crossed a rolling-upgrade boundary,
# we should be able to check for DEPLOYWAIT only.
expected_states = [states.DEPLOYWAIT, states.DEPLOYING]
if node.provision_state not in expected_states:
raise exception.InvalidStateRequested(_(
'Cannot continue deploying on %(node)s. Node is in '
'%(state)s state; should be in one of %(deploy_state)s') %
{'node': node.uuid,
'state': node.provision_state,
'deploy_state': ', '.join(expected_states)})
next_step_index = self._get_node_next_deploy_steps(task)
# TODO(rloo): When deprecation period is over and node is in
# states.DEPLOYWAIT only, delete the 'if' and always 'resume'.
if node.provision_state != states.DEPLOYING:
task.process_event('resume')
task.set_spawn_error_hook(utils.spawn_deploying_error_handler,
task.node)
task.spawn_after(
self._spawn_worker,
_do_next_deploy_step,
task, next_step_index, self.conductor.id)
@METRICS.timer('ConductorManager.do_node_tear_down')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeLocked,
exception.InstanceDeployFailure,
exception.InvalidStateRequested)
def do_node_tear_down(self, context, node_id):
"""RPC method to tear down an existing node deployment.
Validate driver specific information synchronously, and then
spawn a background worker to tear down the node asynchronously.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:raises: InstanceDeployFailure
:raises: NoFreeConductorWorker when there is no free worker to start
async task
:raises: InvalidStateRequested when the requested state is not a valid
target from the current state.
"""
LOG.debug("RPC do_node_tear_down called for node %s.", node_id)
with task_manager.acquire(context, node_id, shared=False,
purpose='node tear down') as task:
try:
# NOTE(ghe): Valid power driver values are needed to perform
# a tear-down. Deploy info is useful to purge the cache but not
# required for this method.
task.driver.power.validate(task)
except exception.InvalidParameterValue as e:
raise exception.InstanceDeployFailure(_(
"Failed to validate power driver interface. "
"Can not delete instance. Error: %(msg)s") % {'msg': e})
try:
task.process_event(
'delete',
callback=self._spawn_worker,
call_args=(self._do_node_tear_down, task,
task.node.provision_state),
err_handler=utils.provisioning_error_handler)
except exception.InvalidState:
raise exception.InvalidStateRequested(
action='delete', node=task.node.uuid,
state=task.node.provision_state)
@task_manager.require_exclusive_lock
def _do_node_tear_down(self, task, initial_state):
"""Internal RPC method to tear down an existing node deployment.
:param task: a task from TaskManager.
:param initial_state: The initial provision state from which node
has moved into deleting state.
"""
node = task.node
try:
if (initial_state in (states.RESCUEWAIT, states.RESCUE,
states.UNRESCUEFAIL, states.RESCUEFAIL)):
# Perform rescue clean up. Rescue clean up will remove
# rescuing network as well.
task.driver.rescue.clean_up(task)
# stop the console
# do it in this thread since we're already out of the main
# conductor thread.
if node.console_enabled:
self._set_console_mode(task, False)
task.driver.deploy.clean_up(task)
task.driver.deploy.tear_down(task)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.exception('Error in tear_down of node %(node)s: %(err)s',
{'node': node.uuid, 'err': e})
node.last_error = _("Failed to tear down. Error: %s") % e
task.process_event('error')
else:
# NOTE(deva): When tear_down finishes, the deletion is done,
# cleaning will start next
LOG.info('Successfully unprovisioned node %(node)s with '
'instance %(instance)s.',
{'node': node.uuid, 'instance': node.instance_uuid})
finally:
# NOTE(deva): there is no need to unset conductor_affinity
# because it is a reference to the most recent conductor which
# deployed a node, and does not limit any future actions.
# But we do need to clear the instance-related fields.
node.instance_info = {}
node.instance_uuid = None
driver_internal_info = node.driver_internal_info
driver_internal_info.pop('instance', None)
driver_internal_info.pop('clean_steps', None)
driver_internal_info.pop('root_uuid_or_disk_id', None)
driver_internal_info.pop('is_whole_disk_image', None)
driver_internal_info.pop('deploy_boot_mode', None)
node.driver_internal_info = driver_internal_info
network.remove_vifs_from_node(task)
node.save()
# Begin cleaning
task.process_event('clean')
self._do_node_clean(task)
def _get_node_next_steps(self, task, step_type,
skip_current_step=True):
"""Get the task's node's next steps.
This determines what the next (remaining) steps are, and
returns the index into the steps list that corresponds to the
next step. The remaining steps are determined as follows:
* If no steps have been started yet, all the steps
must be executed
* If skip_current_step is False, the remaining steps start
with the current step. Otherwise, the remaining steps
start with the step after the current one.
All the steps are in node.driver_internal_info['<step_type>_steps'].
node.<step_type>_step is the current step that was just executed
(or None, {} if no steps have been executed yet).
node.driver_internal_info['<step_type>_step_index'] is the index
index into the steps list (or None, doesn't exist if no steps have
been executed yet) and corresponds to node.<step_type>_step.
:param task: A TaskManager object
:param step_type: The type of steps to process: 'clean' or 'deploy'.
:param skip_current_step: True to skip the current step; False to
include it.
:returns: index of the next step; None if there are none to execute.
"""
valid_types = set(['clean', 'deploy'])
if step_type not in valid_types:
# NOTE(rloo): No need to i18n this, since this would be a
# developer error; it isn't user-facing.
raise exception.Invalid(
'step_type must be one of %(valid)s, not %(step)s'
% {'valid': valid_types, 'step': step_type})
node = task.node
if not getattr(node, '%s_step' % step_type):
# first time through, all steps need to be done. Return the
# index of the first step in the list.
return 0
ind = node.driver_internal_info.get('%s_step_index' % step_type)
if ind is None:
return None
if skip_current_step:
ind += 1
if ind >= len(node.driver_internal_info['%s_steps' % step_type]):
# no steps left to do
ind = None
return ind
def _get_node_next_clean_steps(self, task, skip_current_step=True):
return self._get_node_next_steps(task, 'clean',
skip_current_step=skip_current_step)
@METRICS.timer('ConductorManager.do_node_clean')
@messaging.expected_exceptions(exception.InvalidParameterValue,
exception.InvalidStateRequested,
exception.NodeInMaintenance,
exception.NodeLocked,
exception.NoFreeConductorWorker)
def do_node_clean(self, context, node_id, clean_steps):
"""RPC method to initiate manual cleaning.
:param context: an admin context.
:param node_id: the ID or UUID of a node.
:param clean_steps: an ordered list of clean steps that will be
performed on the node. A clean step is a dictionary with required
keys 'interface' and 'step', and optional key 'args'. If
specified, the 'args' arguments are passed to the clean step
method.::
{ 'interface': <driver_interface>,
'step': <name_of_clean_step>,
'args': {<arg1>: <value1>, ..., <argn>: <valuen>} }
For example (this isn't a real example, this clean step
doesn't exist)::
{ 'interface': deploy',
'step': 'upgrade_firmware',
'args': {'force': True} }
:raises: InvalidParameterValue if power validation fails.
:raises: InvalidStateRequested if the node is not in manageable state.
:raises: NodeLocked if node is locked by another conductor.
:raises: NoFreeConductorWorker when there is no free worker to start
async task.
"""
with task_manager.acquire(context, node_id, shared=False,
purpose='node manual cleaning') as task:
node = task.node
if node.maintenance:
raise exception.NodeInMaintenance(op=_('cleaning'),
node=node.uuid)
# NOTE(rloo): _do_node_clean() will also make similar calls to
# validate power & network, but we are doing it again here so that
# the user gets immediate feedback of any issues. This behaviour
# (of validating) is consistent with other methods like
# self.do_node_deploy().
try:
task.driver.power.validate(task)
task.driver.network.validate(task)
except exception.InvalidParameterValue as e:
msg = (_('Validation failed. Cannot clean node %(node)s. '
'Error: %(msg)s') %
{'node': node.uuid, 'msg': e})
raise exception.InvalidParameterValue(msg)
try:
task.process_event(
'clean',
callback=self._spawn_worker,
call_args=(self._do_node_clean, task, clean_steps),
err_handler=utils.provisioning_error_handler,
target_state=states.MANAGEABLE)
except exception.InvalidState:
raise exception.InvalidStateRequested(
action='manual clean', node=node.uuid,
state=node.provision_state)
[docs] @METRICS.timer('ConductorManager.continue_node_clean')
def continue_node_clean(self, context, node_id):
"""RPC method to continue cleaning a node.
This is useful for cleaning tasks that are async. When they complete,
they call back via RPC, a new worker and lock are set up, and cleaning
continues. This can also be used to resume cleaning on take_over.
:param context: an admin context.
:param node_id: the id or uuid of a node.
:raises: InvalidStateRequested if the node is not in CLEANWAIT state
:raises: NoFreeConductorWorker when there is no free worker to start
async task
:raises: NodeLocked if node is locked by another conductor.
:raises: NodeNotFound if the node no longer appears in the database
"""
LOG.debug("RPC continue_node_clean called for node %s.", node_id)
with task_manager.acquire(context, node_id, shared=False,
purpose='continue node cleaning') as task:
node = task.node
if node.target_provision_state == states.MANAGEABLE:
target_state = states.MANAGEABLE
else:
target_state = None
if node.provision_state != states.CLEANWAIT:
raise exception.InvalidStateRequested(_(
'Cannot continue cleaning on %(node)s, node is in '
'%(state)s state, should be %(clean_state)s') %
{'node': node.uuid,
'state': node.provision_state,
'clean_state': states.CLEANWAIT})
info = node.driver_internal_info
try:
skip_current_step = info.pop('skip_current_clean_step')
except KeyError:
skip_current_step = True
else:
node.driver_internal_info = info
node.save()
next_step_index = self._get_node_next_clean_steps(
task, skip_current_step=skip_current_step)
# If this isn't the final clean step in the cleaning operation
# and it is flagged to abort after the clean step that just
# finished, we abort the cleaning operation.
if node.clean_step.get('abort_after'):
step_name = node.clean_step['step']
if next_step_index is not None:
LOG.debug('The cleaning operation for node %(node)s was '
'marked to be aborted after step "%(step)s '
'completed. Aborting now that it has completed.',
{'node': task.node.uuid, 'step': step_name})
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_clean_abort,
task, step_name),
err_handler=utils.provisioning_error_handler,
target_state=target_state)
return
LOG.debug('The cleaning operation for node %(node)s was '
'marked to be aborted after step "%(step)s" '
'completed. However, since there are no more '
'clean steps after this, the abort is not going '
'to be done.', {'node': node.uuid,
'step': step_name})
task.process_event('resume', target_state=target_state)
task.set_spawn_error_hook(utils.spawn_cleaning_error_handler,
task.node)
task.spawn_after(
self._spawn_worker,
self._do_next_clean_step,
task, next_step_index)
@task_manager.require_exclusive_lock
def _do_node_clean(self, task, clean_steps=None):
"""Internal RPC method to perform cleaning of a node.
:param task: a TaskManager instance with an exclusive lock on its node
:param clean_steps: For a manual clean, the list of clean steps to
perform. Is None For automated cleaning (default).
For more information, see the clean_steps parameter
of :func:`ConductorManager.do_node_clean`.
"""
node = task.node
manual_clean = clean_steps is not None
clean_type = 'manual' if manual_clean else 'automated'
LOG.debug('Starting %(type)s cleaning for node %(node)s',
{'type': clean_type, 'node': node.uuid})
if not manual_clean and not CONF.conductor.automated_clean:
# Skip cleaning, move to AVAILABLE.
node.clean_step = None
node.save()
task.process_event('done')
LOG.info('Automated cleaning is disabled, node %s has been '
'successfully moved to AVAILABLE state.', node.uuid)
return
try:
# NOTE(ghe): Valid power and network values are needed to perform
# a cleaning.
task.driver.power.validate(task)
task.driver.network.validate(task)
except exception.InvalidParameterValue as e:
msg = (_('Validation failed. Cannot clean node %(node)s. '
'Error: %(msg)s') %
{'node': node.uuid, 'msg': e})
return utils.cleaning_error_handler(task, msg)
if manual_clean:
info = node.driver_internal_info
info['clean_steps'] = clean_steps
node.driver_internal_info = info
node.save()
# Do caching of bios settings if supported by driver,
# this will be called for both manual and automated cleaning.
# TODO(zshi) remove this check when classic drivers are removed
try:
task.driver.bios.cache_bios_settings(task)
except Exception as e:
msg = (_('Caching of bios settings failed on node %(node)s. '
'Continuing with node cleaning.')
% {'node': node.uuid})
LOG.exception(msg)
# Allow the deploy driver to set up the ramdisk again (necessary for
# IPA cleaning)
try:
prepare_result = task.driver.deploy.prepare_cleaning(task)
except Exception as e:
msg = (_('Failed to prepare node %(node)s for cleaning: %(e)s')
% {'node': node.uuid, 'e': e})
LOG.exception(msg)
return utils.cleaning_error_handler(task, msg)
if prepare_result == states.CLEANWAIT:
# Prepare is asynchronous, the deploy driver will need to
# set node.driver_internal_info['clean_steps'] and
# node.clean_step and then make an RPC call to
# continue_node_cleaning to start cleaning.
# For manual cleaning, the target provision state is MANAGEABLE,
# whereas for automated cleaning, it is AVAILABLE (the default).
target_state = states.MANAGEABLE if manual_clean else None
task.process_event('wait', target_state=target_state)
return
try:
utils.set_node_cleaning_steps(task)
except (exception.InvalidParameterValue,
exception.NodeCleaningFailure) as e:
msg = (_('Cannot clean node %(node)s. Error: %(msg)s')
% {'node': node.uuid, 'msg': e})
return utils.cleaning_error_handler(task, msg)
steps = node.driver_internal_info.get('clean_steps', [])
step_index = 0 if steps else None
self._do_next_clean_step(task, step_index)
@task_manager.require_exclusive_lock
def _do_next_clean_step(self, task, step_index):
"""Do cleaning, starting from the specified clean step.
:param task: a TaskManager instance with an exclusive lock
:param step_index: The first clean step in the list to execute. This
is the index (from 0) into the list of clean steps in the node's
driver_internal_info['clean_steps']. Is None if there are no steps
to execute.
"""
node = task.node
# For manual cleaning, the target provision state is MANAGEABLE,
# whereas for automated cleaning, it is AVAILABLE.
manual_clean = node.target_provision_state == states.MANAGEABLE
if step_index is None:
steps = []
else:
steps = node.driver_internal_info['clean_steps'][step_index:]
LOG.info('Executing %(state)s on node %(node)s, remaining steps: '
'%(steps)s', {'node': node.uuid, 'steps': steps,
'state': node.provision_state})
# Execute each step until we hit an async step or run out of steps
for ind, step in enumerate(steps):
# Save which step we're about to start so we can restart
# if necessary
node.clean_step = step
driver_internal_info = node.driver_internal_info
driver_internal_info['clean_step_index'] = step_index + ind
node.driver_internal_info = driver_internal_info
node.save()
interface = getattr(task.driver, step.get('interface'))
LOG.info('Executing %(step)s on node %(node)s',
{'step': step, 'node': node.uuid})
try:
result = interface.execute_clean_step(task, step)
except Exception as e:
if isinstance(e, exception.AgentConnectionFailed):
if task.node.driver_internal_info.get('cleaning_reboot'):
LOG.info('Agent is not yet running on node %(node)s '
'after cleaning reboot, waiting for agent to '
'come up to run next clean step %(step)s.',
{'node': node.uuid, 'step': step})
driver_internal_info['skip_current_clean_step'] = False
node.driver_internal_info = driver_internal_info
target_state = (states.MANAGEABLE if manual_clean
else None)
task.process_event('wait', target_state=target_state)
return
msg = (_('Node %(node)s failed step %(step)s: '
'%(exc)s') %
{'node': node.uuid, 'exc': e,
'step': node.clean_step})
LOG.exception(msg)
utils.cleaning_error_handler(task, msg)
return
# Check if the step is done or not. The step should return
# states.CLEANWAIT if the step is still being executed, or
# None if the step is done.
if result == states.CLEANWAIT:
# Kill this worker, the async step will make an RPC call to
# continue_node_clean to continue cleaning
LOG.info('Clean step %(step)s on node %(node)s being '
'executed asynchronously, waiting for driver.',
{'node': node.uuid, 'step': step})
target_state = states.MANAGEABLE if manual_clean else None
task.process_event('wait', target_state=target_state)
return
elif result is not None:
msg = (_('While executing step %(step)s on node '
'%(node)s, step returned invalid value: %(val)s')
% {'step': step, 'node': node.uuid, 'val': result})
LOG.error(msg)
return utils.cleaning_error_handler(task, msg)
LOG.info('Node %(node)s finished clean step %(step)s',
{'node': node.uuid, 'step': step})
# Clear clean_step
node.clean_step = None
driver_internal_info = node.driver_internal_info
driver_internal_info['clean_steps'] = None
driver_internal_info.pop('clean_step_index', None)
driver_internal_info.pop('cleaning_reboot', None)
node.driver_internal_info = driver_internal_info
node.save()
try:
task.driver.deploy.tear_down_cleaning(task)
except Exception as e:
msg = (_('Failed to tear down from cleaning for node %(node)s, '
'reason: %(err)s')
% {'node': node.uuid, 'err': e})
LOG.exception(msg)
return utils.cleaning_error_handler(task, msg,
tear_down_cleaning=False)
LOG.info('Node %s cleaning complete', node.uuid)
event = 'manage' if manual_clean else 'done'
# NOTE(rloo): No need to specify target prov. state; we're done
task.process_event(event)
@task_manager.require_exclusive_lock
def _do_node_verify(self, task):
"""Internal method to perform power credentials verification."""
node = task.node
LOG.debug('Starting power credentials verification for node %s',
node.uuid)
error = None
try:
task.driver.power.validate(task)
except Exception as e:
error = (_('Failed to validate power driver interface for node '
'%(node)s. Error: %(msg)s') %
{'node': node.uuid, 'msg': e})
else:
try:
power_state = task.driver.power.get_power_state(task)
except Exception as e:
error = (_('Failed to get power state for node '
'%(node)s. Error: %(msg)s') %
{'node': node.uuid, 'msg': e})
if error is None:
if power_state != node.power_state:
old_power_state = node.power_state
node.power_state = power_state
task.process_event('done')
notify_utils.emit_power_state_corrected_notification(
task, old_power_state)
else:
task.process_event('done')
else:
LOG.error(error)
node.last_error = error
task.process_event('fail')
@task_manager.require_exclusive_lock
def _do_node_clean_abort(self, task, step_name=None):
"""Internal method to abort an ongoing operation.
:param task: a TaskManager instance with an exclusive lock
:param step_name: The name of the clean step.
"""
node = task.node
try:
task.driver.deploy.tear_down_cleaning(task)
except Exception as e:
LOG.exception('Failed to tear down cleaning for node %(node)s '
'after aborting the operation. Error: %(err)s',
{'node': node.uuid, 'err': e})
error_msg = _('Failed to tear down cleaning after aborting '
'the operation')
utils.cleaning_error_handler(task, error_msg,
tear_down_cleaning=False,
set_fail_state=False)
return
info_message = _('Clean operation aborted for node %s') % node.uuid
last_error = _('By request, the clean operation was aborted')
if step_name:
msg = _(' after the completion of step "%s"') % step_name
last_error += msg
info_message += msg
node.last_error = last_error
node.clean_step = None
node.save()
LOG.info(info_message)
@METRICS.timer('ConductorManager.do_provisioning_action')
@messaging.expected_exceptions(exception.NoFreeConductorWorker,
exception.NodeLocked,
exception.InvalidParameterValue,
exception.InvalidStateRequested,
exception.UnsupportedDriverExtension)
def do_provisioning_action(self, context, node_id, action):
"""RPC method to initiate certain provisioning state transitions.
Initiate a provisioning state change through the state machine,
rather than through an RPC call to do_node_deploy / do_node_tear_down
:param context: an admin context.
:param node_id: the id or uuid of a node.
:param action: an action. One of ironic.common.states.VERBS
:raises: InvalidParameterValue
:raises: InvalidStateRequested
:raises: NoFreeConductorWorker
"""
with task_manager.acquire(context, node_id, shared=False,
purpose='provision action %s'
% action) as task:
node = task.node
if (action == states.VERBS['provide']
and node.provision_state == states.MANAGEABLE):
task.process_event(
'provide',
callback=self._spawn_worker,
call_args=(self._do_node_clean, task),
err_handler=utils.provisioning_error_handler)
return
if (action == states.VERBS['manage']
and node.provision_state == states.ENROLL):
task.process_event(
'manage',
callback=self._spawn_worker,
call_args=(self._do_node_verify, task),
err_handler=utils.provisioning_error_handler)
return
if (action == states.VERBS['adopt']
and node.provision_state in (states.MANAGEABLE,
states.ADOPTFAIL)):
task.process_event(
'adopt',
callback=self._spawn_worker,
call_args=(self._do_adoption, task),
err_handler=utils.provisioning_error_handler)
return
if (action == states.VERBS['abort'] and
node.provision_state in (states.CLEANWAIT,
states.RESCUEWAIT,
states.INSPECTWAIT)):
self._do_abort(task)
return
try:
task.process_event(action)
except exception.InvalidState:
raise exception.InvalidStateRequested(
action=action, node=node.uuid,
state=node.provision_state)
def _do_abort(self, task):
"""Handle node abort for certain states."""
node = task.node
if node.provision_state == states.CLEANWAIT:
# Check if the clean step is abortable; if so abort it.
# Otherwise, indicate in that clean step, that cleaning
# should be aborted after that step is done.
if (node.clean_step and not
node.clean_step.get('abortable')):
LOG.info('The current clean step "%(clean_step)s" for '
'node %(node)s is not abortable. Adding a '
'flag to abort the cleaning after the clean '
'step is completed.',
{'clean_step': node.clean_step['step'],
'node': node.uuid})
clean_step = node.clean_step
if not clean_step.get('abort_after'):
clean_step['abort_after'] = True
node.clean_step = clean_step
node.save()
return
LOG.debug('Aborting the cleaning operation during clean step '
'"%(step)s" for node %(node)s in provision state '
'"%(prov)s".',
{'node': node.uuid,
'prov': node.provision_state,
'step': node.clean_step.get('step')})
target_state = None
if node.target_provision_state == states.MANAGEABLE:
target_state = states.MANAGEABLE
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_clean_abort, task),
err_handler=utils.provisioning_error_handler,
target_state=target_state)
return
if node.provision_state == states.RESCUEWAIT:
utils.remove_node_rescue_password(node, save=True)
task.process_event(
'abort',
callback=self._spawn_worker,
call_args=(self._do_node_rescue_abort, task),
err_handler=utils.provisioning_error_handler)
return
if node.provision_state == states.INSPECTWAIT:
try:
task.driver.inspect.abort(task)
except exception.UnsupportedDriverExtension:
with excutils.save_and_reraise_exception():
intf_name = task.driver.inspect.__class__.__name__
LOG.error('Inspect interface %(intf)s does not '
'support abort operation when aborting '
'inspection of node %(node)s',
{'intf': intf_name, 'node': node.uuid})
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.exception('Error in aborting the inspection of '
'node %(node)s', {'node': node.uuid})
node.last_error = _('Failed to abort inspection. '
'Error: %s') % e
node.save()
node.last_error = _('Inspection was aborted by request.')
task.process_event('abort')
LOG.info('Successfully aborted inspection of node %(node)s',
{'node': node.uuid})
return
@METRICS.timer('ConductorManager._sync_power_states')
@periodics.periodic(spacing=CONF.conductor.sync_power_state_interval)
def _sync_power_states(self, context):
"""Periodic task to sync power states for the nodes.
Attempt to grab a lock and sync only if the following
conditions are met:
1) Node is mapped to this conductor.
2) Node is not in maintenance mode.
3) Node is not in DEPLOYWAIT/CLEANWAIT provision state.
4) Node doesn't have a reservation
NOTE: Grabbing a lock here can cause other methods to fail to
grab it. We want to avoid trying to grab a lock while a node
is in the DEPLOYWAIT/CLEANWAIT state so we don't unnecessarily
cause a deploy/cleaning callback to fail. There's not much we
can do here to avoid failing a brand new deploy to a node that
we've locked here, though.
"""
# FIXME(comstud): Since our initial state checks are outside
# of the lock (to try to avoid the lock), some checks are
# repeated after grabbing the lock so we can unlock quickly.
# The node mapping is not re-checked because it doesn't much
# matter if things happened to re-balance.
#
# This is inefficient and racey. We end up with calling DB API's
# get_node() twice (once here, and once in acquire(). Ideally we
# add a way to pass constraints to task_manager.acquire()
# (through to its DB API call) so that we can eliminate our call
# and first set of checks below.
filters = {'maintenance': False}
node_iter = self.iter_nodes(fields=['id'], filters=filters)
for (node_uuid, driver, conductor_group, node_id) in node_iter:
try:
# NOTE(dtantsur): start with a shared lock, upgrade if needed
with task_manager.acquire(context, node_uuid,
purpose='power state sync',
shared=True) as task:
# NOTE(deva): we should not acquire a lock on a node in
# DEPLOYWAIT/CLEANWAIT, as this could cause
# an error within a deploy ramdisk POSTing back
# at the same time.
# NOTE(dtantsur): it's also pointless (and dangerous) to
# sync power state when a power action is in progress
if (task.node.provision_state in SYNC_EXCLUDED_STATES
or task.node.maintenance
or task.node.target_power_state
or task.node.reservation):
continue
count = do_sync_power_state(
task, self.power_state_sync_count[node_uuid])
if count:
self.power_state_sync_count[node_uuid] = count
else:
# don't bloat the dict with non-failing nodes
del self.power_state_sync_count[node_uuid]
except exception.NodeNotFound:
LOG.info("During sync_power_state, node %(node)s was not "
"found and presumed deleted by another process.",
{'node': node_uuid})
except exception.NodeLocked:
LOG.info("During sync_power_state, node %(node)s was "
"already locked by another process. Skip.",
{'node': node_uuid})
finally:
# Yield on every iteration
eventlet.sleep(0)
@METRICS.timer('ConductorManager._power_failure_recovery')
@periodics.periodic(spacing=CONF.conductor.power_failure_recovery_interval,
enabled=bool(
CONF.conductor.power_failure_recovery_interval))
def _power_failure_recovery(self, context):
"""Periodic task to check power states for nodes in maintenance.
Attempt to grab a lock and sync only if the following
conditions are met:
1) Node is mapped to this conductor.
2) Node is in maintenance with maintenance type of power failure.
3) Node is not reserved.
4) Node is not in the ENROLL state.
"""
def should_sync_power_state_for_recovery(task):
"""Check if ironic should sync power state for recovery."""
# NOTE(dtantsur): it's also pointless (and dangerous) to
# sync power state when a power action is in progress
if (task.node.provision_state == states.ENROLL or
not task.node.maintenance or
task.node.fault != faults.POWER_FAILURE or
task.node.target_power_state or
task.node.reservation):
return False
return True
def handle_recovery(task, actual_power_state):
"""Handle recovery when power sync is succeeded."""
task.upgrade_lock()
node = task.node
# Update power state
old_power_state = node.power_state
node.power_state = actual_power_state
# Clear maintenance related fields
node.maintenance = False
node.maintenance_reason = None
node.fault = None
node.save()
LOG.info("Node %(node)s is recovered from power failure "
"with actual power state '%(state)s'.",
{'node': node.uuid, 'state': actual_power_state})
if old_power_state != actual_power_state:
notify_utils.emit_power_state_corrected_notification(
task, old_power_state)
# NOTE(kaifeng) To avoid conflicts with periodic task of the
# regular power state checking, maintenance is still a required
# condition.
filters = {'maintenance': True,
'fault': faults.POWER_FAILURE}
node_iter = self.iter_nodes(fields=['id'], filters=filters)
for (node_uuid, driver, conductor_group, node_id) in node_iter:
try:
with task_manager.acquire(context, node_uuid,
purpose='power failure recovery',
shared=True) as task:
if not should_sync_power_state_for_recovery(task):
continue
try:
# Validate driver info in case of parameter changed
# in maintenance.
task.driver.power.validate(task)
# The driver may raise an exception, or may return
# ERROR. Handle both the same way.
power_state = task.driver.power.get_power_state(task)
if power_state == states.ERROR:
raise exception.PowerStateFailure(
_("Power driver returned ERROR state "
"while trying to get power state."))
except Exception as e:
LOG.debug("During power_failure_recovery, could "
"not get power state for node %(node)s, "
"Error: %(err)s.",
{'node': task.node.uuid, 'err': e})
else:
handle_recovery(task, power_state)
except exception.NodeNotFound:
LOG.info("During power_failure_recovery, node %(node)s was "
"not found and presumed deleted by another process.",
{'node': node_uuid})
except exception.NodeLocked:
LOG.info("During power_failure_recovery, node %(node)s was "
"already locked by another process. Skip.",
{'node': node_uuid})
finally:
# Yield on every iteration
eventlet.sleep(0)
@METRICS.timer('ConductorManager._check_deploy_timeouts')
@periodics.periodic(spacing=CONF.conductor.check_provision_state_interval)
def _check_deploy_timeouts(self, context):
"""Periodically checks whether a deploy RPC call has timed out.
If a deploy call has timed out, the deploy failed and we clean up.
:param context: request context.
"""
callback_timeout = CONF.conductor.deploy_callback_timeout
if not callback_timeout:
return
filters = {'reserved': False,
'provision_state': states.DEPLOYWAIT,
'maintenance': False,
'provisioned_before': callback_timeout}
sort_key = 'provision_updated_at'
callback_method = utils.cleanup_after_timeout
err_handler = utils.provisioning_error_handler
self._fail_if_in_state(context, filters, states.DEPLOYWAIT,
sort_key, callback_method, err_handler)
@METRICS.timer('ConductorManager._check_orphan_nodes')
@periodics.periodic(spacing=CONF.conductor.check_provision_state_interval)
def _check_orphan_nodes(self, context):
"""Periodically checks the status of nodes that were taken over.
Periodically checks the nodes that are managed by this conductor but
have a reservation from a conductor that went offline.
1. Nodes in DEPLOYING state move to DEPLOY FAIL.
2. Nodes in CLEANING state move to CLEAN FAIL with maintenance set.
3. Nodes in a transient power state get the power operation aborted.
4. Reservation is removed.
The latter operation happens even for nodes in maintenance mode,
otherwise it's not possible to move them out of maintenance.
:param context: request context.
"""
offline_conductors = self.dbapi.get_offline_conductors()
if not offline_conductors:
return
node_iter = self.iter_nodes(
fields=['id', 'reservation', 'maintenance', 'provision_state',
'target_power_state'],
filters={'reserved_by_any_of': offline_conductors})
state_cleanup_required = []
for (node_uuid, driver, conductor_group, node_id, conductor_hostname,
maintenance, provision_state, target_power_state) in node_iter:
# NOTE(lucasagomes): Although very rare, this may lead to a
# race condition. By the time we release the lock the conductor
# that was previously managing the node could be back online.
try:
objects.Node.release(context, conductor_hostname, node_id)
except exception.NodeNotFound:
LOG.warning("During checking for deploying state, node "
"%s was not found and presumed deleted by "
"another process. Skipping.", node_uuid)
continue
except exception.NodeLocked:
LOG.warning("During checking for deploying state, when "
"releasing the lock of the node %s, it was "
"locked by another process. Skipping.",
node_uuid)
continue
except exception.NodeNotLocked:
LOG.warning("During checking for deploying state, when "
"releasing the lock of the node %s, it was "
"already unlocked.", node_uuid)
else:
LOG.warning('Forcibly removed reservation of conductor %(old)s'
' on node %(node)s as that conductor went offline',
{'old': conductor_hostname, 'node': node_uuid})
# TODO(dtantsur): clean up all states that are not stable and
# are not one of WAIT states.
if not maintenance and (provision_state in (states.DEPLOYING,
states.CLEANING)
or target_power_state is not None):
LOG.debug('Node %(node)s taken over from conductor %(old)s '
'requires state clean up: provision state is '
'%(state)s, target power state is %(pstate)s',
{'node': node_uuid, 'old': conductor_hostname,
'state': provision_state,
'pstate': target_power_state})
state_cleanup_required.append(node_uuid)
for node_uuid in state_cleanup_required:
with task_manager.acquire(context, node_uuid,
purpose='power state clean up') as task:
if not task.node.maintenance and task.node.target_power_state:
old_state = task.node.target_power_state
task.node.target_power_state = None
task.node.last_error = _('Pending power operation was '
'aborted due to conductor take '
'over')
task.node.save()
LOG.warning('Aborted pending power operation %(op)s '
'on node %(node)s due to conductor take over',
{'op': old_state, 'node': node_uuid})
self._fail_if_in_state(
context, {'uuid': node_uuid},
{states.DEPLOYING, states.CLEANING},
'provision_updated_at',
callback_method=utils.abort_on_conductor_take_over,
err_handler=utils.provisioning_error_handler)
@METRICS.timer('ConductorManager._do_adoption')
@task_manager.require_exclusive_lock
def _do_adoption(self, task):
"""Adopt the node.
Similar to node takeover, adoption performs a driver boot
validation and then triggers node takeover in order to make the
conductor responsible for the node. Upon completion of takeover,
the node is moved to ACTIVE state.
The goal of this method is to set the conditions for the node to
be managed by Ironic as an ACTIVE node without having performed
a deployment operation.
:param task: a TaskManager instance
"""
node = task.node
LOG.debug('Conductor %(cdr)s attempting to adopt node %(node)s',
{'cdr': self.host, 'node': node.uuid})
try:
# NOTE(TheJulia): A number of drivers expect to know if a
# whole disk image was used prior to their takeover logic
# being triggered, as such we need to populate the
# internal info based on the configuration the user has
# supplied.
iwdi = images.is_whole_disk_image(task.context,
task.node.instance_info)
driver_internal_info = node.driver_internal_info
driver_internal_info['is_whole_disk_image'] = iwdi
node.driver_internal_info = driver_internal_info
# Calling boot validate to ensure that sufficient information
# is supplied to allow the node to be able to boot if takeover
# writes items such as kernel/ramdisk data to disk.
task.driver.boot.validate(task)
# NOTE(TheJulia): While task.driver.boot.validate() is called
# above, and task.driver.power.validate() could be called, it
# is called as part of the transition from ENROLL to MANAGEABLE
# states. As such it is redundant to call here.
self._do_takeover(task)
LOG.info("Successfully adopted node %(node)s",
{'node': node.uuid})
task.process_event('done')
except Exception as err:
msg = (_('Error while attempting to adopt node %(node)s: '
'%(err)s.') % {'node': node.uuid, 'err': err})
LOG.error(msg)
node.last_error = msg
task.process_event('fail')
@METRICS.timer('ConductorManager._do_takeover')
def _do_takeover(self, task):
"""Take over this node.
Prepares a node for takeover by this conductor, performs the takeover,
and changes the conductor associated with the node. The node with the
new conductor affiliation is saved to the DB.
:param task: a TaskManager instance
"""
LOG.debug('Conductor %(cdr)s taking over node %(node)s',
{'cdr': self.host, 'node': task.node.uuid})
task.driver.deploy.prepare(task)
task.driver.deploy.take_over(task)
# NOTE(zhenguo): If console enabled, take over the console session
# as well.
console_error = None
if task.node.console_enabled:
notify_utils.emit_console_notification(
task, 'console_restore', fields.NotificationStatus.START)
try:
task.driver.console.start_console(task)
except Exception as err:
msg = (_('Failed to start console while taking over the '
'node %(node)s: %(err)s.') % {'node': task.node.uuid,
'err': err})
LOG.error(msg)
# If taking over console failed, set node's console_enabled
# back to False and set node's last error.
task.node.last_error = msg
task.node.console_enabled = False
console_error = True
else:
notify_utils.emit_console_notification(
task, 'console_restore', fields.NotificationStatus.END)
# NOTE(lucasagomes): Set the ID of the new conductor managing
# this node
task.node.conductor_affinity = self.conductor.id
task.node.save()
if console_error:
notify_utils.emit_console_notification(
task, 'console_restore', fields.NotificationStatus.ERROR)
@METRICS.timer('ConductorManager._check_cleanwait_timeouts')
@periodics.periodic(spacing=CONF.conductor.check_provision_state_interval)
def _check_cleanwait_timeouts(self, context):
"""Periodically checks for nodes being cleaned.
If a node doing cleaning is unresponsive (detected when it stops
heart beating), the operation should be aborted.
:param context: request context.
"""
callback_timeout = CONF.conductor.clean_callback_timeout
if not callback_timeout:
return
filters = {'reserved': False,
'provision_state': states.CLEANWAIT,
'maintenance': False,
'provisioned_before': callback_timeout}
self._fail_if_in_state(context, filters, states.CLEANWAIT,
'provision_updated_at',
keep_target_state=True,
callback_method=utils.cleanup_cleanwait_timeout)
@METRICS.timer('ConductorManager._check_rescuewait_timeouts')
@periodics.periodic(spacing=CONF.conductor.check_rescue_state_interval,
enabled=bool(CONF.conductor.rescue_callback_timeout))
def _check_rescuewait_timeouts(self, context):
"""Periodically checks if rescue has timed out waiting for heartbeat.
If a rescue call has timed out, fail the rescue and clean up.
:param context: request context.
"""
callback_timeout = CONF.conductor.rescue_callback_timeout
filters = {'reserved': False,
'provision_state': states.RESCUEWAIT,
'maintenance': False,
'provisioned_before': callback_timeout}
self._fail_if_in_state(context, filters, states.RESCUEWAIT,
'provision_updated_at',
keep_target_state=True,
callback_method=utils.cleanup_rescuewait_timeout
)
@METRICS.timer('ConductorManager._sync_local_state')
@periodics.periodic(spacing=CONF.conductor.sync_local_state_interval)
def _sync_local_state(self, context):
"""Perform any actions necessary to sync local state.
This is called periodically to refresh the conductor's copy of the
consistent hash ring. If any mappings have changed, this method then
determines which, if any, nodes need to be "taken over".
The ensuing actions could include preparing a PXE environment,
updating the DHCP server, and so on.
"""
filters = {'reserved': False,
'maintenance': False,
'provision_state': states.ACTIVE}
node_iter = self.iter_nodes(fields=['id', 'conductor_affinity'],
filters=filters)
workers_count = 0
for (node_uuid, driver, conductor_group, node_id,
conductor_affinity) in node_iter:
if conductor_affinity == self.conductor.id:
continue
# Node is mapped here, but not updated by this conductor last
try:
with task_manager.acquire(context, node_uuid,
purpose='node take over') as task:
# NOTE(deva): now that we have the lock, check again to
# avoid racing with deletes and other state changes
node = task.node
if (node.maintenance
or node.conductor_affinity == self.conductor.id
or node.provision_state != states.ACTIVE):
continue
task.spawn_after(self._spawn_worker,
self._do_takeover, task)
except exception.NoFreeConductorWorker:
break
except (exception.NodeLocked, exception.NodeNotFound):
continue
workers_count += 1
if workers_count == CONF.conductor.periodic_max_workers:
break
@METRICS.timer('ConductorManager.validate_driver_interfaces')
@messaging.expected_exceptions(exception.NodeLocked)
def validate_driver_interfaces(self, context, node_id):
"""Validate the `core` and `standardized` interfaces for drivers.
:param context: request context.
:param node_id: node id or uuid.
:returns: a dictionary containing the results of each
interface validation.
"""
LOG.debug('RPC validate_driver_interfaces called for node %s.',
node_id)
ret_dict = {}
lock_purpose = 'driver interface validation'
with task_manager.acquire(context, node_id, shared=True,
purpose=lock_purpose) as task:
# NOTE(sirushtim): the is_whole_disk_image variable is needed by
# deploy drivers for doing their validate(). Since the deploy
# isn't being done yet and the driver information could change in
# the meantime, we don't know if the is_whole_disk_image value will
# change or not. It isn't saved to the DB, but only used with this
# node instance for the current validations.
iwdi = images.is_whole_disk_image(context,
task.node.instance_info)
task.node.driver_internal_info['is_whole_disk_image'] = iwdi
for iface_name in task.driver.non_vendor_interfaces:
iface = getattr(task.driver, iface_name)
result = reason = None
try:
iface.validate(task)
if iface_name == 'deploy':
utils.validate_instance_info_traits(task.node)
result = True
except (exception.InvalidParameterValue,
exception.UnsupportedDriverExtension) as e:
result = False
reason = str(e)
except Exception as e:
result = False
reason = (_('Unexpected exception, traceback saved '
'into log by ironic conductor service '
'that is running on %(host)s: %(error)s')
% {'host': self.host, 'error': e})
LOG.exception(
'Unexpected exception occurred while validating '
'%(iface)s driver interface for driver '
'%(driver)s: %(err)s on node %(node)s.',
{'iface': iface_name, 'driver': task.node.driver,
'err': e, 'node': task.node.uuid})
ret_dict[iface_name] = {}
ret_dict[iface_name]['result'] = result
if reason is not None:
ret_dict[iface_name]['reason'] = reason
return ret_dict
@METRICS.timer('ConductorManager.destroy_node')
@messaging.expected_exceptions(exception.NodeLocked,
exception.NodeAssociated,
exception.InvalidState)
def destroy_node(self, context, node_id):
"""Delete a node.
:param context: request context.
:param node_id: node id or uuid.
:raises: NodeLocked if node is locked by another conductor.
:raises: NodeAssociated if the node contains an instance
associated with it.
:raises: InvalidState if the node is in the wrong provision
state to perform deletion.
"""
# NOTE(dtantsur): we allow deleting a node in maintenance mode even if
# we would disallow it otherwise. That's done for recovering hopelessly
# broken nodes (e.g. with broken BMC).
with task_manager.acquire(context, node_id,
purpose='node deletion') as task:
node = task.node
if not node.maintenance and node.instance_uuid is not None:
raise exceptio