diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index c54cb8bfb89..7ca7688dacf 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -242,6 +242,16 @@ def get_limit_and_marker(request): return limit, marker +def get_limit_and_offset(request): + """Get limit and offset parameters from request.""" + params = get_pagination_params(request) + limit = CONF.api.max_limit + limit = min(limit, params.get('limit', limit)) + offset = params.get('offset', None) + + return limit, offset + + def get_id_from_href(href): """Return the id or uuid portion of a url. diff --git a/nova/api/openstack/compute/routes.py b/nova/api/openstack/compute/routes.py index 6083c94b660..c47f40847c7 100644 --- a/nova/api/openstack/compute/routes.py +++ b/nova/api/openstack/compute/routes.py @@ -684,6 +684,7 @@ def _create_controller(main_controller, action_controller_list): }), ('/os-server-groups/{id}', { 'GET': [server_groups_controller, 'show'], + 'PUT': [server_groups_controller, 'update'], 'DELETE': [server_groups_controller, 'delete'] }), ('/os-services', { diff --git a/nova/api/openstack/compute/schemas/server_groups.py b/nova/api/openstack/compute/schemas/server_groups.py index dcaa7616d47..6eb48427627 100644 --- a/nova/api/openstack/compute/schemas/server_groups.py +++ b/nova/api/openstack/compute/schemas/server_groups.py @@ -99,3 +99,18 @@ 'properties': {}, 'additionalProperties': True, } + +update = { + 'type': 'object', + 'properties': { + 'add_members': { + 'type': 'array', + 'items': parameter_types.server_id, + }, + 'remove_members': { + 'type': 'array', + 'items': parameter_types.server_id, + } + }, + 'additionalProperties': False +} diff --git a/nova/api/openstack/compute/server_groups.py b/nova/api/openstack/compute/server_groups.py index b8c906552f2..f3367008d63 100644 --- a/nova/api/openstack/compute/server_groups.py +++ b/nova/api/openstack/compute/server_groups.py @@ -16,6 +16,7 @@ """The Server Group API Extension.""" import collections +import itertools from oslo_log import log as logging import webob @@ -26,6 +27,7 @@ from nova.api.openstack.compute.schemas import server_groups as schema from nova.api.openstack import wsgi from nova.api import validation +from nova.compute import api as compute import nova.conf from nova import context as nova_context import nova.exception @@ -40,12 +42,21 @@ CONF = nova.conf.CONF -def _get_not_deleted(context, uuids): +GROUP_POLICY_OBJ_MICROVERSION = "2.64" + + +def _get_not_deleted(context, uuids, not_deleted_inst=None): + if not_deleted_inst: + # short-cut if we already pre-built a list of not deleted instances to + # be more efficient + return {u: not_deleted_inst[u] for u in uuids + if u in not_deleted_inst} + mappings = objects.InstanceMappingList.get_by_instance_uuids( context, uuids) inst_by_cell = collections.defaultdict(list) cell_mappings = {} - found_inst_uuids = [] + found_inst = {} # Get a master list of cell mappings, and a list of instance # uuids organized by cell @@ -53,7 +64,7 @@ def _get_not_deleted(context, uuids): if not im.cell_mapping: # Not scheduled yet, so just throw it in the final list # and move on - found_inst_uuids.append(im.instance_uuid) + found_inst[im.instance_uuid] = None continue if im.cell_mapping.uuid not in cell_mappings: cell_mappings[im.cell_mapping.uuid] = im.cell_mapping @@ -67,11 +78,11 @@ def _get_not_deleted(context, uuids): {'cell': cell_mapping.identity, 'num': len(inst_uuids)}) filters = {'uuid': inst_uuids, 'deleted': False} with nova_context.target_cell(context, cell_mapping) as ctx: - found_inst_uuids.extend([ - inst.uuid for inst in objects.InstanceList.get_by_filters( - ctx, filters=filters)]) + instances = objects.InstanceList.get_by_filters( + ctx, filters=filters) + found_inst.update({inst.uuid: inst.host for inst in instances}) - return found_inst_uuids + return found_inst def _should_enable_custom_max_server_rules(context, rules): @@ -86,7 +97,20 @@ def _should_enable_custom_max_server_rules(context, rules): class ServerGroupController(wsgi.Controller): """The Server group API controller for the OpenStack API.""" - def _format_server_group(self, context, group, req): + def __init__(self, **kwargs): + super(ServerGroupController, self).__init__(**kwargs) + self.compute_api = compute.API() + + def _format_server_group(self, context, group, req, + not_deleted_inst=None): + """Format ServerGroup according to API version. + + Displays only not-deleted members. + + :param:not_deleted_inst: Pre-built dict of instance-uuid: host for + multiple server-groups that are found to be + not deleted. + """ # the id field has its value as the uuid of the server group # There is no 'uuid' key in server_group seen by clients. # In addition, clients see policies as a ["policy-name"] list; @@ -105,7 +129,8 @@ def _format_server_group(self, context, group, req): members = [] if group.members: # Display the instances that are not deleted. - members = _get_not_deleted(context, group.members) + members = list(_get_not_deleted(context, group.members, + not_deleted_inst)) server_group['members'] = members # Add project id information to the response data for # API version v2.13 @@ -149,6 +174,7 @@ def delete(self, req, id): def index(self, req): """Returns a list of server groups.""" context = req.environ['nova.context'] + limit, offset = common.get_limit_and_offset(req) project_id = context.project_id # NOTE(gmann): Using context's project_id as target here so # that when we remove the default target from policy class, @@ -166,13 +192,19 @@ def index(self, req): # Until then, let's keep the old behaviour. context.can(sg_policies.POLICY_ROOT % 'index:all_projects', target={'project_id': project_id}) - sgs = objects.InstanceGroupList.get_all(context) + sgs = objects.InstanceGroupList.get_all(context, + limit=limit, + offset=offset) else: sgs = objects.InstanceGroupList.get_by_project_id( - context, project_id) - limited_list = common.limited(sgs.objects, req) - result = [self._format_server_group(context, group, req) - for group in limited_list] + context, project_id, limit=limit, offset=offset) + + members = list(itertools.chain.from_iterable(sg.members + for sg in sgs + if sg.members)) + not_deleted = _get_not_deleted(context, members) + result = [self._format_server_group(context, group, req, not_deleted) + for group in sgs] return {'server_groups': result} @wsgi.Controller.api_version("2.1") @@ -248,3 +280,140 @@ def create(self, req, body): raise exc.HTTPForbidden(explanation=msg) return {'server_group': self._format_server_group(context, sg, req)} + + @wsgi.Controller.api_version("2.64") + @validation.schema(schema.update) + @wsgi.expected_errors((400, 404)) + def update(self, req, id, body): + """Update a server-group's members + + Striving for idempotency, we accept already removed or already + contained members. + + We always remove first and then check if we can add the requested + members. That way, removing an instance for a host and adding another + one works in one request. + + We do all requested changes or no change. + """ + context = req.environ['nova.context'] + project_id = context.project_id + context.can(sg_policies.POLICY_ROOT % 'update', + target={'project_id': project_id}) + try: + sg = objects.InstanceGroup.get_by_uuid(context, id) + except nova.exception.InstanceGroupNotFound as e: + raise webob.exc.HTTPNotFound(explanation=e.format_message()) + + members_to_remove = set(body.get('remove_members', [])) + members_to_add = set(body.get('add_members', [])) + LOG.info('Called update for server-group %s with add_members: %s and ' + 'remove_members %s', + id, ', '.join(members_to_add), ', '.join(members_to_remove)) + + overlap = members_to_remove & members_to_add + if overlap: + msg = ('Parameters "add_members" and "remove_members" are ' + 'overlapping in {}'.format(', '.join(overlap))) + raise exc.HTTPBadRequest(explanation=msg) + + if not members_to_remove and not members_to_add: + LOG.info("No update requested.") + formatted_sg = self._format_server_group(context, sg, req) + return {'server_group': formatted_sg} + + # don't do work if it's not necessary. we might be able to get a fast + # way out if this request is already fulfilled + members_to_remove = members_to_remove & set(sg.members) + members_to_add = members_to_add - set(sg.members) + + if not members_to_remove and not members_to_add: + LOG.info("State already satisfied.") + formatted_sg = self._format_server_group(context, sg, req) + return {'server_group': formatted_sg} + + # retrieve all the instances to add, failing if one doesn't exist, + # because we need to check the hosts against the policy and adding + # non-existent instances doesn't make sense + members_to_search = members_to_add | members_to_remove + found_instances_hosts = _get_not_deleted(context, members_to_search) + missing_uuids = members_to_add - set(found_instances_hosts) + if missing_uuids: + msg = ("One or more members in add_members cannot be found: {}" + .format(', '.join(missing_uuids))) + raise exc.HTTPBadRequest(explanation=msg) + + # check if (some of) the VMs are already members of another + # instance_group. We cannot support this as they might contradict. + found_server_groups = \ + objects.InstanceGroupList.get_by_instance_uuids(context, + members_to_add) + other_server_groups = [_x.uuid for _x in found_server_groups + if _x.uuid != id] + if other_server_groups: + msg = ("One or more members in add_members is already assigned " + "to another server group. Server groups: {}" + .format(', '.join(other_server_groups))) + raise exc.HTTPBadRequest(explanation=msg) + + # check if the policy is still valid with these changes + if sg.policy in ('affinity', 'anti-affinity'): + current_members_hosts = _get_not_deleted(context, sg.members) + current_hosts = set(h for u, h in current_members_hosts.items() + if u not in members_to_remove) + if sg.policy == 'affinity': + outliers = [u for u, h in found_instances_hosts.items() + if h and h not in current_hosts] + elif sg.policy == 'anti-affinity': + outliers = [u for u, h in found_instances_hosts.items() + if h and h in current_hosts] + else: + outliers = None + LOG.warning('server-group update check not implemented for ' + 'policy %s', sg.policy) + if outliers: + LOG.info('Update of server-group %s with policy %s aborted: ' + 'policy violation by %s', + id, sg.policy, ', '.join(outliers)) + msg = ("Adding instance(s) {} would violate policy '{}'." + .format(', '.join(outliers), sg.policy)) + raise exc.HTTPBadRequest(explanation=msg) + + # update the server group and save it + if members_to_remove: + objects.InstanceGroup.remove_members(context, sg.id, + members_to_remove, sg.uuid) + if members_to_add: + try: + objects.InstanceGroup.add_members(context, id, members_to_add) + except Exception: + LOG.exception('Failed to add members.') + if members_to_remove: + LOG.info('Trying to add removed members again after ' + 'error.') + objects.InstanceGroup.add_members(context, id, + members_to_remove) + raise + + LOG.info("Changed server-group %s in DB.", id) + + # refresh InstanceGroup object, because we changed it directly in the + # DB. + sg.refresh() + + # update the request-specs of the updated members + for member_uuid in found_instances_hosts: + request_spec = \ + objects.RequestSpec.get_by_instance_uuid(context, member_uuid) + if member_uuid in members_to_add: + request_spec.instance_group = sg + else: + request_spec.instance_group = None + request_spec.save() + + # tell the compute hosts about the update, so they can sync if + # necessary + hosts_to_update = set(h for h in found_instances_hosts.values() if h) + self.compute_api.sync_server_group(context, hosts_to_update, id) + + return {'server_group': self._format_server_group(context, sg, req)} diff --git a/nova/bigvm/__init__.py b/nova/bigvm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/nova/bigvm/manager.py b/nova/bigvm/manager.py new file mode 100644 index 00000000000..8a332f118f3 --- /dev/null +++ b/nova/bigvm/manager.py @@ -0,0 +1,729 @@ +# Copyright 2019 SAP SE +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +BigVM service +""" +import itertools + +import os_resource_classes as orc +import os_traits +from oslo_log import log as logging +from oslo_messaging import exceptions as oslo_exceptions +from oslo_service import periodic_task + +import nova.conf +from nova import context as nova_context +from nova import exception +from nova import manager +from nova.objects.aggregate import AggregateList +from nova.objects.cell_mapping import CellMappingList +from nova.objects.compute_node import ComputeNodeList +from nova.objects.host_mapping import HostMappingList +from nova.scheduler.client.report import get_placement_request_id +from nova.scheduler.client.report import NESTED_PROVIDER_API_VERSION +from nova.scheduler.client.report import SchedulerReportClient +from nova.scheduler.utils import ResourceRequest +from nova.utils import BIGVM_EXCLUSIVE_TRAIT +from nova.virt.vmwareapi import special_spawning + +LOG = logging.getLogger(__name__) + +CONF = nova.conf.CONF + +MEMORY_MB = orc.MEMORY_MB +BIGVM_RESOURCE = special_spawning.BIGVM_RESOURCE +BIGVM_DISABLED_TRAIT = 'CUSTOM_BIGVM_DISABLED' +VMWARE_HV_TYPE = 'VMware vCenter Server' +SHARD_PREFIX = 'vc-' +HV_SIZE_BUCKET_THRESHOLD_PERCENT = 10 + + +class BigVmManager(manager.Manager): + """Takes care of the needs of big VMs""" + + def __init__(self, *args, **kwargs): + self.placement_client = SchedulerReportClient() + self.special_spawn_rpc = special_spawning.SpecialVmSpawningInterface() + + super(BigVmManager, self).__init__(service_name='bigvm', + *args, **kwargs) + + @periodic_task.periodic_task(spacing=CONF. + prepare_empty_host_for_spawning_interval, + run_immediately=True) + def _prepare_empty_host_for_spawning(self, context): + """Handle freeing up hosts per hv_size per VC for spawning + + The general workflow is: + 1) Find all hypervisor sizes (hv_size) existing in a vCenter (VC) + within an availability zone (az). + 2) Choose one host per hv_size per VC and create a child resource + provider (rp) for it with a well-known name. This marks that + host as responsible for the hv_size in that VC. The child rp has no + resources, yet. This also triggers the vmware-driver to free up a + host in the cluster. + 3) Either directly or in the next iteration every host without + resources is checked for its status by calling the vmware + driver. If we're done freeing up the host, we add the + BIGVM_RESOURCE to the child rp to make it consumable. + 4) In every iteration, we check for reserved BIGVM_RESOURCEs on a + child rp and if the host is still free. Reserving a resource happens + after consumption, so we know the host is not longer free and have + to clean up the child rp and the vmware part. The host might be used + again if a migration happened in the background. We need to clean up + the child rp in this case and redo the scheduling. + + We only want to fill up clusters to a certain point, configurable via + bigvm_cluster_max_usage_percent. resource-providers having more RAM + usage than this, will not be used for a hv_size - if they are not + exclusively used for HANA flavors.. Additionally, we check in every + iteration, if we have to give up a freed-up host, because the cluster + reached the limit. + """ + client = self.placement_client + + # make sure our custom trait exists + client._ensure_traits(context, [BIGVM_DISABLED_TRAIT, + BIGVM_EXCLUSIVE_TRAIT]) + + vcenters, bigvm_providers, vmware_providers = \ + self._get_providers(context) + + self._check_and_clean_providers(context, client, bigvm_providers, + vmware_providers) + + missing_hv_sizes_per_vc = self._get_missing_hv_sizes(context, + vcenters, bigvm_providers, vmware_providers) + + if not any(missing_hv_sizes_per_vc.values()): + LOG.info('Free host for spawning defined for every ' + 'vCenter and hypervisor-size.') + return + + def _flatten(list_of_lists): + return itertools.chain.from_iterable(list_of_lists) + + # retrieve allocation candidates for all hv sizes. we later have to + # filter them by VC, because our placement doesn't know about VCs. + candidates = {} + for hv_size in set(_flatten(missing_hv_sizes_per_vc.values())): + resources = ResourceRequest() + resources._add_resource(MEMORY_MB, hv_size) + res = client.get_allocation_candidates(context, resources) + if res is None: + continue + alloc_reqs, provider_summaries, allocation_request_version = res + + # filter out providers, that don't match the full host, e.g. don't + # allow 3 TB on a 6 TB host, as we need a fully free host + provider_summaries = {p: d for p, d in provider_summaries.items() + if vmware_providers.get(p, {}).get('hv_size') == hv_size} + + if not provider_summaries: + LOG.warning('Could not find enough resources to free up a ' + 'host for hypervisor size %(hv_size)d.', + {'hv_size': hv_size}) + continue + + # filter out providers that are too full already + filtered_provider_summaries = {} + for p, d in provider_summaries.items(): + # Hosts exclusively used for hana_* flavors cannot be too full + if BIGVM_EXCLUSIVE_TRAIT in d['traits']: + filtered_provider_summaries[p] = d + continue + + used = vmware_providers.get(p, {})\ + .get('memory_mb_used_percent', 100) + if used > CONF.bigvm_cluster_max_usage_percent: + continue + filtered_provider_summaries[p] = d + + if not filtered_provider_summaries: + LOG.warning('Could not find a resource-provider to free up a ' + 'host for hypervisor size %(hv_size)d, because ' + 'all clusters are used more than %(max_used)d.', + {'hv_size': hv_size, + 'max_used': CONF.bigvm_cluster_max_usage_percent}) + continue + + # filter out providers that are disabled in general or for bigVMs + # specifically + provider_summaries = filtered_provider_summaries + filtered_provider_summaries = {} + for p, d in provider_summaries.items(): + if os_traits.COMPUTE_STATUS_DISABLED in d['traits'] \ + or BIGVM_DISABLED_TRAIT in d['traits']: + continue + filtered_provider_summaries[p] = d + + if not filtered_provider_summaries: + LOG.warning('Could not find a resource-provider to free up a ' + 'host for hypervisor size %(hv_size)d, because ' + 'all providers with enough space are disabled.', + {'hv_size': hv_size}) + continue + + candidates[hv_size] = (alloc_reqs, filtered_provider_summaries) + + for vc in vcenters: + for hv_size in missing_hv_sizes_per_vc[vc]: + if hv_size not in candidates: + LOG.warning('Could not find a resource-provider to free ' + 'up a host for hypervisor size %(hv_size)d in ' + '%(vc)s.', + {'hv_size': hv_size, 'vc': vc}) + continue + alloc_reqs, provider_summaries = candidates[hv_size] + + # filter providers by VC, as placement returned all matching + # providers + providers = {p: d for p, d in provider_summaries.items() + if vmware_providers.get(p, {}).get('vc') == vc} + + # select the one with the least usage + def _free_memory(p): + memory = providers[p]['resources'][MEMORY_MB] + return memory['capacity'] - memory['used'] + + provider_uuids = sorted((p for p in providers), + key=_free_memory, reverse=True) + + try: + for rp_uuid in provider_uuids: + host = vmware_providers[rp_uuid]['host'] + cm = vmware_providers[rp_uuid]['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + if self._free_host_for_provider(cctxt, rp_uuid, + host): + break + except oslo_exceptions.MessagingTimeout as e: + # we don't know if the timeout happened after we started + # freeing a host already or because we couldn't reach the + # nova-compute node. Therefore, we move on to the next HV + # size for that VC and hope the timeout resolves for the + # next run. + LOG.exception(e) + LOG.warning('Skipping HV size %(hv_size)s in VC %(vc)s ' + 'because of error', + {'hv_size': hv_size, 'vc': vc}) + + def _get_providers(self, context): + """Return our special and the basic vmware resource-providers + + This returns a list of vcenters and two dicts, where the + resource-provider uuid is the key. The value contains a dict with the + important information for each resource-provider, like host, az, vc and + cell_mapping + either the hypervisor size (vmware provider) or the + resource-provider dict (special provider). + """ + client = self.placement_client + + vmware_hvs = {} + for cm in CellMappingList.get_all(context): + with nova_context.target_cell(context, cm) as cctxt: + vmware_hvs.update({cn.uuid: cn.host for cn in + ComputeNodeList.get_by_hypervisor_type(cctxt, + VMWARE_HV_TYPE) + if not cn.deleted}) + + host_azs = {} + host_vcs = {} + for agg in AggregateList.get_all(context): + if not agg.availability_zone: + continue + + if agg.name == agg.availability_zone: + for host in agg.hosts: + host_azs[host] = agg.name + elif agg.name.startswith(SHARD_PREFIX): + for host in agg.hosts: + host_vcs[host] = agg.name + + vcenters = set(host_vcs.values()) + + host_mappings = {hm.host: hm.cell_mapping + for hm in HostMappingList.get_all(context)} + + # find all resource-providers that we added and also a list of vmware + # resource-providers + bigvm_providers = {} + vmware_providers = {} + resp = client.get('/resource_providers', + version=NESTED_PROVIDER_API_VERSION) + for rp in resp.json()['resource_providers']: + if rp['name'].startswith(CONF.bigvm_deployment_rp_name_prefix): + # We use the _root_ RP and not the parent because that will + # always map to the ComputeNode, even if we decide to nest RPs + # further. + host_rp_uuid = rp['root_provider_uuid'] + host = vmware_hvs[host_rp_uuid] + cell_mapping = host_mappings[host] + bigvm_providers[rp['uuid']] = {'rp': rp, + 'host': host, + 'az': host_azs[host], + 'vc': host_vcs[host], + 'cell_mapping': cell_mapping, + 'host_rp_uuid': host_rp_uuid} + elif rp['uuid'] not in vmware_hvs: # ignore baremetal + continue + else: + # retrieve the MEMORY_MB resource + url = '/resource_providers/{}/inventories/{}'.format( + rp['uuid'], MEMORY_MB) + resp = client.get(url) + if resp.status_code != 200: + LOG.error('Could not retrieve inventory for RP %(rp)s.', + {'rp': rp['uuid']}) + continue + memory_mb_inventory = resp.json() + + # retrieve the usage + url = '/resource_providers/{}/usages' + resp = client.get(url.format(rp['uuid'])) + if resp.status_code != 200: + LOG.error('Could not retrieve usages for RP %(rp)s.', + {'rp': rp['uuid']}) + continue + usages = resp.json()['usages'] + + hv_size = memory_mb_inventory['max_unit'] + memory_mb_total = (memory_mb_inventory['total'] - + memory_mb_inventory['reserved']) + memory_mb_used_percent = (usages[MEMORY_MB] / float( + memory_mb_total) * 100) + + host = vmware_hvs[rp['uuid']] + # ignore hypervisors we would never use anyways + if hv_size < CONF.bigvm_mb: + LOG.debug('Ignoring %(host)s (%(hv_size)s < %(bigvm_mb)s)', + {'host': host, 'hv_size': hv_size, + 'bigvm_mb': CONF.bigvm_mb}) + continue + + cell_mapping = host_mappings[host] + if host not in host_azs or host not in host_vcs: + # seen this happening during buildup + LOG.debug('Ignoring %(host)s as it is not assigned to an ' + 'AZ or VC.', + {'host': host}) + continue + + # retrieve traits so we can find disabled and hana exclusive + # hosts + traits = client.get_provider_traits(context, rp['uuid']) + vmware_providers[rp['uuid']] = { + 'hv_size': hv_size, + 'host': host, + 'az': host_azs[host], + 'vc': host_vcs[host], + 'cell_mapping': cell_mapping, + 'traits': traits, + 'memory_mb_used_percent': memory_mb_used_percent} + + # make sure the placement cache is filled + client.get_provider_tree_and_ensure_root(context, rp['uuid'], + rp['name']) + + # retrieve all bigvm provider's inventories + for rp_uuid, rp in bigvm_providers.items(): + inventory = client._get_inventory(context, rp_uuid) + rp['inventory'] = inventory['inventories'] if inventory else {} + + # make sure grouping by hv_size works properly later on, even if there + # are marginal differences in the reported hv_size. we need to have all + # vmware_providers to have the same hv_size if they're in the same + # "bucket", e.g. they're supposed to be 3 TB HVs. To make sure the + # placement query for allocation-candidates works, we use the smallest + # size and assign it to all in the same bucket. + hv_size_bucket = None + for rp_uuid, rp in sorted(vmware_providers.items(), + key=lambda x: x[1]['hv_size']): + if hv_size_bucket is None: + # first one is always a new bucket + hv_size_bucket = rp['hv_size'] + continue + + threshold = HV_SIZE_BUCKET_THRESHOLD_PERCENT * hv_size_bucket / 100 + if rp['hv_size'] - hv_size_bucket > threshold: + # set key if the difference to the last key is over the + # threshold + hv_size_bucket = rp['hv_size'] + + rp['hv_size'] = hv_size_bucket + + return (vcenters, bigvm_providers, vmware_providers) + + def _check_and_clean_providers(self, context, client, bigvm_providers, + vmware_providers): + + # check for reserved resources which indicate that the free host was + # consumed + providers_to_delete = {rp_uuid: rp + for rp_uuid, rp in bigvm_providers.items() + if rp['inventory'].get(BIGVM_RESOURCE, {}) + .get('reserved')} + + # check if we don't have a valid vmware provider for it (anymore) and + # thus cannot be a valid provider ourselves + providers_to_delete.update({ + rp_uuid: rp for rp_uuid, rp in bigvm_providers.items() + if rp_uuid not in providers_to_delete and + rp['host_rp_uuid'] not in vmware_providers}) + + # check for resource-providers having more than + # bigvm_cluster_max_usage_percent usage + for rp_uuid, rp in bigvm_providers.items(): + if rp_uuid in providers_to_delete: + # no need to check if we already remove it anyways + continue + + host_rp = vmware_providers[rp['host_rp_uuid']] + # Hosts exclusively used for hana_* flavors cannot be too full + if BIGVM_EXCLUSIVE_TRAIT in host_rp['traits']: + continue + + used_percent = host_rp['memory_mb_used_percent'] + if used_percent > CONF.bigvm_cluster_max_usage_percent: + providers_to_delete[rp_uuid] = rp + LOG.info('Resource-provider %(host_rp_uuid)s with free host ' + 'is overused. Marking %(rp_uuid)s for deletion.', + {'host_rp_uuid': rp['host_rp_uuid'], + 'rp_uuid': rp_uuid}) + + # check if a provider got used in the background without our knowledge + for rp_uuid, rp in bigvm_providers.items(): + if rp_uuid in providers_to_delete: + # no need to check if we already remove it anyways + continue + + # if we have no resources on the resource-provider, we don't expect + # it to be free, yet + if not rp['inventory'].get(BIGVM_RESOURCE, {}): + continue + + # ask the compute-node if the host is still free. anything other + # than FREE_HOST_STATE_DONE means we've got an unexpected state and + # should re-schedule that size + cm = rp['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + state = self.special_spawn_rpc.free_host(cctxt, rp['host']) + if state != special_spawning.FREE_HOST_STATE_DONE: + LOG.info('Checking on already freed up host %(host)s ' + 'returned with state %(state)s. Marking ' + '%(rp_uuid)s for deletion.', + {'host': rp['host'], + 'state': state, + 'rp_uuid': rp_uuid}) + providers_to_delete[rp_uuid] = rp + + # check if a provider was disabled by now + for rp_uuid, rp in bigvm_providers.items(): + if rp_uuid in providers_to_delete: + # no need to check if we already remove it anyways + continue + + host_rp = vmware_providers[rp['host_rp_uuid']] + if os_traits.COMPUTE_STATUS_DISABLED in host_rp['traits'] \ + or BIGVM_DISABLED_TRAIT in host_rp['traits']: + providers_to_delete[rp_uuid] = rp + LOG.info('Resource-provider %(host_rp_uuid)s got disabled in' + ' general or specifically for bigVMs. Marking' + ' %(rp_uuid)s for deletion.', + {'host_rp_uuid': rp['host_rp_uuid'], + 'rp_uuid': rp_uuid}) + + for rp_uuid, rp in providers_to_delete.items(): + self._clean_up_consumed_provider(context, rp_uuid, rp) + + # clean up our list of resource-providers from consumed or overused + # hosts + for rp_uuid in providers_to_delete: + del bigvm_providers[rp_uuid] + + def _get_allocations_for_consumer(self, context, consumer_uuid): + """Same as SchedulerReportClient.get_allocations_for_consumer() but + includes user_id and project_id in the returned values, by doing the + request with a newer version. + """ + client = self.placement_client + url = '/allocations/%s' % consumer_uuid + resp = client.get(url, global_request_id=context.global_id, + version=1.17) + if not resp: + return {} + else: + return resp.json() + + def _remove_provider_from_consumer_allocations(self, context, + consumer_uuid, rp_uuid): + """This is basically the same as + SchedulerClient.remove_provider_from_instance_allocation, but without + the resize-on-same-host detection - it simply removes the provider from + the allocations of the consumer. + """ + client = self.placement_client + + # get the allocation details, because we need user_id and + # project_id to call the delete function + current_allocs = \ + self._get_allocations_for_consumer(context, consumer_uuid) + + LOG.debug('Found the following allocations for consumer ' + '%(consumer_uuid)s: %(allocations)s', + {'consumer_uuid': consumer_uuid, + 'allocations': current_allocs}) + + new_allocs = [ + { + 'resource_provider': { + 'uuid': alloc_rp_uuid, + }, + 'resources': alloc['resources'], + } + for alloc_rp_uuid, alloc in current_allocs['allocations'].items() + if alloc_rp_uuid != rp_uuid + ] + payload = {'allocations': new_allocs, + 'project_id': current_allocs['project_id'], + 'user_id': current_allocs['user_id']} + LOG.debug("Sending updated allocation %s for instance %s after " + "removing resources for %s.", + new_allocs, consumer_uuid, rp_uuid) + url = '/allocations/%s' % consumer_uuid + r = client.put(url, payload, version='1.10', + global_request_id=context.global_id) + if r.status_code != 204: + LOG.warning("Failed to save allocation for %s. Got HTTP %s: %s", + consumer_uuid, r.status_code, r.text) + return r.status_code == 204 + + def _clean_up_consumed_provider(self, context, rp_uuid, rp): + """Clean up after a resource-provider was consumed + + We need to remove all allocations, the resource-provider itself and + also the hostgroup from the vCenter. + """ + client = self.placement_client + + # find the consumer + allocations = client.get_allocations_for_resource_provider( + context, rp_uuid).allocations + # we might have already deleted them and got killed or the VM got + # deleted in the mean time + failures = 0 + for consumer_uuid, resources in allocations.items(): + # delete the allocations + # we can't use + # SchedulerClient.remove_provider_from_instance_allocation here, as + # this would detect a resize and try to remove the resources, but + # keep an allocation. We need the specific allocation for our + # resource-provider removed. + if self._remove_provider_from_consumer_allocations(context, + consumer_uuid, rp_uuid): + LOG.info('Removed bigvm allocations for %(consumer_uuid)s ' + 'from RP', {'consumer_uuid': consumer_uuid}) + else: + LOG.error('Could not remove bigvm allocations for ' + '%(consumer_uuid)s corresponding RP.', + {'consumer_uuid': consumer_uuid}) + failures += 1 + + if failures: + # skip removing the resource-provider because we couldn't + # remove all allocations. we'll retry on the next run + LOG.warning('Skippping removal of resource-provider ' + '%(rp_uuid)s as we could not remove some ' + 'allocations.', + {'rp_uuid': rp_uuid}) + return + + # remove the hostgroup from the host + cm = rp['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + if not self.special_spawn_rpc.remove_host_from_hostgroup(cctxt, + rp['host']): + LOG.warning('Skipping removal of resource-provider ' + '%(rp_uuid)s as we could not remove the hostgroup ' + 'from the vCenter.', + {'rp_uuid': rp_uuid}) + return + + # delete the resource-provider + client._delete_provider(rp_uuid) + LOG.info('Removed resource-provider %(rp_uuid)s.', + {'rp_uuid': rp_uuid}) + + def _get_missing_hv_sizes(self, context, vcenters, + bigvm_providers, vmware_providers): + """Search and return hypervisor sizes having no freed-up host + + Returns a dict containing a set of hv sizes missing a freed-up host for + each vCenter. + """ + found_hv_sizes_per_vc = {vc: set() for vc in vcenters} + + for rp_uuid, rp in bigvm_providers.items(): + host_rp_uuid = rp['host_rp_uuid'] + hv_size = vmware_providers[host_rp_uuid]['hv_size'] + found_hv_sizes_per_vc[rp['vc']].add(hv_size) + + # if there are no resources in that resource-provider, it means, + # that we started freeing up a host. We have to check the process + # state and add the resources once it's done. + if not rp['inventory'].get(BIGVM_RESOURCE): + cm = rp['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + state = self.special_spawn_rpc.free_host(cctxt, rp['host']) + + if state == special_spawning.FREE_HOST_STATE_DONE: + self._add_resources_to_provider(context, rp_uuid, rp) + elif state == special_spawning.FREE_HOST_STATE_ERROR: + LOG.warning('Freeing a host for spawning failed on ' + '%(host)s.', + {'host': rp['host']}) + # do some cleanup, so another compute-node is used + found_hv_sizes_per_vc[rp['vc']].remove(hv_size) + self._clean_up_consumed_provider(context, rp_uuid, rp) + else: + LOG.info('Waiting for host on %(host)s to free up.', + {'host': rp['host']}) + + hv_sizes_per_vc = { + vc: set(rp['hv_size'] for rp in vmware_providers.values() + if rp['vc'] == vc) + for vc in vcenters} + + missing_hv_sizes_per_vc = { + vc: hv_sizes_per_vc[vc] - found_hv_sizes_per_vc[vc] + for vc in vcenters} + + return missing_hv_sizes_per_vc + + def _add_resources_to_provider(self, context, rp_uuid, rp): + """Add our custom resources to the provider so they can be consumed. + + This should be called once the host is freed up in the cluster. + """ + client = self.placement_client + inv_data = {BIGVM_RESOURCE: { + # we use 2 here so we can reserve 1 later. in queens we can't + # reserve $total + 'max_unit': 2, 'min_unit': 2, 'total': 2}} + + client._ensure_resource_provider( + context, rp_uuid, rp['rp']['name'], + parent_provider_uuid=rp['host_rp_uuid']) + try: + client.set_inventory_for_provider(context, rp_uuid, inv_data) + except Exception as err: + LOG.error('Adding inventory to the resource-provider for ' + 'spawning on %(host)s failed: %(err)s', + {'host': rp['host'], 'err': err}) + else: + LOG.info('Added inventory to the resource-provider for spawning ' + 'on %(host)s.', + {'host': rp['host']}) + + def _free_host_for_provider(self, context, rp_uuid, host): + """Takes care of creating a child resource provider in placement to + "claim" a resource-provider/host for freeing up a host. Then calls the + driver to actually free up the host in the cluster. + """ + client = self.placement_client + needs_cleanup = True + new_rp_uuid = None + try: + # TODO(jkulik) try to reserve the necessary memory for freeing a + # full hypervisor + + # create a child resource-provider + new_rp_name = '{}-{}'.format(CONF.bigvm_deployment_rp_name_prefix, + host) + # this is basically copied from placement client, but we don't want + # to set the uuid manually which it doesn't support + url = "/resource_providers" + payload = { + 'name': new_rp_name, + 'parent_provider_uuid': rp_uuid + } + resp = client.post(url, payload, + version=NESTED_PROVIDER_API_VERSION, + global_request_id=context.global_id) + placement_req_id = get_placement_request_id(resp) + if resp.status_code == 201: + new_rp_uuid = resp.headers['Location'].split('/')[-1] + msg = ("[%(placement_req_id)s] Created resource provider " + "record via placement API for host %(host)s for " + "special spawning: %(rp_uuid)s") + args = { + 'host': host, + 'placement_req_id': placement_req_id, + 'rp_uuid': new_rp_uuid + } + LOG.info(msg, args) + else: + msg = ("[%(placement_req_id)s] Failed to create resource " + "provider record in placement API for %(host)s for " + "special spawning. Got %(status_code)d: %(err_text)s.") + args = { + 'host': host, + 'status_code': resp.status_code, + 'err_text': resp.text, + 'placement_req_id': placement_req_id, + } + LOG.error(msg, args) + raise exception.ResourceProviderCreationFailed( + name=new_rp_name) + # make sure the placement cache is filled + client.get_provider_tree_and_ensure_root(context, new_rp_uuid, + new_rp_name) + + # Remove the following once elektra doesn't look at aggregates + # anymore to show spawnable bigvm flavors: + + # ensure the parent resource-provider has its uuid as aggregate set + # in addition to its previous aggregates + agg_info = client._get_provider_aggregates(context, rp_uuid) + if rp_uuid not in agg_info.aggregates: + agg_info.aggregates.add(rp_uuid) + client.set_aggregates_for_provider( + context, rp_uuid, agg_info.aggregates, + generation=agg_info.generation) + # add the newly-created resource-provider to the parent uuid's + # aggregate + client.set_aggregates_for_provider(context, new_rp_uuid, [rp_uuid]) + # Remove until here after elektra change. + + # find a host and let DRS free it up + state = self.special_spawn_rpc.free_host(context, host) + + if state == special_spawning.FREE_HOST_STATE_DONE: + # there were free resources available immediately + needs_cleanup = False + new_rp = {'host': host, + 'rp': {'name': new_rp_name}, + 'host_rp_uuid': rp_uuid} + self._add_resources_to_provider(context, new_rp_uuid, new_rp) + elif state == special_spawning.FREE_HOST_STATE_STARTED: + # it started working on it. we have to check back later + # if it's done + needs_cleanup = False + finally: + # clean up placement, if something went wrong + if needs_cleanup and new_rp_uuid is not None: + client._delete_provider(new_rp_uuid) + + return not needs_cleanup diff --git a/nova/cmd/bigvm.py b/nova/cmd/bigvm.py new file mode 100644 index 00000000000..0d39516d4d8 --- /dev/null +++ b/nova/cmd/bigvm.py @@ -0,0 +1,44 @@ +# Copyright 2019 SAP SE +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Starter script for BigVM manager.""" + +import sys + +from oslo_log import log as logging +from oslo_reports import guru_meditation_report as gmr +from oslo_reports import opts as gmr_opts + +import nova.conf +from nova import config +from nova import objects +from nova import service +from nova import version + +CONF = nova.conf.CONF + + +def main(): + config.parse_args(sys.argv) + logging.setup(CONF, "nova") + objects.register_all() + gmr_opts.set_defaults(CONF) + objects.Service.enable_min_version_cache() + + gmr.TextGuruMeditation.setup_autorun(version, conf=CONF) + + server = service.Service.create(binary='nova-bigvm') + service.serve(server) + service.wait() diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index 5a00a3c6a40..a9b73dfe46f 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -2377,8 +2377,11 @@ def heal_allocations(self, max_count=None, verbose=False, dry_run=False, return 0 @staticmethod - def _get_rp_uuid_for_host(ctxt, host): - """Finds the resource provider (compute node) UUID for the given host. + def _get_rp_uuids_for_host(ctxt, host): + """Finds the resource provider UUID(s) for the given host. + + This is usually a single value, but we return a list as we must support + Ironic nodes, that have the same host but different node uuids. :param ctxt: cell-targeted nova RequestContext :param host: name of the compute host @@ -2408,12 +2411,13 @@ def _get_rp_uuid_for_host(ctxt, host): nodes = objects.ComputeNodeList.get_all_by_host( cctxt, host) - if len(nodes) > 1: - # This shouldn't happen, so we need to bail since we - # won't know which node to use. + if len(nodes) > 1 and 'ironic' not in host: + # Multiple nodes for non-ironic hosts shouldn't happen, so + # we need to bail since we won't know which node to use. raise exception.TooManyComputesForHost( num_computes=len(nodes), host=host) - return nodes[0].uuid + + return [node.uuid for node in nodes] @action_description( _("Mirrors compute host aggregates to resource provider aggregates " @@ -2466,7 +2470,7 @@ def sync_aggregates(self, verbose=False): # Since hosts can be in more than one aggregate, keep track of the host # to its corresponding resource provider uuid to avoid redundant # lookups. - host_to_rp_uuid = {} + host_to_rp_uuids = {} unmapped_hosts = set() # keep track of any missing host mappings computes_not_found = set() # keep track of missing nodes providers_not_found = {} # map of hostname to missing provider uuid @@ -2474,18 +2478,24 @@ def sync_aggregates(self, verbose=False): output(_('Processing aggregate: %s') % aggregate.name) for host in aggregate.hosts: output(_('Processing host: %s') % host) - rp_uuid = host_to_rp_uuid.get(host) - if not rp_uuid: + rp_uuids = host_to_rp_uuids.get(host) + if not rp_uuids: try: - rp_uuid = self._get_rp_uuid_for_host(ctxt, host) - host_to_rp_uuid[host] = rp_uuid + rp_uuids = self._get_rp_uuids_for_host(ctxt, host) + host_to_rp_uuids[host] = rp_uuids except exception.HostMappingNotFound: # Don't fail on this now, we can dump it at the end. unmapped_hosts.add(host) continue except exception.ComputeHostNotFound: - # Don't fail on this now, we can dump it at the end. - computes_not_found.add(host) + # Don't fail on this now, we can dump it at the end. We + # only need to dump it, if it's a non-ironic host + # though, as for ironic there can be hosts that + # currently don't have any node assigned e.g. + # conductor-groups that are only used for + # testing/during buildup. + if 'ironic' not in host: + computes_not_found.add(host) continue except exception.TooManyComputesForHost as e: # TODO(mriedem): Should we treat this like the other @@ -2497,30 +2507,33 @@ def sync_aggregates(self, verbose=False): # the matching resource provider, found via compute node uuid, # is in the same aggregate in placement, found via aggregate # uuid. - try: - placement.aggregate_add_host(ctxt, aggregate.uuid, - rp_uuid=rp_uuid) - output(_('Successfully added host (%(host)s) and ' - 'provider (%(provider)s) to aggregate ' - '(%(aggregate)s).') % - {'host': host, 'provider': rp_uuid, - 'aggregate': aggregate.uuid}) - except exception.ResourceProviderNotFound: - # The resource provider wasn't found. Store this for later. - providers_not_found[host] = rp_uuid - except exception.ResourceProviderAggregateRetrievalFailed as e: - print(e.message) - return 2 - except exception.NovaException as e: - # The exception message is too generic in this case - print(_('Failed updating provider aggregates for ' - 'host (%(host)s), provider (%(provider)s) ' - 'and aggregate (%(aggregate)s). Error: ' - '%(error)s') % - {'host': host, 'provider': rp_uuid, - 'aggregate': aggregate.uuid, - 'error': e.message}) - return 3 + for rp_uuid in rp_uuids: + try: + placement.aggregate_add_host(ctxt, aggregate.uuid, + rp_uuid=rp_uuid) + output(_('Successfully added host (%(host)s) and ' + 'provider (%(provider)s) to aggregate ' + '(%(aggregate)s).') % + {'host': host, 'provider': rp_uuid, + 'aggregate': aggregate.uuid}) + except exception.ResourceProviderNotFound: + # The resource provider wasn't found. Store this for + # later. + providers_not_found[host] = rp_uuid + except (exception. + ResourceProviderAggregateRetrievalFailed) as e: + print(e.message) + return 2 + except exception.NovaException as e: + # The exception message is too generic in this case + print(_('Failed updating provider aggregates for ' + 'host (%(host)s), provider (%(provider)s) ' + 'and aggregate (%(aggregate)s). Error: ' + '%(error)s') % + {'host': host, 'provider': rp_uuid, + 'aggregate': aggregate.uuid, + 'error': e.message}) + return 3 # Now do our error handling. Note that there is no real priority on # the error code we return. We want to dump all of the issues we hit diff --git a/nova/compute/api.py b/nova/compute/api.py index 6f2f8a3c77b..549fe380d32 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -6098,6 +6098,10 @@ def deny_share(self, context, instance, share_mapping): self.compute_rpcapi.deny_share( context, instance, share_mapping) + def sync_server_group(self, context, hosts, sg_uuid): + for host in hosts: + self.compute_rpcapi.sync_server_group(context, host, sg_uuid) + def target_host_cell(fn): """Target a host-based function to a cell. diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 60cbdee3931..ae724e79ca7 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -32,6 +32,7 @@ import functools import inspect import math +import random import sys import time import traceback @@ -1949,7 +1950,10 @@ def _do_validation(context, instance, group): max_server = rules['max_server_per_host'] else: max_server = 1 - if len(members_on_host) >= max_server: + resource_scheduling = self.driver.capabilities.get( + "resource_scheduling", False) + if (len(members_on_host) >= max_server and + not resource_scheduling): raise exception.GroupAffinityViolation( instance_uuid=instance.uuid, policy='Anti-affinity') @@ -2147,8 +2151,11 @@ def _get_device_name_for_instance(self, instance, bdms, block_device_obj): return self.driver.get_device_name_for_instance( instance, bdms, block_device_obj) except NotImplementedError: - return compute_utils.get_device_name_for_instance( - instance, bdms, block_device_obj.get("device_name")) + @utils.synchronized(instance.uuid + "-bdms") + def _do_get_device_name_for_instance(): + return compute_utils.get_device_name_for_instance( + instance, bdms, block_device_obj.get("device_name")) + return _do_get_device_name_for_instance() def _default_block_device_names(self, instance, image_meta, block_devices): """Verify that all the devices have the device_name set. If not, @@ -8048,7 +8055,22 @@ def reserve_block_device_name(self, context, instance, device, raise exception.MultiattachNotSupportedByVirtDriver( volume_id=volume_id) - @utils.synchronized(instance.uuid) + # There is only one driver, which calls out to the hypervisor for that, + # so we need to lock the instance against manipulations + # Otherwise, we can do our own logic and have to synchronise that + # Since we do NOT hold a lock on instance level, we need to introduce + # one on the instance bdm level in _get_device_name_for_instance, so + # that we are protected against concurrency issues with other calls to + # that function + driver_specific_device_name = self.driver.capabilities.get( + "driver_specific_device_name", False) + if driver_specific_device_name: + synchronized = utils.synchronized(instance.uuid) + else: + def synchronized(f): + return f + + @synchronized def do_reserve(): bdms = ( objects.BlockDeviceMappingList.get_by_instance_uuid( @@ -11005,6 +11027,8 @@ def query_driver_power_state_and_sync(): self._query_driver_power_state_and_sync(context, db_instance) try: + greenthread.sleep(random.randint( + 1, CONF.sync_power_state_interval)) query_driver_power_state_and_sync() except Exception: LOG.exception("Periodic sync_power_state task had an " @@ -12208,6 +12232,10 @@ def _update_migrate_vifs_profile_with_pci(self, "%(profile)s", {'port_id': port_id, 'profile': profile}) + def sync_server_group(self, context, sg_uuid): + """Calls the driver to update the server-group in the backend""" + self.driver.sync_server_group(context, sg_uuid) + # TODO(sbauza): Remove this proxy class in the X release once we drop the 5.x # support. diff --git a/nova/compute/resource_tracker.py b/nova/compute/resource_tracker.py index 57a3a4ec5e5..53b7d760046 100644 --- a/nova/compute/resource_tracker.py +++ b/nova/compute/resource_tracker.py @@ -39,6 +39,7 @@ import nova.conf from nova import exception from nova.i18n import _ +from nova import metrics from nova import objects from nova.objects import base as obj_base from nova.objects import fields @@ -846,6 +847,20 @@ def _copy_resources(self, compute_node, resources, initial=False): if conf_alloc_ratio not in (0.0, None): setattr(compute_node, attr, conf_alloc_ratio) + # apply the reservations so that limes's quota can see them + resources = copy.deepcopy(resources) + resources['vcpus'] = resources.get('vcpus', 0) - \ + resources.get('vcpus_reserved', + CONF.reserved_host_cpus) + resources['vcpus'] = max(resources['vcpus'], 0) + resources['memory_mb'] = resources.get('memory_mb', 0) - \ + resources.get('memory_mb_reserved', + CONF.reserved_host_memory_mb) + resources['memory_mb'] = max(resources['memory_mb'], 0) + resources['local_gb'] = resources.get('local_gb', 0) - \ + CONF.reserved_host_disk_mb / 1024 + resources['local_gb'] = max(resources['local_gb'], 0) + # now copy rest to compute_node compute_node.update_from_virt_driver(resources) @@ -1606,7 +1621,7 @@ def _update_usage_from_migrations(self, context, migrations, nodename): continue def _update_usage_from_instance(self, context, instance, nodename, - is_removed=False): + is_removed=False, bdms=None): """Update usage for a single instance.""" uuid = instance['uuid'] @@ -1638,7 +1653,8 @@ def _update_usage_from_instance(self, context, instance, nodename, instance, sign=sign) # new instance, update compute node resource usage: - self._update_usage(self._get_usage_dict(instance, instance), + self._update_usage(self._get_usage_dict(instance, instance, + bdms=bdms), nodename, sign=sign) # Stop tracking removed instances in the is_bfv cache. This needs to @@ -1664,20 +1680,29 @@ def _update_usage_from_instances(self, context, instances, nodename): cn = self.compute_nodes[nodename] # set some initial values, reserve room for host/hypervisor: - cn.local_gb_used = CONF.reserved_host_disk_mb / 1024 - cn.memory_mb_used = CONF.reserved_host_memory_mb - cn.vcpus_used = CONF.reserved_host_cpus + cn.local_gb_used = 0 + cn.memory_mb_used = 0 + cn.vcpus_used = 0 cn.free_ram_mb = (cn.memory_mb - cn.memory_mb_used) cn.free_disk_gb = (cn.local_gb - cn.local_gb_used) cn.current_workload = 0 cn.running_vms = 0 instance_by_uuid = {} + for instance in instances: + instance_by_uuid[instance.uuid] = instance + + bdms_by_instance_uuid = ( + objects.BlockDeviceMappingList.bdms_by_instance_uuid( + context, instance_by_uuid.keys())) + for instance in instances: if not vm_states.allow_resource_removal( vm_state=instance['vm_state'], task_state=instance.task_state): - self._update_usage_from_instance(context, instance, nodename) + bdms = bdms_by_instance_uuid.get(instance.uuid) + self._update_usage_from_instance(context, instance, nodename, + bdms=bdms) instance_by_uuid[instance.uuid] = instance return instance_by_uuid @@ -1705,6 +1730,7 @@ def _remove_deleted_instances_allocations(self, context, cn, # the (potentially expensive) context.elevated construction below. return read_deleted_context = context.elevated(read_deleted='yes') + orphan_allocations = 0 for consumer_uuid, alloc in allocations.items(): if consumer_uuid in self.tracked_instances: LOG.debug("Instance %s actively managed on this compute host " @@ -1737,6 +1763,7 @@ def _remove_deleted_instances_allocations(self, context, cn, "compute host but is not found in the database.", {'uuid': instance_uuid}, exc_info=False) + orphan_allocations += 1 continue # NOTE(mriedem): A cross-cell migration will work with instance @@ -1814,6 +1841,7 @@ def _remove_deleted_instances_allocations(self, context, cn, "the source host that might need to be removed: " "%s.", instance_uuid, instance.host, instance.node, alloc) + metrics.gauge("allocation.orphans", orphan_allocations) def delete_allocation_for_evacuated_instance(self, context, instance, node, node_type='source'): @@ -1850,7 +1878,7 @@ def _get_flavor(self, instance, prefix, migration): # them. In that case - just get the instance flavor. return instance.flavor - def _get_usage_dict(self, object_or_dict, instance, **updates): + def _get_usage_dict(self, object_or_dict, instance, bdms=None, **updates): """Make a usage dict _update methods expect. Accepts a dict or an Instance or Flavor object, and a set of updates. @@ -1860,6 +1888,7 @@ def _get_usage_dict(self, object_or_dict, instance, **updates): :param instance: nova.objects.Instance for the related operation; this is needed to determine if the instance is volume-backed + :param bdms: Optional a BlockDeviceMappingList for the instance :param updates: key-value pairs to update the passed object. Currently only considers 'numa_topology', all other keys are ignored. @@ -1874,7 +1903,7 @@ def _is_bfv(): is_bfv = self.is_bfv[instance.uuid] else: is_bfv = compute_utils.is_volume_backed_instance( - instance._context, instance) + instance._context, instance, bdms=bdms) self.is_bfv[instance.uuid] = is_bfv return is_bfv diff --git a/nova/compute/rpcapi.py b/nova/compute/rpcapi.py index 348797b63d2..c09a9a711e0 100644 --- a/nova/compute/rpcapi.py +++ b/nova/compute/rpcapi.py @@ -1583,3 +1583,9 @@ def cache_images(self, ctxt, host, image_ids): call_monitor_timeout=CONF.rpc_response_timeout, timeout=CONF.long_rpc_timeout) return cctxt.call(ctxt, 'cache_images', image_ids=image_ids) + + def sync_server_group(self, ctxt, host, sg_uuid): + version = '5.0' + client = self.router.client(ctxt) + cctxt = client.prepare(server=host, version=version) + return cctxt.cast(ctxt, "sync_server_group", sg_uuid=sg_uuid) diff --git a/nova/compute/utils.py b/nova/compute/utils.py index fee66d85c5c..ee25341ee2e 100644 --- a/nova/compute/utils.py +++ b/nova/compute/utils.py @@ -899,6 +899,21 @@ def notify_about_server_group_add_member(context, group_id): notification.emit(context) +@rpc.if_notifications_enabled +def notify_about_server_group_remove_member(context, group_id): + group = objects.InstanceGroup.get_by_uuid(context, group_id) + payload = sg_notification.ServerGroupPayload(group) + notification = sg_notification.ServerGroupNotification( + priority=fields.NotificationPriority.INFO, + publisher=notification_base.NotificationPublisher( + host=CONF.host, source=fields.NotificationSource.API), + event_type=notification_base.EventType( + object='server_group', + action=fields.NotificationAction.REMOVE_MEMBER), + payload=payload) + notification.emit(context) + + @rpc.if_notifications_enabled def notify_about_instance_rebuild(context, instance, host, action=fields.NotificationAction.REBUILD, diff --git a/nova/conductor/manager.py b/nova/conductor/manager.py index 32b2fe8ec2a..b7a86c10967 100644 --- a/nova/conductor/manager.py +++ b/nova/conductor/manager.py @@ -348,6 +348,14 @@ def _get_request_spec_for_cold_migrate(context, instance, flavor, sh = filter_properties.setdefault('scheduler_hints', {}) sh.update(new_scheduler_hints) + volume_sizes = [bdm.volume_size for bdm in instance.get_bdms() + if bdm.destination_type == 'volume'] + sh = filter_properties.setdefault('scheduler_hints', {}) + # NOTE(jkulik): We have to update the "volume_sizes" key even if there + # are no volumes attached currently, because the "scheduler_hints" are + # persisted in the DB and thus could contain old information. + sh.update({'volume_sizes': volume_sizes}) + # NOTE(sbauza): If a reschedule occurs when prep_resize(), then # it only provides filter_properties legacy dict back to the # conductor with no RequestSpec part of the payload for = CONF.bigvm_mb + +Related options: + +* ``[DEFAULT] bigvm_mb`` """), ] diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index abf02bdd3e3..c1169d5d336 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -79,6 +79,18 @@ This option is ignored if serial_port_service_uri is not specified. +* serial_port_service_uri +"""), + cfg.BoolOpt('serial_port_connected_default', + default=True, + help=""" +Configure, whether the added serial port should be connected by default + +This option can be used to enable the possibility to collect logs only for +specific VMs, by reconfiguring and rebooting them in the vCenter. + +Related options: +This option is ignored if serial_port_service_uri is not specified. * serial_port_service_uri """), cfg.StrOpt('serial_log_dir', @@ -93,6 +105,78 @@ Specifies the server where the Virtual Serial Port Concentrator is storing console log files and responding to get requests. If defined it will override serial_log_dir. +"""), + cfg.BoolOpt('reserve_all_memory', + default=False, + help=""" +If true, it is not possible to over-commit memory in the vCenter, but there's +also no swap file pre-created on the ephemeral storage. Swap still works, but +is hot-created. + +For big VMs as determined by the `bigvm_mb` setting, this setting is not used. +Big VMs always reserve all their memory. +"""), + cfg.IntOpt('memory_reservation_cluster_hosts_max_fail', + default=0, + min=0, + help=""" +Allow reserving instance memory while at least n hypervisors of memory remain +unreserved in the cluster. This is a safety margin so a certain number of +cluster hypervisors are allowed to fail. If more memory would be reserved and +all the anticipated HV failures occurred, existing VMs with reserved memory +would no longer be able to start. + +This setting applies to VMs with flavors which have a nonzero extra_spec +"resources:CUSTOM_MEMORY_RESERVABLE_MB" set. + +The default value of 0 also leads to this setting being ignored and falling +back on `memory_reservation_max_ratio_fallback`. +"""), + cfg.FloatOpt('memory_reservation_max_ratio_fallback', + default=1.0, + help=""" +Allow reserving instance memory up to a maximum ratio in the cluster. This is a +safety margin so a certain ratio of cluster hypervisors are allowed to fail. If +more memory than that would be reserved and all the anticipated HV failures +occurred, existing VMs with reserved memory would no longer be able to start. + +This setting applies to VMs with flavors which have a nonzero extra_spec +"resources:CUSTOM_MEMORY_RESERVABLE_MB" set. + +This is a fallback when the `memory_reservation_cluster_hosts_max_fail` config +is set to 0. +"""), + cfg.StrOpt('hostgroup_reservations_json_file', + help=""" +Specifies the path to a JSON file containing vcpus and memory reservations per +hostgroup. HVs found to be in a hostgroup specified in this dict will report +the amount of vcpus/memory as reserved to the placement API. This enables us to +specify different reservations for HANA-used HVs and for "normal" HVs. The +special key `__default__` holds values applied if there is no specification for +an HV's hostgroup. It defaults to 0. + +There are two ways to specify a hostgroup's values: as static numbers and as +percent. Static numbers take precedence over percent if both are present. +Additionally, values are inherited from the `__default__` group if not present +for a group. To prohibit inheriting a value, explicitly set it to `null`. + +Example configuration could be: + {"__default__": {"memory_percent": 10, "vcpus": 10}, + "hana_hosts": {"memory_mb": 16384, "vcpus": 5}, + "normal_hosts": {"vcpus": null, "vcpus_percent": 2}} + +If there are cluster-wide reservations defined via `reserved_host_cpus` or +`reserved_host_memory_mb`, they are added to the sum of the hostgroup-defined +ones. + +NOTE: Percentage values will be rounded down to the next full vcpu/MB. +"""), + cfg.StrOpt('special_spawning_vm_group', + default='bigvm_free_host_antiaffinity_vmgroup', + help=""" +For spawning a VM on a prepared empty host, we need all other VMs to be in a +DRS VM group that prohibits them from running on the prepared host. This +setting configures the name of that VM group. """), ] @@ -304,6 +388,74 @@ Possible values: * Any string or empty to keep VMware default +"""), + cfg.StrOpt('bigvm_deployment_free_host_hostgroup_name', + default='', + help=""" +Name of the hostgroup used to free up a host for special spawning. + +All VMs in the cluster have to be in a VM group that has a rule specifying a +must-not-run-on-hostgroup rule for this hostgroup. Putting a host in this +hostgroup will then result in the host getting freed up by DRS. +"""), +] + +vmwareapi_driver_opts = [ + cfg.FloatOpt('server_group_sync_loop_max_group_spacing', + default=5, + min=0.5, + help=""" +Amount of time in seconds between server-group sync via sync-loop + +The sync-loop for server-groups in the driver iterates over all applicable +server-groups and calls "sync_server_group()" on them, sleeping a random time +before calling the sync. The amount of time slept at max can be defined by this +config value. + +Possible values: + * floating point value of seconds passed to random.uniform(0.5, X) +"""), + cfg.IntOpt('server_group_sync_loop_spacing', + default=3600, + help=""" +Amount of time in seconds to wait between server-group sync-loop runs + +The sync-loop thread for server-groups runs continuously, sleeping after +syncing all groups. This setting defines how long to sleep between runs. + +Possible values: + * integer >= time in seconds to sleep between runs + * integer < 0: disable the sync-loop +"""), + cfg.StrOpt('default_hw_version', + default=None, + help=""" +Set a default hardware version for VMs, that can be overridden in flavors + +This version is especially useful in multi-cluster environments where clusters +get upgraded individually but movement of VMs is necessary between them +nonetheless. It is recommend to set this to the highest version supported by +all clusters in the environment. + +Example: "vmx-13" for vSphere 6.5. +Versions can be looked up here: https://kb.vmware.com/s/article/1003746 +"""), + cfg.BoolOpt('full_clone_snapshots', + default=False, + help=""" +Use full clones for creating image snapshots instead of linked clones. +With the right hardware support, it might be faster, especially on the export. +"""), + cfg.BoolOpt('clone_from_snapshot', + default=True, + help=""" +Create a snapshot of the VM before cloning it +"""), + cfg.BoolOpt('fetch_image_from_other_datastores', + default=True, + help=""" +Before fetching from Glance an image missing on the datastore first look +for it on other datastores and clone it from there if available. """), ] @@ -311,7 +463,8 @@ vmware_utils_opts + vmwareapi_opts + spbm_opts + - vmops_opts) + vmops_opts + + vmwareapi_driver_opts) def register_opts(conf): diff --git a/nova/db/main/api.py b/nova/db/main/api.py index 16da893a156..3affb6d1f32 100644 --- a/nova/db/main/api.py +++ b/nova/db/main/api.py @@ -3403,6 +3403,22 @@ def migration_get_in_progress_by_instance(context, instance_uuid, return query.all() +def _migration_filter_item(query, filters, name): + if name not in filters: + return query + + column = getattr(models.Migration, name) + + item = filters[name] + if isinstance(item, str) or item is None: + return query.filter(column == item) + + try: + return query.filter(column.in_(item)) + except TypeError: # in_ expects an iterable, we fall back to comparison + return query.filter(column == item) + + @pick_context_manager_reader def migration_get_all_by_filters(context, filters, sort_keys=None, sort_dirs=None, @@ -3412,44 +3428,20 @@ def migration_get_all_by_filters(context, filters, return [] query = model_query(context, models.Migration) - if "uuid" in filters: - # The uuid filter is here for the MigrationLister and multi-cell - # paging support in the compute API. - uuid = filters["uuid"] - uuid = [uuid] if isinstance(uuid, str) else uuid - query = query.filter(models.Migration.uuid.in_(uuid)) model_object = models.Migration query = _get_query_nova_resource_by_changes_time(query, filters, model_object) - if "status" in filters: - status = filters["status"] - status = [status] if isinstance(status, str) else status - query = query.filter(models.Migration.status.in_(status)) - if "host" in filters: - host = filters["host"] - query = query.filter(sql.or_( - models.Migration.source_compute == host, - models.Migration.dest_compute == host)) - elif "source_compute" in filters: - host = filters['source_compute'] - query = query.filter(models.Migration.source_compute == host) if "node" in filters: node = filters['node'] query = query.filter(sql.or_( models.Migration.source_node == node, models.Migration.dest_node == node)) - if "migration_type" in filters: - migtype = filters["migration_type"] - query = query.filter(models.Migration.migration_type == migtype) if "hidden" in filters: hidden = filters["hidden"] query = query.filter(models.Migration.hidden == hidden) - if "instance_uuid" in filters: - instance_uuid = filters["instance_uuid"] - query = query.filter(models.Migration.instance_uuid == instance_uuid) if 'user_id' in filters: user_id = filters['user_id'] query = query.filter(models.Migration.user_id == user_id) @@ -3457,6 +3449,23 @@ def migration_get_all_by_filters(context, filters, project_id = filters['project_id'] query = query.filter(models.Migration.project_id == project_id) + # The uuid filter is here for the MigrationLister and multi-cell + # paging support in the compute API. + for item in ("uuid", "status", "migration_type", "instance_uuid"): + query = _migration_filter_item(query, filters, item) + + if "host" in filters: + host = filters["host"] + query = query.filter(sql.or_( + models.Migration.source_compute == host, + models.Migration.dest_compute == host)) + else: + query = _migration_filter_item(query, filters, "source_compute") + + if "hidden" in filters: + hidden = filters["hidden"] + query = query.filter(models.Migration.hidden == hidden) + if marker: try: marker = migration_get_by_uuid(context, marker) diff --git a/nova/network/neutron.py b/nova/network/neutron.py index 00edf5c3c04..c303a8f532b 100644 --- a/nova/network/neutron.py +++ b/nova/network/neutron.py @@ -61,6 +61,17 @@ _ADMIN_AUTH = None +# NOTE(jkulik): We need to monkeypatch this exception into +# python-neutronclient, because we don't get the error_type from the exception +# back from python-neutronclient other than via a custom class named like it. +class PortBindingAlreadyExistsClient(neutron_client_exc.Conflict): + pass + + +neutron_client_exc.PortBindingAlreadyExistsClient = \ + PortBindingAlreadyExistsClient + + def reset_state(): global _ADMIN_AUTH global _SESSION @@ -1597,12 +1608,25 @@ def bind_ports_to_host(self, context, instance, host, data = {'binding': binding} try: - binding = client.create_port_binding(port_id, data)['binding'] + step = 'creation' + try: + binding = client.create_port_binding( + port_id, data + )['binding'] + except neutron_client_exc.PortBindingAlreadyExistsClient: + # In case we actually managed to create the binding, + # but something went wrong in between, we can recover + step = 'recovery' + binding = client.show_port_binding( + port_id, host + )['binding'] except neutron_client_exc.NeutronClientException: # Something failed, so log the error and rollback any # successful bindings. - LOG.error('Binding failed for port %s and host %s.', + LOG.error('Binding %s failed for port %s and host %s.', + step, port_id, host, instance=instance, exc_info=True) + for rollback_port_id in bindings_by_port_id: try: client.delete_port_binding(rollback_port_id, host) diff --git a/nova/objects/compute_node.py b/nova/objects/compute_node.py index dfc1b2ae284..789d1a10204 100644 --- a/nova/objects/compute_node.py +++ b/nova/objects/compute_node.py @@ -505,7 +505,8 @@ def get_all_by_uuids(cls, context, compute_uuids): @db.select_db_reader_mode def _db_compute_node_get_by_hv_type(context, hv_type): db_computes = context.session.query(models.ComputeNode).filter( - models.ComputeNode.hypervisor_type == hv_type).all() + models.ComputeNode.hypervisor_type == hv_type, + models.ComputeNode.deleted == 0).all() return db_computes @classmethod diff --git a/nova/objects/fields.py b/nova/objects/fields.py index d64234f00c6..bc0bbaaae2f 100644 --- a/nova/objects/fields.py +++ b/nova/objects/fields.py @@ -904,9 +904,10 @@ class NotificationSource(BaseNovaEnum): # bumped to 3.0 CONSOLE = 'nova-console' METADATA = 'nova-metadata' + BIGVM = 'nova-bigvm' ALL = (API, COMPUTE, CONDUCTOR, SCHEDULER, - NETWORK, CONSOLEAUTH, CELLS, CONSOLE, METADATA) + NETWORK, CONSOLEAUTH, CELLS, CONSOLE, METADATA, BIGVM) @staticmethod def get_source_by_binary(binary): @@ -965,6 +966,7 @@ class NotificationAction(BaseNovaEnum): ADD_HOST = 'add_host' REMOVE_HOST = 'remove_host' ADD_MEMBER = 'add_member' + REMOVE_MEMBER = 'remove_member' UPDATE_METADATA = 'update_metadata' LOCK = 'lock' UNLOCK = 'unlock' @@ -985,10 +987,10 @@ class NotificationAction(BaseNovaEnum): LIVE_MIGRATION_ROLLBACK_DEST, REBUILD, INTERFACE_DETACH, RESIZE_CONFIRM, RESIZE_PREP, RESIZE_REVERT, SHELVE_OFFLOAD, SOFT_DELETE, TRIGGER_CRASH_DUMP, UNRESCUE, UNSHELVE, ADD_HOST, - REMOVE_HOST, ADD_MEMBER, UPDATE_METADATA, LOCK, UNLOCK, - REBUILD_SCHEDULED, UPDATE_PROP, LIVE_MIGRATION_FORCE_COMPLETE, - CONNECT, USAGE, BUILD_INSTANCES, MIGRATE_SERVER, REBUILD_SERVER, - SELECT_DESTINATIONS, IMAGE_CACHE) + REMOVE_HOST, ADD_MEMBER, REMOVE_MEMBER, UPDATE_METADATA, LOCK, + UNLOCK, REBUILD_SCHEDULED, UPDATE_PROP, + LIVE_MIGRATION_FORCE_COMPLETE, CONNECT, USAGE, BUILD_INSTANCES, + MIGRATE_SERVER, REBUILD_SERVER, SELECT_DESTINATIONS, IMAGE_CACHE) # TODO(rlrossit): These should be changed over to be a StateMachine enum from diff --git a/nova/objects/instance_group.py b/nova/objects/instance_group.py index 8a12a876934..d1c3719b052 100644 --- a/nova/objects/instance_group.py +++ b/nova/objects/instance_group.py @@ -34,14 +34,23 @@ LOG = logging.getLogger(__name__) -def _instance_group_get_query(context, id_field=None, id=None): +def _instance_group_get_query(context, id_field=None, id=None, + project_id=None, + limit=None, + offset=None): query = context.session.query(api_models.InstanceGroup).\ options(orm.joinedload(api_models.InstanceGroup._policies)).\ options(orm.joinedload(api_models.InstanceGroup._members)) if not context.is_admin: query = query.filter_by(project_id=context.project_id) + elif project_id is not None: + query = query.filter_by(project_id=project_id) if id and id_field: query = query.filter(id_field == id) + if limit is not None: + query = query.limit(limit) + if offset is not None: + query = query.offset(offset) return query @@ -333,9 +342,6 @@ def _add_members_in_db(context, group_uuid, members): @staticmethod @api_db_api.context_manager.writer def _remove_members_in_db(context, group_id, instance_uuids): - # There is no public method provided for removing members because the - # user-facing API doesn't allow removal of instance group members. We - # need to be able to remove members to address quota races. context.session.query(api_models.InstanceGroupMember).\ filter_by(group_id=group_id).\ filter(api_models.InstanceGroupMember.instance_uuid. @@ -490,6 +496,16 @@ def add_members(cls, context, group_uuid, instance_uuids): compute_utils.notify_about_server_group_add_member(context, group_uuid) return list(members) + @base.remotable_classmethod + def remove_members(cls, context, group_id, instance_uuids, group_uuid): + payload = {'server_group_id': group_uuid, + 'instance_uuids': instance_uuids} + cls._remove_members_in_db(context, group_id, instance_uuids) + compute_utils.notify_about_server_group_update(context, + "removemember", payload) + compute_utils.notify_about_server_group_remove_member(context, + group_uuid) + @base.remotable def get_hosts(self, exclude=None): """Get a list of hosts for non-deleted instances in the group @@ -545,10 +561,11 @@ class InstanceGroupList(base.ObjectListBase, base.NovaObject): @staticmethod @api_db_api.context_manager.reader - def _get_from_db(context, project_id=None): - query = _instance_group_get_query(context) - if project_id is not None: - query = query.filter_by(project_id=project_id) + def _get_from_db(context, project_id=None, limit=None, offset=None): + query = _instance_group_get_query(context, + project_id=project_id, + limit=limit, + offset=offset) return query.all() @staticmethod @@ -563,15 +580,34 @@ def _get_counts_from_db(context, project_id, user_id=None): counts['user'] = {'server_groups': query.count()} return counts + @staticmethod + @api_db_api.context_manager.reader + def _get_from_db_by_instance_uuids(context, instance_uuids): + if not instance_uuids: + return [] + + members = set(instance_uuids) + groups = context.session.query(api_models.InstanceGroup).\ + join(api_models.InstanceGroupMember, + api_models.InstanceGroupMember.group_id == + api_models.InstanceGroup.id).\ + filter(api_models.InstanceGroupMember.instance_uuid.in_(members)).\ + options(orm.joinedload(api_models.InstanceGroup._policies)).\ + options(orm.contains_eager(api_models.InstanceGroup._members))\ + .all() + return groups + @base.remotable_classmethod - def get_by_project_id(cls, context, project_id): - api_db_groups = cls._get_from_db(context, project_id=project_id) + def get_by_project_id(cls, context, project_id, limit=None, offset=None): + api_db_groups = cls._get_from_db(context, project_id=project_id, + limit=limit, + offset=offset) return base.obj_make_list(context, cls(context), objects.InstanceGroup, api_db_groups) @base.remotable_classmethod - def get_all(cls, context): - api_db_groups = cls._get_from_db(context) + def get_all(cls, context, limit=None, offset=None): + api_db_groups = cls._get_from_db(context, limit=limit, offset=offset) return base.obj_make_list(context, cls(context), objects.InstanceGroup, api_db_groups) @@ -589,3 +625,10 @@ def get_counts(cls, context, project_id, user_id=None): 'user': {'server_groups': }} """ return cls._get_counts_from_db(context, project_id, user_id=user_id) + + @base.remotable_classmethod + def get_by_instance_uuids(cls, context, instance_uuids): + api_db_groups = cls._get_from_db_by_instance_uuids(context, + instance_uuids) + return base.obj_make_list(context, cls(context), objects.InstanceGroup, + api_db_groups) diff --git a/nova/objects/migrate_data.py b/nova/objects/migrate_data.py index 1340c34d971..a2516eb1582 100644 --- a/nova/objects/migrate_data.py +++ b/nova/objects/migrate_data.py @@ -373,16 +373,58 @@ def obj_make_compatible(self, primitive, target_version): class VMwareLiveMigrateData(LiveMigrateData): # Version 1.0: Initial version # Version 1.1: Inherited pci_dev_map_src_dst from LiveMigrateData - VERSION = '1.1' + # Version 1.2: Added dest_cluster_ref, is_same_vcenter, + # instance_already_migrated, relocate_defaults_json, + # vif_infos_json for cross-vcenter migration + VERSION = '1.2' fields = { 'cluster_name': fields.StringField(nullable=False), 'datastore_regex': fields.StringField(nullable=False), + 'dest_cluster_ref': fields.StringField(nullable=False), + 'is_same_vcenter': fields.BooleanField(default=True), + 'instance_already_migrated': fields.BooleanField(default=False), + 'relocate_defaults_json': fields.SensitiveStringField(default="{}"), + 'vif_infos_json': fields.StringField(default="[]"), } + @property + def relocate_defaults(self): + return jsonutils.loads(self.relocate_defaults_json) + + @relocate_defaults.setter + def relocate_defaults(self, relocate_defaults_dict): + self.relocate_defaults_json = jsonutils.dumps(relocate_defaults_dict) + + @property + def vif_infos(self): + return jsonutils.loads(self.vif_infos_json) + + @vif_infos.setter + def vif_infos(self, vif_infos): + self.vif_infos_json = jsonutils.dumps(vif_infos) + + def to_legacy_dict(self, pre_migration_result=False): + legacy = super(VMwareLiveMigrateData, self).to_legacy_dict() + for field in self.fields: + if self.obj_attr_is_set(field): + legacy[field] = getattr(self, field) + return legacy + + def from_legacy_dict(self, legacy): + super(VMwareLiveMigrateData, self).from_legacy_dict(legacy) + for field in self.fields: + if field in legacy: + setattr(self, field, legacy[field]) + def obj_make_compatible(self, primitive, target_version): super(VMwareLiveMigrateData, self).obj_make_compatible( primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) - if (target_version < (1, 1)): + if target_version < (1, 2): + for k in ('dest_cluster_ref', 'is_same_vcenter', + 'instance_already_migrated', 'relocate_defaults_json', + 'vif_infos_json'): + primitive.pop(k, None) + if target_version < (1, 1): primitive.pop('pci_dev_map_src_dst', None) diff --git a/nova/policies/server_groups.py b/nova/policies/server_groups.py index 8dfbe7c2020..45b130ff4b2 100644 --- a/nova/policies/server_groups.py +++ b/nova/policies/server_groups.py @@ -82,6 +82,18 @@ ], scope_types=['project'] ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'update', + check_str=base.PROJECT_MEMBER_OR_ADMIN, + description="Update members of a server group", + operations=[ + { + 'path': '/os-server-groups/{server_group_id}', + 'method': 'PUT' + } + ], + scope_types=['system', 'project'] + ), ] diff --git a/nova/rpc.py b/nova/rpc.py index 7a92650414b..25e5c71bcf7 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -384,6 +384,7 @@ class LegacyValidatingNotifier(object): 'scheduler.select_destinations.end', 'scheduler.select_destinations.start', 'servergroup.addmember', + 'servergroup.removemember', 'servergroup.create', 'servergroup.delete', 'volume.usage', diff --git a/nova/scheduler/client/report.py b/nova/scheduler/client/report.py index 8f9abb9c936..74689f10b57 100644 --- a/nova/scheduler/client/report.py +++ b/nova/scheduler/client/report.py @@ -364,7 +364,6 @@ def get_allocation_candidates(self, context, resources): LOG.error(msg, args) return None, None, None - @safe_connect def _get_provider_aggregates(self, context, rp_uuid): """Queries the placement API for a resource provider's aggregates. @@ -2638,3 +2637,27 @@ def _get_core_usages(usages): self._handle_usages_error_from_placement(resp, project_id, user_id=user_id) return total_counts + + def get_traits(self, context): + """Queries the placement API for all traits. + + :param context: The security context + :return: A set() of trait names + :raise: NovaException on errors. + :raise: keystoneauth1.exceptions.ClientException if placement API + communication fails. + """ + resp = self.get("/traits", version='1.6', + global_request_id=context.global_id) + + if resp.status_code == 200: + json = resp.json() + return set(json['traits']) + + placement_req_id = get_placement_request_id(resp) + LOG.error( + "[%(placement_req_id)s] Failed to retrieve traits from " + "placement API. Got %(status_code)d: %(err_text)s.", + {'placement_req_id': placement_req_id, + 'status_code': resp.status_code, 'err_text': resp.text}) + raise exception.NovaException('Failed to get traits from Placement') diff --git a/nova/scheduler/filters/bigvm_filter.py b/nova/scheduler/filters/bigvm_filter.py new file mode 100644 index 00000000000..354480318ba --- /dev/null +++ b/nova/scheduler/filters/bigvm_filter.py @@ -0,0 +1,101 @@ +# Copyright (c) 2019 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from oslo_log import log as logging + +import nova.conf +from nova import context +from nova.scheduler.client import report +from nova.scheduler import filters +from nova.utils import is_big_vm + +LOG = logging.getLogger(__name__) + +CONF = nova.conf.CONF + + +class BigVmBaseFilter(filters.BaseHostFilter): + + _HV_SIZE_CACHE = {} + _HV_SIZE_CACHE_RETENTION_TIME = 10 * 60 + + RUN_ON_REBUILD = False + + def _get_hv_size(self, host_state): + # expire the cache 10min after last write + time_diff = time.time() - self._HV_SIZE_CACHE.get('last_modified', 0) + if time_diff > self._HV_SIZE_CACHE_RETENTION_TIME: + self._HV_SIZE_CACHE = {} + + if host_state.uuid not in self._HV_SIZE_CACHE: + placement_client = report.SchedulerReportClient() + elevated = context.get_admin_context() + res = placement_client._get_inventory(elevated, host_state.uuid) + if not res: + return None + inventories = res.get('inventories', {}) + hv_size_mb = inventories.get('MEMORY_MB', {}).get('max_unit') + self._HV_SIZE_CACHE[host_state.uuid] = hv_size_mb + + self._HV_SIZE_CACHE['last_modified'] = time.time() + + return self._HV_SIZE_CACHE[host_state.uuid] + + +class BigVmClusterUtilizationFilter(BigVmBaseFilter): + """Only schedule big VMs to a vSphere cluster (i.e. nova-compute host) if + the memory-utilization of the cluster is below a threshold depending on the + hypervisor size and the requested memory. + """ + + def _get_max_ram_percent(self, requested_ram_mb, hypervisor_ram_mb): + """We want the hosts to have on average half the requested memory free. + """ + requested_ram_mb = float(requested_ram_mb) + hypervisor_ram_mb = float(hypervisor_ram_mb) + hypervisor_max_used_ram_mb = hypervisor_ram_mb - requested_ram_mb / 2 + return hypervisor_max_used_ram_mb / hypervisor_ram_mb * 100 + + def host_passes(self, host_state, spec_obj): + requested_ram_mb = spec_obj.memory_mb + # not scheduling a big VM -> every host is fine + if not is_big_vm(requested_ram_mb, spec_obj.flavor): + return True + + hypervisor_ram_mb = self._get_hv_size(host_state) + if hypervisor_ram_mb is None: + return False + + free_ram_mb = host_state.free_ram_mb + total_usable_ram_mb = host_state.total_usable_ram_mb + used_ram_mb = total_usable_ram_mb - free_ram_mb + used_ram_percent = float(used_ram_mb) / total_usable_ram_mb * 100.0 + + max_ram_percent = self._get_max_ram_percent(requested_ram_mb, + hypervisor_ram_mb) + + if used_ram_percent > max_ram_percent: + LOG.info("%(host_state)s does not have less than " + "%(max_ram_percent)s %% RAM utilization (has " + "%(used_ram_percent)s %%) and is thus not suitable " + "for big VMs.", + {'host_state': host_state, + 'max_ram_percent': max_ram_percent, + 'used_ram_percent': used_ram_percent}) + return False + + return True diff --git a/nova/scheduler/filters/shard_filter.py b/nova/scheduler/filters/shard_filter.py index bc9b5ff7f40..d6c4e73a719 100644 --- a/nova/scheduler/filters/shard_filter.py +++ b/nova/scheduler/filters/shard_filter.py @@ -19,6 +19,7 @@ from nova.scheduler.mixins import ProjectTagMixin from nova.scheduler import utils from nova import utils as nova_utils +from nova.utils import BIGVM_EXCLUSIVE_TRAIT LOG = logging.getLogger(__name__) @@ -35,8 +36,9 @@ class ShardFilter(filters.BaseHostFilter, ProjectTagMixin): """ _ALL_SHARDS = "sharding_enabled" + _HANA_USE_SHARDING = "use_individual_shard_tags_hana" _SHARD_PREFIX = 'vc-' - _PROJECT_TAG_TAGS = [_ALL_SHARDS] + _PROJECT_TAG_TAGS = [_ALL_SHARDS, _HANA_USE_SHARDING] _PROJECT_TAG_PREFIX = _SHARD_PREFIX def _get_shards(self, project_id): @@ -45,6 +47,14 @@ def _get_shards(self, project_id): # _get_shards() so it's clear what we return return self._get_tags(project_id) + def filter_all(self, filter_obj_list, spec_obj): + if self._ignore_hana_flavor(spec_obj): + LOG.debug("Hana/BigVM flavor requested. Ignoring sharding.") + return filter_obj_list + + return super(ShardFilter, self).filter_all( + filter_obj_list, spec_obj) + def host_passes(self, host_state, spec_obj): # Only VMware if utils.is_non_vmware_spec(spec_obj): @@ -94,3 +104,20 @@ def host_passes(self, host_state, spec_obj): 'host_shard': host_shard_names, 'project_shards': shards}) return False + + def _ignore_hana_flavor(self, spec_obj): + if not CONF.filter_scheduler.sharding_ignore_hana: + return False + + proj_tags = self._get_tags(spec_obj.project_id) + if self._HANA_USE_SHARDING in proj_tags: + return False + + strategy = CONF.filter_scheduler.hana_detection_strategy + extra_specs = spec_obj.flavor.extra_specs + + if strategy == "hana_exclusive_host": + trait = f"trait:{BIGVM_EXCLUSIVE_TRAIT}" + return extra_specs.get(trait) == "required" + elif strategy == "memory_mb": + return spec_obj.flavor.memory_mb >= CONF.bigvm_mb diff --git a/nova/service.py b/nova/service.py index 3107c113583..0e1aa55d7dc 100644 --- a/nova/service.py +++ b/nova/service.py @@ -60,6 +60,7 @@ 'nova-compute': 'nova.compute.manager.ComputeManager', 'nova-conductor': 'nova.conductor.manager.ConductorManager', 'nova-scheduler': 'nova.scheduler.manager.SchedulerManager', + 'nova-bigvm': 'nova.bigvm.manager.BigVmManager', } diff --git a/nova/tests/unit/api/openstack/compute/test_server_groups.py b/nova/tests/unit/api/openstack/compute/test_server_groups.py index 5c4c9095764..0e91ebd49a3 100644 --- a/nova/tests/unit/api/openstack/compute/test_server_groups.py +++ b/nova/tests/unit/api/openstack/compute/test_server_groups.py @@ -165,13 +165,13 @@ def test_create_server_group_rbac_default(self): # test as non-admin self.controller.create(self.member_req, body={'server_group': sgroup}) - def _create_instance(self, ctx, cell): + def _create_instance(self, ctx, cell, host='host1'): with context.target_cell(ctx, cell) as cctx: instance = objects.Instance(context=cctx, image_ref=uuidsentinel.fake_image_ref, compute_id=123, node='node1', reservation_id='a', - host='host1', + host=host, project_id=fakes.FAKE_PROJECT_ID, vm_state='fake', system_metadata={'key': 'value'}) @@ -184,10 +184,10 @@ def _create_instance(self, ctx, cell): im.create() return instance - def _create_instance_group(self, context, members): + def _create_instance_group(self, context, members, policy=None): ig = objects.InstanceGroup(context=context, name='fake_name', user_id='fake_user', project_id=fakes.FAKE_PROJECT_ID, - members=members) + members=members, policy=policy) ig.create() return ig.uuid @@ -275,16 +275,26 @@ def _test_list_server_group(self, mock_get_all, mock_get_by_project, tenant_specific = {'server_groups': tenant_groups} def return_all_server_groups(): - return objects.InstanceGroupList( - objects=[objects.InstanceGroup( - **server_group_db(sg)) for sg in all_groups]) + if limited: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in tenant_groups]) + else: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in all_groups]) mock_get_all.return_value = return_all_server_groups() def return_tenant_server_groups(): - return objects.InstanceGroupList( - objects=[objects.InstanceGroup( - **server_group_db(sg)) for sg in tenant_groups]) + if limited: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in []]) + else: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in tenant_groups]) mock_get_by_project.return_value = return_tenant_server_groups() @@ -780,6 +790,331 @@ def test_additional_params(self): self.assertRaises(self.validation_error, self.controller.create, req, body={'server_group': sgroup}) + def test_update_server_group_not_found(self): + """We raise a 404 if the server group does not exist.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.update, req, uuidsentinel.group1, body={}) + + @mock.patch('nova.compute.api.API.sync_server_group') + def test_update_server_group_empty(self, mock_sync): + """We do not fail if the user doesn't request any changes""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + res_dict = self.controller.update(req, ig_uuid, body={}) + result_members = res_dict['server_group']['members'] + self.assertEqual(3, len(result_members)) + for member in members: + self.assertIn(member, result_members) + mock_sync.assert_not_called() + + def test_update_server_group_add_remove_overlap(self): + """We do not accept changes, if there's a server to be both added and + removed, because the result would depend on implementation details if + we remove first or add first. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'add_members': [uuidsentinel.uuid1, uuidsentinel.uuid2], + 'remove_members': [uuidsentinel.uuid2, uuidsentinel.uuid3], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn('Parameters "add_members" and "remove_members" are ' + 'overlapping in {}'.format(uuidsentinel.uuid2), + str(result)) + + @mock.patch('nova.compute.api.API.sync_server_group') + def test_update_server_group_remove_nonexisting(self, mock_sync): + """Don't fail if the user tries to remove a server not being member of + the server group. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'remove_members': [uuidsentinel.uuid4], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(3, len(result_members)) + for member in members: + self.assertIn(member, result_members) + mock_sync.assert_not_called() + + @mock.patch('nova.compute.api.API.sync_server_group') + def test_update_server_group_add_already_added(self, mock_sync): + """Don't fail if the user adds a server that's already a member of the + server group. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'add_members': [members[0]], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(3, len(result_members)) + for member in members: + self.assertIn(member, result_members) + mock_sync.assert_not_called() + + def test_update_server_group_add_against_policy_affinity(self): + """Fail if adding the server would break the policy.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host2') + + body = { + 'add_members': [new_instance.uuid], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn("Adding instance(s) {} would violate policy 'affinity'." + .format(new_instance.uuid), + str(result)) + + def test_update_server_group_add_against_policy_anti_affinity(self): + """Fail if adding the server would break the policy.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='anti-affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host1') + + body = { + 'add_members': [new_instance.uuid], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn("Adding instance(s) {} would violate policy " + "'anti-affinity'.".format(new_instance.uuid), + str(result)) + + @mock.patch('nova.compute.api.API.sync_server_group') + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_add_with_remove_fixes_policy(self, + mock_req_spec, + mock_sync): + """Don't fail if adding a server would break the policy, but the remove + in the same request fixes that. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1'), + self._create_instance(ctx, cell1, host='host2')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='anti-affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host2') + + body = { + 'add_members': [new_instance.uuid], + 'remove_members': [instances[1].uuid], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(2, len(result_members)) + for member in [instances[0].uuid, new_instance.uuid]: + self.assertIn(member, result_members) + req_context = req.environ['nova.context'] + mock_sync.assert_called_with(req_context, set(['host2']), + ig_uuid) + + def test_update_server_group_add_nonexisting_instance(self): + """Fail if the instances the user tries to add does not exist.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'add_members': [uuidsentinel.uuid1], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn('One or more members in add_members cannot be found: ' + '{}'.format(uuidsentinel.uuid1), + str(result)) + + @mock.patch('nova.compute.api.API.sync_server_group') + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_add_instance_multiple_cells(self, + mock_req_spec, + mock_sync): + """Don't fail if the instance the user tries to add is in another cell. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + cell1 = self.cells[uuidsentinel.cell1] + cell2 = self.cells[uuidsentinel.cell2] + new_instances = [self._create_instance(ctx, cell1), + self._create_instance(ctx, cell2)] + body = { + 'add_members': [i.uuid for i in new_instances], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(5, len(result_members)) + for member in members: + self.assertIn(member, result_members) + for instance in new_instances: + self.assertIn(instance.uuid, result_members) + expected_hosts = \ + set([i.host for i in instances] + [i.host for i in new_instances]) + req_context = req.environ['nova.context'] + mock_sync.assert_called_with(req_context, expected_hosts, ig_uuid) + + @mock.patch('nova.compute.api.API.sync_server_group') + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_add_against_soft_policy(self, mock_req_spec, + mock_sync): + """Don't fail if the policy would fail, but it's a soft-* policy - they + are best-effort by design. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='soft-anti-affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host1') + + body = { + 'add_members': [new_instance.uuid], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(2, len(result_members)) + for member in [instances[0].uuid, new_instance.uuid]: + self.assertIn(member, result_members) + req_context = req.environ['nova.context'] + mock_sync.assert_called_with(req_context, set(['host1']), ig_uuid) + + @mock.patch('nova.objects.InstanceGroupList.get_by_instance_uuids') + def test_update_server_group_add_already_in_other(self, mock_gbiu): + """If any of the servers is already part of another server-group, we + fail. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances]) + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host1') + ig_uuid2 = self._create_instance_group(ctx, [new_instance.uuid]) + + mock_gbiu.return_value = objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db({'id': ig_uuid}))]) + + body = { + 'add_members': [instances[0].uuid], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid2, body=body) + self.assertIn('One or more members in add_members is already ' + 'assigned to another server group. Server groups: {}' + .format(ig_uuid), str(result)) + + @mock.patch('nova.objects.InstanceGroupList.get_by_instance_uuids') + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_updates_request_spec(self, mock_req_spec, + mock_gbiu): + """If we update the server-group membership of a server, we also have + to update the appropriate RequestSpec attribute. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig = objects.InstanceGroup(context=ctx, name='fake_name', + user_id='fake_user', project_id='fake', + members=[i.uuid for i in instances], policy='anti-affinity') + ig.create() + + new_instance = self._create_instance(ctx, cell1, host='host2') + + mock_gbiu.return_value = objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db({'id': ig.uuid}))]) + + fake_spec_remove = mock.Mock(instance_group=ig) + fake_spec_add = mock.Mock(instance_group=None) + + def _get_req_spec(context, instance_uuid): + if instance_uuid == new_instance.uuid: + return fake_spec_add + else: + return fake_spec_remove + + mock_req_spec.side_effect = _get_req_spec + + self.assertEqual(ig.uuid, fake_spec_remove.instance_group.uuid) + self.assertIsNone(fake_spec_add.instance_group) + + body = { + 'add_members': [new_instance.uuid], + 'remove_members': [instances[0].uuid] + } + res_dict = self.controller.update(req, ig.uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(1, len(result_members)) + for member in [new_instance.uuid]: + self.assertIn(member, result_members) + + fake_spec_remove.save.assert_called() + self.assertIsNone(fake_spec_remove.instance_group) + ig.members = [new_instance.uuid] + self.assertEqual(ig.uuid, fake_spec_add.instance_group.uuid) + fake_spec_add.save.assert_called() + class ServerGroupTestV275(ServerGroupTestV264): wsgi_api_version = '2.75' diff --git a/nova/tests/unit/cmd/test_manage.py b/nova/tests/unit/cmd/test_manage.py index be2541105d0..e7d18a22047 100644 --- a/nova/tests/unit/cmd/test_manage.py +++ b/nova/tests/unit/cmd/test_manage.py @@ -2741,8 +2741,8 @@ def test_sync_aggregates_get_provider_aggs_placement_server_error( mock_agg_add.side_effect = ( exception.ResourceProviderAggregateRetrievalFailed( uuid=uuidsentinel.rp_uuid)) - with mock.patch.object(self.cli, '_get_rp_uuid_for_host', - return_value=uuidsentinel.rp_uuid): + with mock.patch.object(self.cli, '_get_rp_uuids_for_host', + return_value=[uuidsentinel.rp_uuid]): result = self.cli.sync_aggregates(verbose=True) self.assertEqual(2, result) self.assertIn('Failed to get aggregates for resource provider with ' @@ -2764,8 +2764,8 @@ def test_sync_aggregates_put_aggregates_fails_provider_not_found( """ mock_agg_add.side_effect = exception.ResourceProviderNotFound( name_or_uuid=uuidsentinel.rp_uuid) - with mock.patch.object(self.cli, '_get_rp_uuid_for_host', - return_value=uuidsentinel.rp_uuid): + with mock.patch.object(self.cli, '_get_rp_uuids_for_host', + return_value=[uuidsentinel.rp_uuid]): result = self.cli.sync_aggregates(verbose=True) self.assertEqual(6, result) self.assertIn('Unable to find matching resource provider record in ' @@ -2790,8 +2790,8 @@ def test_sync_aggregates_put_aggregates_fails_generation_conflict( """ mock_agg_add.side_effect = exception.ResourceProviderUpdateConflict( uuid=uuidsentinel.rp_uuid, generation=1, error="Conflict!") - with mock.patch.object(self.cli, '_get_rp_uuid_for_host', - return_value=uuidsentinel.rp_uuid): + with mock.patch.object(self.cli, '_get_rp_uuids_for_host', + return_value=[uuidsentinel.rp_uuid]): result = self.cli.sync_aggregates(verbose=True) self.assertEqual(3, result) self.assertIn("Failed updating provider aggregates for " diff --git a/nova/tests/unit/compute/test_compute.py b/nova/tests/unit/compute/test_compute.py index b0f7de18429..43f333fd061 100644 --- a/nova/tests/unit/compute/test_compute.py +++ b/nova/tests/unit/compute/test_compute.py @@ -1671,6 +1671,9 @@ def setUp(self): 'm1.small') self.tiny_flavor = objects.Flavor.get_by_name(self.context, 'm1.tiny') + self.stub_out('eventlet.greenthread.sleep', + lambda *a, **kw: None) + def fake_share_info(self): share_mapping = {} share_mapping['id'] = 1 diff --git a/nova/tests/unit/compute/test_resource_tracker.py b/nova/tests/unit/compute/test_resource_tracker.py index 3af6eb2be8b..d88b90dc835 100644 --- a/nova/tests/unit/compute/test_resource_tracker.py +++ b/nova/tests/unit/compute/test_resource_tracker.py @@ -766,15 +766,15 @@ def test_no_instances_no_migrations_reserved_disk_ram_and_cpu( get_cn_mock.assert_called_once_with(mock.ANY, uuids.cn1) expected_resources = copy.deepcopy(_COMPUTE_NODE_FIXTURES[0]) vals = { - 'free_disk_gb': 5, # 6GB avail - 1 GB reserved - 'local_gb': 6, - 'free_ram_mb': 0, # 512MB avail - 512MB reserved - 'memory_mb_used': 512, # 0MB used + 512MB reserved - 'vcpus_used': 1, - 'local_gb_used': 1, # 0GB used + 1 GB reserved - 'memory_mb': 512, + 'free_disk_gb': 5, # local_gb + 'local_gb': 5, # 6GB avail - 1 GB reserved + 'free_ram_mb': 0, # memory_mb + 'memory_mb_used': 0, # 0MB used + 'vcpus_used': 0, + 'local_gb_used': 0, # 0GB used + 'memory_mb': 0, # 512MB avail - 512MB reserved 'current_workload': 0, - 'vcpus': 4, + 'vcpus': 3, # 4 CPUs - 1 CPU reserved 'running_vms': 0 } _update_compute_node(expected_resources, **vals) @@ -783,6 +783,54 @@ def test_no_instances_no_migrations_reserved_disk_ram_and_cpu( actual_resources)) update_mock.assert_called_once() + @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', + return_value=objects.InstancePCIRequests(requests=[])) + @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', + return_value=objects.PciDeviceList()) + @mock.patch('nova.objects.ComputeNode.get_by_uuid') + @mock.patch('nova.objects.MigrationList.get_in_progress_and_error') + @mock.patch('nova.objects.InstanceList.get_by_host_and_node') + def test_no_instances_no_migrations_reserved_disk_ram_and_cpu_driver( + self, get_mock, migr_mock, get_cn_mock, pci_mock, + instance_pci_mock): + # Setup with config reservations and driver-returned reservations. + # Driver-returned reservations should include the config ones and thus + # overwrite the config ones. + self.flags(reserved_host_disk_mb=1024, + reserved_host_memory_mb=512, + reserved_host_cpus=1) + virt_resources = copy.deepcopy(_VIRT_DRIVER_AVAIL_RESOURCES) + virt_resources.update(vcpus_reserved=2, + memory_mb_reserved=128) + self._setup_rt(virt_resources=virt_resources) + + get_mock.return_value = [] + migr_mock.return_value = [] + get_cn_mock.return_value = _COMPUTE_NODE_FIXTURES[0] + + update_mock = self._update_available_resources() + + get_cn_mock.assert_called_once_with(mock.ANY, uuids.cn1) + expected_resources = copy.deepcopy(_COMPUTE_NODE_FIXTURES[0]) + vals = { + 'free_disk_gb': 5, # local_gb + 'local_gb': 5, # 6GB avail - 1 GB reserved + 'free_ram_mb': 384, # memory_mb + 'memory_mb_used': 0, # 0MB used + 'vcpus_used': 0, + 'local_gb_used': 0, # 0GB used + 'memory_mb': 384, # 512MB avail - 128MB reserved + 'current_workload': 0, + 'vcpus': 2, # 4 vcpus - 2 reserved + 'running_vms': 0 + } + _update_compute_node(expected_resources, **vals) + actual_resources = update_mock.call_args[0][1] + self.assertTrue(obj_base.obj_equal_prims(expected_resources, + actual_resources)) + + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', return_value=objects.InstancePCIRequests(requests=[])) @@ -793,7 +841,8 @@ def test_no_instances_no_migrations_reserved_disk_ram_and_cpu( @mock.patch('nova.objects.InstanceList.get_by_host_and_node') def test_some_instances_no_migrations(self, get_mock, migr_mock, get_cn_mock, pci_mock, - instance_pci_mock, bfv_check_mock): + instance_pci_mock, bfv_check_mock, + bdms_by_uuid_mock): # Setup virt resources to match used resources to number # of defined instances on the hypervisor # Note that the usage numbers here correspond to only the first @@ -1022,6 +1071,8 @@ def test_no_instances_dest_evacuation(self, get_mock, get_inst_mock, actual_resources)) update_mock.assert_called_once() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', return_value=objects.InstancePCIRequests(requests=[])) @@ -1039,7 +1090,8 @@ def test_some_instances_source_and_dest_migration(self, get_mock, get_mig_ctxt_mock, pci_mock, instance_pci_mock, - bfv_check_mock): + bfv_check_mock, + bdm_by_instance_mock): # We test the behavior of update_available_resource() when # there is an active migration that involves this compute node # as the destination host AND the source host, and the resource @@ -1158,6 +1210,8 @@ def test_no_instances_err_migration( obj_base.obj_equal_prims(expected_resources, actual_resources)) update_mock.assert_called_once() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', new=mock.Mock(return_value=False)) @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', @@ -1171,7 +1225,8 @@ def test_no_instances_err_migration( def test_populate_assigned_resources(self, mock_get_instances, mock_get_instance, mock_get_migrations, - mock_get_cn): + mock_get_cn, + bdm_by_instance_mock): # when update_available_resources, rt.assigned_resources # will be populated, resources assigned to tracked migrations # and instances will be tracked in rt.assigned_resources. @@ -1211,6 +1266,8 @@ def test_populate_assigned_resources(self, mock_get_instances, self.assertEqual(expected_assigned_resources, self.rt.assigned_resources) + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', new=mock.Mock(return_value=False)) @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', @@ -1224,7 +1281,8 @@ def test_populate_assigned_resources(self, mock_get_instances, def test_check_resources_startup_success(self, mock_get_instances, mock_get_instance, mock_get_migrations, - mock_get_cn): + mock_get_cn, + bdms_by_instance_mock): # When update_available_resources is running on startup, # it will trigger this function to check if there are # assigned resources not in provider tree. If so, the reason @@ -1255,6 +1313,8 @@ def test_check_resources_startup_success(self, mock_get_instances, update_mock = self._update_available_resources(startup=True) update_mock.assert_called_once() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', new=mock.Mock(return_value=False)) @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', @@ -1264,7 +1324,8 @@ def test_check_resources_startup_success(self, mock_get_instances, @mock.patch('nova.objects.InstanceList.get_by_host_and_node') def test_check_resources_startup_fail(self, mock_get_instances, mock_get_migrations, - mock_get_cn): + mock_get_cn, + bdms_by_uuid_mock): # Similar to testcase test_check_resources_startup_success, # and this one is for check_resources failed resource = objects.Resource(provider_uuid=self.compute.uuid, @@ -2728,6 +2789,8 @@ def test_claim_numa(self, save_mock, migr_mock, check_bfv_mock): class TestResize(BaseTestCase): @mock.patch('nova.compute.resource_tracker.ResourceTracker.' '_sync_compute_service_disabled_trait', new=mock.Mock()) + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', return_value=False) @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', @@ -2740,7 +2803,7 @@ class TestResize(BaseTestCase): @mock.patch('nova.objects.ComputeNode.save') def test_resize_claim_same_host(self, save_mock, get_mock, migr_mock, get_cn_mock, pci_mock, instance_pci_mock, - is_bfv_mock): + is_bfv_mock, bdms_by_uuid_mock): # Resize an existing instance from its current flavor (instance type # 1) to a new flavor (instance type 2) and verify that the compute # node's resources are appropriately updated to account for the new @@ -3307,6 +3370,8 @@ def test_resize_claim_two_instances(self, save_mock, get_mock, migr_mock, class TestRebuild(BaseTestCase): @mock.patch('nova.compute.resource_tracker.ResourceTracker.' '_sync_compute_service_disabled_trait', new=mock.Mock()) + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', return_value=objects.InstancePCIRequests(requests=[])) @@ -3317,7 +3382,8 @@ class TestRebuild(BaseTestCase): @mock.patch('nova.objects.InstanceList.get_by_host_and_node') @mock.patch('nova.objects.ComputeNode.save') def test_rebuild_claim(self, save_mock, get_mock, migr_mock, get_cn_mock, - pci_mock, instance_pci_mock, bfv_check_mock): + pci_mock, instance_pci_mock, bfv_check_mock, + bdms_by_instance_mock): # Rebuild an instance, emulating an evacuate command issued against the # original instance. The rebuild operation uses the resource tracker's # _move_claim() method, but unlike with resize_claim(), rebuild_claim() @@ -3640,16 +3706,18 @@ def setUp(self): self.rt.compute_nodes[_NODENAME] = cn self.instance = _INSTANCE_FIXTURES[0].obj_clone() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') def test_get_usage_dict_return_0_root_gb_for_bfv_instance( - self, mock_check_bfv): + self, mock_check_bfv, mock_batch_bdms): mock_check_bfv.return_value = True # Make sure the cache is empty. self.assertNotIn(self.instance.uuid, self.rt.is_bfv) result = self.rt._get_usage_dict(self.instance, self.instance) self.assertEqual(0, result['root_gb']) mock_check_bfv.assert_called_once_with( - self.instance._context, self.instance) + self.instance._context, self.instance, bdms=None) # Make sure we updated the cache. self.assertIn(self.instance.uuid, self.rt.is_bfv) self.assertTrue(self.rt.is_bfv[self.instance.uuid]) @@ -4029,18 +4097,40 @@ def test_update_usage_from_instances_goes_negative(self): # NOTE(danms): The resource tracker _should_ report negative resources # for things like free_ram_mb if overcommit is being used. This test # ensures that we don't collapse negative values to zero. + # NOTE(jkulik): We've switched how this works at SAP; because we don't + # want limes to see the memory of a compute-node if it's actually + # reserved (as limes didn't care for free/used at the time). Therefore, + # we actually collapse reserved memory to 0 for reserved resources, but + # we don't do it for instance resources. self.flags(reserved_host_memory_mb=2048) self.flags(reserved_host_disk_mb=(11 * 1024)) cn = objects.ComputeNode(memory_mb=1024, local_gb=10) + resources = {'hypervisor_hostname': 'foo', 'memory_mb': 1024, + 'local_gb': 10} + self.rt._copy_resources(cn, resources) self.rt.compute_nodes['foo'] = cn + @mock.patch('nova.objects.BlockDeviceMappingList' + '.bdms_by_instance_uuid', + return_value={}) @mock.patch.object(self.rt, '_update_usage_from_instance') - def test(uufi): - self.rt._update_usage_from_instances('ctxt', [], 'foo') + @mock.patch('nova.objects.Service.get_minimum_version', + return_value=22) + def test(version_mock, uufi, bdms): + def _update_usage(*args, **kwargs): + # simulate an instance with 512 MB memory and 1 GB disk using + # resources + cn.memory_mb_used += 512 + cn.local_gb_used += 1 + cn.free_ram_mb = cn.memory_mb - cn.memory_mb_used + cn.free_disk_gb = cn.local_gb - cn.local_gb_used + uufi.side_effect = _update_usage + self.rt._update_usage_from_instances('ctxt', [self.instance], + 'foo') test() - self.assertEqual(-1024, cn.free_ram_mb) + self.assertEqual(-512, cn.free_ram_mb) self.assertEqual(-1, cn.free_disk_gb) def test_delete_allocation_for_evacuated_instance(self): diff --git a/nova/tests/unit/conductor/test_conductor.py b/nova/tests/unit/conductor/test_conductor.py index ae511439381..dce7b9047ff 100644 --- a/nova/tests/unit/conductor/test_conductor.py +++ b/nova/tests/unit/conductor/test_conductor.py @@ -402,15 +402,17 @@ def _prepare_rebuild_args(self, update_args=None): return rebuild_args, compute_rebuild_args @mock.patch.object(objects.InstanceMapping, 'get_by_instance_uuid') + @mock.patch.object(objects.Instance, 'get_bdms') @mock.patch.object(objects.RequestSpec, 'save') @mock.patch.object(migrate.MigrationTask, 'execute') @mock.patch.object(utils, 'get_image_from_system_metadata') @mock.patch.object(objects.RequestSpec, 'from_components') def _test_cold_migrate(self, spec_from_components, get_image_from_metadata, - migration_task_execute, spec_save, get_im, + migration_task_execute, spec_save, get_bdms, get_im, clean_shutdown=True): get_im.return_value.cell_mapping = ( objects.CellMappingList.get_all(self.context)[0]) + get_bdms.return_value = [] get_image_from_metadata.return_value = 'image' inst = fake_instance.fake_db_instance(image_ref='image_ref') inst_obj = objects.Instance._from_db_object( @@ -3748,6 +3750,7 @@ def test_set_vm_state_and_notify(self, mock_set): 'request_spec') @mock.patch.object(objects.InstanceMapping, 'get_by_instance_uuid') + @mock.patch.object(objects.Instance, 'get_bdms') @mock.patch.object(objects.RequestSpec, 'from_components') @mock.patch.object(scheduler_utils, 'setup_instance_group') @mock.patch.object(utils, 'get_image_from_system_metadata') @@ -3758,7 +3761,8 @@ def test_set_vm_state_and_notify(self, mock_set): @mock.patch.object(migrate.MigrationTask, '_preallocate_migration') def test_cold_migrate_no_valid_host_back_in_active_state( self, _preallocate_migration, rollback_mock, notify_mock, - select_dest_mock, metadata_mock, sig_mock, spec_fc_mock, im_mock): + select_dest_mock, metadata_mock, sig_mock, spec_fc_mock, bdms_mock, + im_mock): inst_obj = objects.Instance( image_ref='fake-image_ref', instance_type_id=self.flavor.id, @@ -3782,6 +3786,7 @@ def test_cold_migrate_no_valid_host_back_in_active_state( updates = {'vm_state': vm_states.ACTIVE, 'task_state': None} + bdms_mock.return_value = [] im_mock.return_value = objects.InstanceMapping( cell_mapping=objects.CellMapping.get_by_uuid(self.context, uuids.cell1)) @@ -3800,6 +3805,7 @@ def test_cold_migrate_no_valid_host_back_in_active_state( rollback_mock.assert_called_once_with(exc_info) @mock.patch.object(objects.InstanceMapping, 'get_by_instance_uuid') + @mock.patch.object(objects.Instance, 'get_bdms') @mock.patch.object(scheduler_utils, 'setup_instance_group') @mock.patch.object(objects.RequestSpec, 'from_components') @mock.patch.object(utils, 'get_image_from_system_metadata') @@ -3810,7 +3816,8 @@ def test_cold_migrate_no_valid_host_back_in_active_state( @mock.patch.object(migrate.MigrationTask, '_preallocate_migration') def test_cold_migrate_no_valid_host_back_in_stopped_state( self, _preallocate_migration, rollback_mock, notify_mock, - select_dest_mock, metadata_mock, spec_fc_mock, sig_mock, im_mock): + select_dest_mock, metadata_mock, spec_fc_mock, sig_mock, bdms_mock, + im_mock): inst_obj = objects.Instance( image_ref='fake-image_ref', vm_state=vm_states.STOPPED, @@ -3830,6 +3837,7 @@ def test_cold_migrate_no_valid_host_back_in_stopped_state( fake_spec = objects.RequestSpec(image=objects.ImageMeta()) spec_fc_mock.return_value = fake_spec + bdms_mock.return_value = [] im_mock.return_value = objects.InstanceMapping( cell_mapping=objects.CellMapping.get_by_uuid(self.context, uuids.cell1)) @@ -3872,15 +3880,17 @@ def test_cold_migrate_no_valid_host_error_msg(self): mock.patch.object(migrate.MigrationTask, 'execute', side_effect=exc.NoValidHost(reason="")), - mock.patch.object(migrate.MigrationTask, 'rollback') + mock.patch.object(migrate.MigrationTask, 'rollback'), + mock.patch.object(inst_obj, 'get_bdms', return_value=[]) ) as (image_mock, set_vm_mock, task_execute_mock, - task_rollback_mock): + task_rollback_mock, inst_bdms_mock): nvh = self.assertRaises(exc.NoValidHost, self.conductor._cold_migrate, self.context, inst_obj, self.flavor, {}, True, fake_spec, None) self.assertIn('cold migrate', nvh.message) + @mock.patch.object(objects.Instance, 'get_bdms') @mock.patch.object(utils, 'get_image_from_system_metadata') @mock.patch.object(migrate.MigrationTask, 'execute') @mock.patch.object(migrate.MigrationTask, 'rollback') @@ -3892,7 +3902,8 @@ def test_cold_migrate_no_valid_host_in_group(self, set_vm_mock, task_rollback_mock, task_exec_mock, - image_mock): + image_mock, + bdms_mock): inst_obj = objects.Instance( image_ref='fake-image_ref', vm_state=vm_states.STOPPED, @@ -3912,6 +3923,7 @@ def test_cold_migrate_no_valid_host_in_group(self, fake_spec = fake_request_spec.fake_spec_obj() spec_fc_mock.return_value = fake_spec + bdms_mock.return_value = [] image_mock.return_value = image task_exec_mock.side_effect = exception @@ -3924,7 +3936,8 @@ def test_cold_migrate_no_valid_host_in_group(self, 'migrate_server', updates, exception, fake_spec) filter_properties = {'scheduler_hints': {'source_host': ['host1'], - 'source_node': ['node1']}} + 'source_node': ['node1'], + 'volume_sizes': []}} spec_fc_mock.assert_called_once_with( self.context, inst_obj.uuid, image, self.flavor, inst_obj.numa_topology, inst_obj.pci_requests, filter_properties, @@ -3935,6 +3948,7 @@ def test_cold_migrate_no_valid_host_in_group(self, '_is_selected_host_in_source_cell', return_value=True) @mock.patch.object(objects.InstanceMapping, 'get_by_instance_uuid') + @mock.patch.object(objects.Instance, 'get_bdms') @mock.patch.object(scheduler_utils, 'setup_instance_group') @mock.patch.object(objects.RequestSpec, 'from_components') @mock.patch.object(utils, 'get_image_from_system_metadata') @@ -3947,7 +3961,7 @@ def test_cold_migrate_no_valid_host_in_group(self, def test_cold_migrate_exception_host_in_error_state_and_raise( self, _preallocate_migration, prep_resize_mock, rollback_mock, notify_mock, select_dest_mock, metadata_mock, spec_fc_mock, - sig_mock, im_mock, check_cell_mock): + sig_mock, bdms_mock, im_mock, check_cell_mock): inst_obj = objects.Instance( image_ref='fake-image_ref', vm_state=vm_states.STOPPED, @@ -3966,6 +3980,7 @@ def test_cold_migrate_exception_host_in_error_state_and_raise( fake_spec = objects.RequestSpec(image=objects.ImageMeta()) spec_fc_mock.return_value = fake_spec + bdms_mock.return_value = [] im_mock.return_value = objects.InstanceMapping( cell_mapping=objects.CellMapping.get_by_uuid(self.context, uuids.cell1)) @@ -4033,13 +4048,15 @@ def test_cold_migrate_exception_instance_refresh_not_found( mock_execute.assert_called_once_with() mock_refresh.assert_called_once_with() + @mock.patch.object(objects.Instance, 'get_bdms') @mock.patch.object(objects.RequestSpec, 'save') @mock.patch.object(migrate.MigrationTask, 'execute') @mock.patch.object(utils, 'get_image_from_system_metadata') def test_cold_migrate_updates_flavor_if_existing_reqspec(self, image_mock, task_exec_mock, - spec_save_mock): + spec_save_mock, + bdms_mock): inst_obj = objects.Instance( image_ref='fake-image_ref', vm_state=vm_states.STOPPED, @@ -4056,6 +4073,7 @@ def test_cold_migrate_updates_flavor_if_existing_reqspec(self, image = 'fake-image' fake_spec = fake_request_spec.fake_spec_obj() + bdms_mock.return_value = [] image_mock.return_value = image # Just make sure we have an original flavor which is different from # the new one @@ -4068,10 +4086,11 @@ def test_cold_migrate_updates_flavor_if_existing_reqspec(self, # ...and persisted spec_save_mock.assert_called_once_with() + @mock.patch.object(objects.Instance, 'get_bdms') @mock.patch('nova.objects.RequestSpec.from_primitives') @mock.patch.object(objects.RequestSpec, 'save') def test_cold_migrate_reschedule_legacy_request_spec( - self, spec_save_mock, from_primitives_mock): + self, spec_save_mock, from_primitives_mock, bdms_mock): """Tests the scenario that compute RPC API is pinned to less than 5.1 so conductor passes a legacy dict request spec to compute and compute sends it back to conductor on a reschedule during prep_resize so @@ -4079,6 +4098,7 @@ def test_cold_migrate_reschedule_legacy_request_spec( """ instance = objects.Instance(system_metadata={}) fake_spec = fake_request_spec.fake_spec_obj() + bdms_mock.return_value = [] from_primitives_mock.return_value = fake_spec legacy_spec = fake_spec.to_legacy_request_spec_dict() filter_props = {} @@ -4122,9 +4142,10 @@ def test_resize_no_valid_host_error_msg(self): mock.patch.object(migrate.MigrationTask, 'execute', side_effect=exc.NoValidHost(reason="")), - mock.patch.object(migrate.MigrationTask, 'rollback') + mock.patch.object(migrate.MigrationTask, 'rollback'), + mock.patch.object(inst_obj, 'get_bdms', return_value=[]) ) as (image_mock, brs_mock, vm_st_mock, task_execute_mock, - task_rb_mock): + task_rb_mock, inst_bdms_mock): nvh = self.assertRaises(exc.NoValidHost, self.conductor._cold_migrate, self.context, inst_obj, flavor_new, {}, diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py index bde7fd3f5ea..8b6adca5837 100644 --- a/nova/tests/unit/notifications/objects/test_notification.py +++ b/nova/tests/unit/notifications/objects/test_notification.py @@ -375,7 +375,7 @@ def test_payload_is_not_generated_if_notification_format_is_unversioned( 'ComputeTaskNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'ComputeTaskPayload': '1.0-59a9b78b6199470a83c8d07bffd13f5e', 'DestinationPayload': '1.0-d8faf610201bf5f460892243f6632a37', - 'EventType': '1.22-fab37f98c6e7513edece32ad5420f549', + 'EventType': '1.22-23923a7e77f1a188085deb00fe5e850b', 'ExceptionNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'ExceptionPayload': '1.1-34f006107693b8c9eaf4b104157d21b4', 'FlavorNotification': '1.0-a73147b93b520ff0061865849d3dfa56', @@ -426,7 +426,7 @@ def test_payload_is_not_generated_if_notification_format_is_unversioned( 'MetricPayload': '1.0-6a932f141c9ae2f50fc9d79c548338aa', 'MetricsNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'MetricsPayload': '1.0-a087851790cd7e76883bdc64a146917a', - 'NotificationPublisher': '2.2-b6ad48126247e10b46b6b0240e52e614', + 'NotificationPublisher': '2.2-ff8ef16673817ca7a3ea69c689e260c6', 'RequestSpecPayload': '1.1-9530d710bf7eaa101a93f6745fbe7aea', 'SchedulerRetriesPayload': '1.0-6e7e6204e638c0a070412f0e765c320c', 'SelectDestinationsNotification': '1.0-a73147b93b520ff0061865849d3dfa56', diff --git a/nova/tests/unit/objects/test_instance_group.py b/nova/tests/unit/objects/test_instance_group.py index 5ea566fea7e..b6143b8127e 100644 --- a/nova/tests/unit/objects/test_instance_group.py +++ b/nova/tests/unit/objects/test_instance_group.py @@ -363,7 +363,35 @@ def test_get_by_hint(self, mock_name, mock_uuid): class TestInstanceGroupObject(test_objects._LocalTest, _TestInstanceGroupObject): - pass + def test_get_all_with_limits(self): + entries = {'entry1': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg1, + 'name': 'sg1'}, + 'entry2': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg2, + 'name': 'sg2'}, + 'entry3': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg3, + 'name': 'sg3'}, + 'entry4': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg4, + 'name': 'sg4'}} + for key, value in entries.items(): + objects.instance_group.InstanceGroup._create_in_db(self.context, + value) + + servg = objects.InstanceGroupList.get_all(self.context, + limit=2, + offset=1) + self.assertEqual(2, len(servg)) + servg = objects.InstanceGroupList.get_all(self.context, + limit=2, + offset=3) + self.assertEqual(1, len(servg)) class TestRemoteInstanceGroupObject(test_objects._RemoteTest, @@ -393,7 +421,9 @@ def test_list_all(self, mock_api_get): mock_api_get.side_effect = _mock_db_list_get inst_list = objects.InstanceGroupList.get_all(self.context) self.assertEqual(4, len(inst_list.objects)) - mock_api_get.assert_called_once_with(self.context) + mock_api_get.assert_called_once_with(self.context, + limit=None, + offset=None) @mock.patch('nova.objects.InstanceGroupList._get_from_db') def test_list_by_project_id(self, mock_api_get): @@ -401,7 +431,8 @@ def test_list_by_project_id(self, mock_api_get): objects.InstanceGroupList.get_by_project_id( self.context, mock.sentinel.project_id) mock_api_get.assert_called_once_with( - self.context, project_id=mock.sentinel.project_id) + self.context, project_id=mock.sentinel.project_id, + limit=None, offset=None) class TestInstanceGroupListObject(test_objects._LocalTest, diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index fb3a8491457..f8aa712b284 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1115,8 +1115,8 @@ def obj_name(cls): 'InstanceExternalEvent': '1.5-1ec57351a9851c1eb43ccd90662d6dd0', 'InstanceFault': '1.2-7ef01f16f1084ad1304a513d6d410a38', 'InstanceFaultList': '1.2-6bb72de2872fe49ded5eb937a93f2451', - 'InstanceGroup': '1.11-852ac511d30913ee88f3c3a869a8f30a', - 'InstanceGroupList': '1.8-90f8f1a445552bb3bbc9fa1ae7da27d4', + 'InstanceGroup': '1.11-a26b476ba20883380476d4cb8c4b8d41', + 'InstanceGroupList': '1.8-b713c6273b2c468d11cd058a3b55c373', 'InstanceInfoCache': '1.5-cd8b96fefe0fc8d4d337243ba0bf0e1e', 'InstanceList': '2.6-238f125650c25d6d12722340d726f723', 'InstanceMapping': '1.2-3bd375e65c8eb9c45498d2f87b882e03', @@ -1183,7 +1183,7 @@ def obj_name(cls): 'VirtCPUTopology': '1.0-fc694de72e20298f7c6bab1083fd4563', 'VirtualInterface': '1.3-efd3ca8ebcc5ce65fff5a25f31754c54', 'VirtualInterfaceList': '1.0-9750e2074437b3077e46359102779fc6', - 'VMwareLiveMigrateData': '1.1-14a63b00f1b8880453df6d19a883bc73', + 'VMwareLiveMigrateData': '1.2-093beb7b7866660440a44c441166c0f3', 'VolumeUsage': '1.0-6c8190c46ce1469bb3286a1f21c2e475', 'XenDeviceBus': '1.0-272a4f899b24e31e42b2b9a7ed7e9194', } diff --git a/nova/tests/unit/scheduler/client/test_report.py b/nova/tests/unit/scheduler/client/test_report.py index ed43b7843da..6075e13a57c 100644 --- a/nova/tests/unit/scheduler/client/test_report.py +++ b/nova/tests/unit/scheduler/client/test_report.py @@ -2859,6 +2859,17 @@ def test_get_provider_aggregates_error(self, log_mock): self.ks_adap_mock.get.reset_mock() log_mock.reset_mock() + def test_get_provider_aggregates_conn_failure(self): + self.ks_adap_mock.get.side_effect = ks_exc.EndpointNotFound() + self.assertRaises( + ks_exc.ClientException, + self.client._get_provider_aggregates, self.context, uuids.cn) + expected_url = '/resource_providers/' + uuids.cn + '/aggregates' + + self.ks_adap_mock.get.assert_called_once_with( + expected_url, microversion='1.19', + global_request_id=self.context.global_id) + class TestTraits(SchedulerReportClientTestCase): trait_api_kwargs = {'microversion': '1.6'} diff --git a/nova/tests/unit/scheduler/filters/test_bigvm_filters.py b/nova/tests/unit/scheduler/filters/test_bigvm_filters.py new file mode 100644 index 00000000000..711151edb79 --- /dev/null +++ b/nova/tests/unit/scheduler/filters/test_bigvm_filters.py @@ -0,0 +1,179 @@ +# Copyright (c) 2019 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time +from unittest import mock + +import nova.conf +from nova import objects +from nova.scheduler.filters import bigvm_filter +from nova import test +from nova.tests.unit.scheduler import fakes +from oslo_utils.fixture import uuidsentinel + +CONF = nova.conf.CONF + + +@mock.patch('nova.scheduler.client.report.' + 'SchedulerReportClient._get_inventory') +class TestBigVmBaseFilter(test.NoDBTestCase): + + def setUp(self): + super(TestBigVmBaseFilter, self).setUp() + self.filt_cls = bigvm_filter.BigVmBaseFilter() + self.hv_size = CONF.bigvm_mb + 1024 + + def test_big_vm_host_without_inventory(self, mock_inv): + mock_inv.return_value = {} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.assertIsNone(self.filt_cls._get_hv_size(host)) + + def test_big_vm_host_with_placement_error(self, mock_inv): + mock_inv.return_value = None + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.assertIsNone(self.filt_cls._get_hv_size(host)) + + def test_big_vm_host_with_empty_inventory(self, mock_inv): + mock_inv.return_value = {'inventories': {}} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.assertIsNone(self.filt_cls._get_hv_size(host)) + + def test_big_vm_get_hv_size_with_cache(self, mock_inv): + mock_inv.return_value = {} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.filt_cls._HV_SIZE_CACHE = { + host.uuid: 1234, + 'last_modified': time.time() + } + self.assertEqual(self.filt_cls._get_hv_size(host), 1234) + + def test_big_vm_get_hv_size_cache_timeout(self, mock_inv): + mock_inv.return_value = {'inventories': {'MEMORY_MB': + {'max_unit': 23}}} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + mod = time.time() - self.filt_cls._HV_SIZE_CACHE_RETENTION_TIME + self.filt_cls._HV_SIZE_CACHE = { + host.uuid: 1234, + 'last_modified': mod + } + self.assertEqual(self.filt_cls._get_hv_size(host), 23) + + +class TestBigVmClusterUtilizationFilter(test.NoDBTestCase): + + def setUp(self): + super(TestBigVmClusterUtilizationFilter, self).setUp() + self.hv_size = CONF.bigvm_mb + 1024 + self.filt_cls = bigvm_filter.BigVmClusterUtilizationFilter() + self.filt_cls._HV_SIZE_CACHE = { + uuidsentinel.host1: self.hv_size, + 'last_modified': time.time() + } + + def test_big_vm_with_small_vm_passes(self): + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=1024, extra_specs={})) + host = fakes.FakeHostState('host1', 'compute', {}) + self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + + def test_baremetal_instance_passes(self): + extra_specs = {'capabilities:cpu_arch': 'x86_64'} + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, + extra_specs=extra_specs)) + host = fakes.FakeHostState('host1', 'compute', {}) + self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_hv_size(self): + """If there's no inventory for this host, it should not even have + passed placement API checks, so we stop it here. + """ + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1}) + self.filt_cls._HV_SIZE_CACHE[host.uuid] = None + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_enough_ram(self): + # there's enough RAM available in the cluster but not enough (~50 % of + # the requested size on average + # 12 hosts (bigvm + 1 GB size) + # 11 big VM + some smaller (12 * 1 GB) already deployed + # -> still bigvm_mb left, but ram utilization ratio of all hosts is too + # high + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': CONF.bigvm_mb, + 'total_usable_ram_mb': total_ram}) + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_enough_ram_ignores_ram_ratio(self): + # same as test_big_vm_without_enough_ram but with more theoretical RAM + # via `ram_allocation_ratio`. big VMs reserve all memory so the ratio + # does not count for them. + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': CONF.bigvm_mb, + 'total_usable_ram_mb': total_ram, + 'ram_allocation_ratio': 1.5}) + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_enough_ram_percent(self): + # there's just closely not enough RAM available + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + hv_percent = self.filt_cls._get_max_ram_percent(CONF.bigvm_mb, + self.hv_size) + free_ram_mb = total_ram - (total_ram * hv_percent / 100.0) - 128 + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': free_ram_mb, + 'total_usable_ram_mb': total_ram}) + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_with_enough_ram(self): + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + hv_percent = self.filt_cls._get_max_ram_percent(CONF.bigvm_mb, + self.hv_size) + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': total_ram - (total_ram * hv_percent / 100.0), + 'total_usable_ram_mb': total_ram}) + self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) diff --git a/nova/tests/unit/scheduler/filters/test_shard_filter.py b/nova/tests/unit/scheduler/filters/test_shard_filter.py index 8c6edcce34e..41c7cd85808 100644 --- a/nova/tests/unit/scheduler/filters/test_shard_filter.py +++ b/nova/tests/unit/scheduler/filters/test_shard_filter.py @@ -16,11 +16,14 @@ from unittest import mock +import nova.conf from nova import objects from nova.scheduler.filters import shard_filter from nova import test from nova.tests.unit.scheduler import fakes +CONF = nova.conf.CONF + class TestShardFilter(test.NoDBTestCase): @@ -234,3 +237,69 @@ def test_non_vmware_spec(self, mock_is_non_vmware_spec): self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) mock_is_non_vmware_spec.assert_called_once_with(spec_obj) + + def test_ignores_hana_by_trait(self): + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'required'} + self._shard_filter_ignoring_hana(True, extra_specs) + + def test_ignores_hana_by_memory(self): + CONF.set_override('bigvm_mb', 8192) + CONF.set_override('hana_detection_strategy', 'memory_mb', + 'filter_scheduler') + extra_specs = {} + self._shard_filter_ignoring_hana(True, + extra_specs=extra_specs, + memory_mb=8192) + + def test_sharding_ignore_hana_config(self): + CONF.set_override('sharding_ignore_hana', False, 'filter_scheduler') + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'required'} + self._shard_filter_ignoring_hana(False, + extra_specs=extra_specs) + + def test_use_individual_shard_tags_hana_tag(self): + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'required'} + tag = 'use_individual_shard_tags_hana' + self._shard_filter_ignoring_hana(False, + extra_specs=extra_specs, + project_tag=tag) + + def test_allow_hana_by_trait_forbidden(self): + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'forbidden'} + self._shard_filter_ignoring_hana(False, extra_specs) + + def test_allow_hana_by_memory(self): + CONF.set_override('bigvm_mb', 8192) + CONF.set_override('hana_detection_strategy', 'memory_mb', + 'filter_scheduler') + extra_specs = {} + self._shard_filter_ignoring_hana(False, + extra_specs=extra_specs, + memory_mb=8191) + + def _shard_filter_ignoring_hana(self, expect_to_ignore, extra_specs=None, + project_tag=None, memory_mb=4096): + aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1', + 'host2']), + objects.Aggregate(id=1, name='vc-a-1', hosts=['host1'])] + + host_allow = fakes.FakeHostState( + 'host1', 'compute', {'aggregates': aggs}) + host_forbid = fakes.FakeHostState( + 'host2', 'compute', {'aggregates': aggs[:1]}) + + spec_obj = objects.RequestSpec( + context=mock.sentinel.ctx, project_id='foo', + flavor=objects.Flavor(extra_specs=extra_specs, + memory_mb=memory_mb)) + + self.filt_cls._PROJECT_TAG_CACHE['foo'] = ['vc-a-0', 'vc-a-1', + 'vc-b-0'] + if project_tag: + self.filt_cls._PROJECT_TAG_CACHE['foo'].append(project_tag) + result = list(self.filt_cls.filter_all( + [host_allow, host_forbid], spec_obj)) + if expect_to_ignore: + self.assertEqual(result, [host_allow, host_forbid]) + else: + self.assertEqual(result, [host_allow]) diff --git a/nova/tests/unit/test_policy.py b/nova/tests/unit/test_policy.py index 19f3165717a..519cf21ab8e 100644 --- a/nova/tests/unit/test_policy.py +++ b/nova/tests/unit/test_policy.py @@ -471,6 +471,7 @@ def setUp(self): "os_compute_api:os-server-groups:show", "os_compute_api:os-server-groups:create", "os_compute_api:os-server-groups:delete", +"os_compute_api:os-server-groups:update", "os_compute_api:os-shelve:shelve", "os_compute_api:os-shelve:unshelve", "os_compute_api:os-volumes:create", diff --git a/nova/tests/unit/test_utils.py b/nova/tests/unit/test_utils.py index b34136b0522..440ca0ed1d4 100644 --- a/nova/tests/unit/test_utils.py +++ b/nova/tests/unit/test_utils.py @@ -16,6 +16,7 @@ import hashlib import os import os.path +import string import tempfile from unittest import mock @@ -31,6 +32,7 @@ from oslo_utils import encodeutils from oslo_utils import fixture as utils_fixture +from nova.conf import base as conf_base from nova import context from nova import exception from nova.objects import base as obj_base @@ -140,11 +142,40 @@ def test_hostname_truncated_no_hyphen(self): def test_generate_password(self): password = utils.generate_password() - self.assertTrue([c for c in password if c in '0123456789']) - self.assertTrue([c for c in password - if c in 'abcdefghijklmnopqrstuvwxyz']) - self.assertTrue([c for c in password - if c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ']) + for group in conf_base._DEFAULT_PASSWORD_SYMBOLS: + if group: + self.assertTrue([c for c in password if c in group]) + + @mock.patch.object(CONF, "password_all_group_samples", 0) + def test_generate_password_zero_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + + @mock.patch.object(CONF, "password_all_group_samples", 2) + def test_generate_password_two_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + for group in conf_base._DEFAULT_PASSWORD_SYMBOLS: + if group: + self.assertGreaterEqual( + len([c for c in password if c in group]), + CONF.password_all_group_samples) + + @mock.patch.object(CONF, "password_all_group_samples", 7) + @mock.patch.object(CONF, "password_length", 13) + def test_generate_password_too_many_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + + @mock.patch.object(CONF, "password_symbol_groups", [string.ascii_lowercase, + string.ascii_uppercase, string.punctuation]) + def test_generate_password_custom_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + self.assertGreaterEqual( + len([c for c in password if c in string.punctuation]), + CONF.password_all_group_samples) + self.assertFalse([c for c in password if c in string.digits]) @mock.patch('nova.privsep.path.chown') def test_temporary_chown(self, mock_chown): diff --git a/nova/tests/unit/virt/vmwareapi/fake.py b/nova/tests/unit/virt/vmwareapi/fake.py index 8f9b109dd75..5685d8ca6f8 100644 --- a/nova/tests/unit/virt/vmwareapi/fake.py +++ b/nova/tests/unit/virt/vmwareapi/fake.py @@ -438,6 +438,7 @@ def __init__(self, **kwargs): self.set("summary.config.memorySizeMB", kwargs.get("mem", 1)) self.set("summary.config.instanceUuid", kwargs.get("instanceUuid")) self.set("version", kwargs.get("version")) + self.set("resourcePool", kwargs.get("resourcePool")) devices = _create_array_of_type('VirtualDevice') devices.VirtualDevice = kwargs.get("virtual_device", []) @@ -741,10 +742,90 @@ def __init__(self): class HostSystem(ManagedObject): """Host System class.""" + NUM_CPU_CORES = 8 + NUM_CPU_PKGS = 2 + NUM_CPU_THREADS = 16 - def __init__(self, name="ha-host", connected=True, ds_ref=None, + @staticmethod + def create_summary_hardware(): + hardware = DataObject() + hardware.numCpuCores = HostSystem.NUM_CPU_CORES + hardware.numCpuPkgs = HostSystem.NUM_CPU_PKGS + hardware.numCpuThreads = HostSystem.NUM_CPU_THREADS + hardware.vendor = "Intel" + hardware.cpuModel = "Intel(R) Xeon(R)" + hardware.uuid = "host-uuid" + hardware.memorySize = units.Gi + return hardware + + @staticmethod + def create_summary(connected, maintenance_mode): + summary = DataObject() + summary.hardware = HostSystem.create_summary_hardware() + summary.runtime = HostSystem.create_summary_runtime(connected, + maintenance_mode) + quickstats = DataObject() + quickstats.overallMemoryUsage = 500 + summary.quickStats = quickstats + + product = DataObject() + product.name = "VMware ESXi" + # Avoid deprecation warning + product.version = constants.NEXT_MIN_VC_VERSION + config = DataObject() + config.product = product + summary.config = config + + return summary + + @staticmethod + def create_summary_runtime(connected, maintenance_mode): + runtime = DataObject() + if connected: + runtime.connectionState = "connected" + else: + runtime.connectionState = "disconnected" + + runtime.inMaintenanceMode = maintenance_mode + return runtime + + @staticmethod + def create_cpu_pkgs(): + cpuPkg = DataObject('HostCpuPackage') + cpuPkg.description = 'Intel(R) Xeon(R) CPU E5-4650 v4 @ 2.20GHz' + cpuPkg.vendor = 'intel' + cpu_pkgs = _create_array_of_type('HostCpuPackage') + cpu_pkgs.HostCpuPackage = HostSystem.NUM_CPU_PKGS * [cpuPkg] + return cpu_pkgs + + @staticmethod + def create_hardware_cpu_info(): + cpuInfo = DataObject('CpuInfo') + cpuInfo.numCpuCores = HostSystem.NUM_CPU_CORES + cpuInfo.numCpuPackages = HostSystem.NUM_CPU_PKGS + cpuInfo.numCpuThreads = HostSystem.NUM_CPU_THREADS + return cpuInfo + + @staticmethod + def create_config_feature_capability(): + aes = DataObject('HostFeatureCapability') + aes.featureName = 'cpuid.AES' + aes.value = '1' + avx = DataObject('HostFeatureCapability') + avx.featureName = 'cpuid.AVX' + avx.value = '1' + amd = DataObject('HostFeatureCapability') + amd.featureName = 'cpuid.AMD' + amd.value = '0' + caps = _create_array_of_type('HostFeatureCapability') + caps.HostFeatureCapability = [aes, avx, amd] + return caps + + def __init__(self, name=None, connected=True, ds_ref=None, maintenance_mode=False): super(HostSystem, self).__init__("host") + if not name: + name = "ha-{}".format(self.mo_id) self.set("name", name) if _no_objects_of_type("HostNetworkSystem"): create_host_network_system() @@ -765,50 +846,31 @@ def __init__(self, name="ha-host", connected=True, ds_ref=None, datastores.ManagedObjectReference = [ds_ref] self.set("datastore", datastores) - summary = DataObject() - hardware = DataObject() - hardware.numCpuCores = 8 - hardware.numCpuPkgs = 2 - hardware.numCpuThreads = 16 - hardware.vendor = "Intel" - hardware.cpuModel = "Intel(R) Xeon(R)" - hardware.uuid = "host-uuid" - hardware.memorySize = units.Gi - summary.hardware = hardware - - runtime = DataObject() - if connected: - runtime.connectionState = "connected" - else: - runtime.connectionState = "disconnected" - - runtime.inMaintenanceMode = maintenance_mode - - summary.runtime = runtime - - quickstats = DataObject() - quickstats.overallMemoryUsage = 500 - summary.quickStats = quickstats - - product = DataObject() - product.name = "VMware ESXi" - # Avoid deprecation warning - product.version = constants.NEXT_MIN_VC_VERSION - config = DataObject() - config.product = product - summary.config = config + caps = self.create_config_feature_capability() + self.set('config.featureCapability', caps) pnic_do = DataObject() pnic_do.device = "vmnic0" net_info_pnic = DataObject() net_info_pnic.PhysicalNic = [pnic_do] + self.set("config.network.pnic", net_info_pnic) + + cpu_pkgs = self.create_cpu_pkgs() + self.set('hardware.cpuPkg', cpu_pkgs) + + cpu_info = self.create_hardware_cpu_info() + self.set('hardware.cpuInfo', cpu_info) + + summary = self.create_summary(connected, maintenance_mode) self.set("summary", summary) + self.set("summary.config", summary.config) + self.set("summary.config.product", summary.config.product) + self.set("summary.hardware", summary.hardware) + self.set("summary.runtime", summary.runtime) + self.set("summary.quickStats", summary.quickStats) + self.set("capability.maxHostSupportedVcpus", 600) - self.set("summary.hardware", hardware) - self.set("summary.runtime", runtime) - self.set("summary.quickStats", quickstats) - self.set("config.network.pnic", net_info_pnic) self.set("connected", connected) if _no_objects_of_type("Network"): @@ -1058,6 +1120,7 @@ def create_vm(uuid=None, name=None, "extra_config": extraConfig, "virtual_device": devices, "instanceUuid": uuid, + "resourcePool": res_pool_ref, "version": version} vm = VirtualMachine(**vm_dict) _create_object(vm) diff --git a/nova/tests/unit/virt/vmwareapi/test_configdrive.py b/nova/tests/unit/virt/vmwareapi/test_configdrive.py index 7e8b1c1b63c..fb700f1fc5b 100644 --- a/nova/tests/unit/virt/vmwareapi/test_configdrive.py +++ b/nova/tests/unit/virt/vmwareapi/test_configdrive.py @@ -33,7 +33,7 @@ from nova.virt.vmwareapi import vmops -class ConfigDriveTestCase(test.NoDBTestCase): +class ConfigDriveTestCase(test.TestCase): REQUIRES_LOCKING = True @@ -52,7 +52,8 @@ def setUp(self, mock_register, mock_service): vmwareapi_fake.reset() stubs.set_stubs(self) self.glance = self.useFixture(nova_fixtures.GlanceFixture(self)) - self.conn = driver.VMwareVCDriver(fake.FakeVirtAPI) + virtapi = fake.FakeComputeVirtAPI(mock.MagicMock()) + self.conn = driver.VMwareVCDriver(virtapi) self.network_info = utils.get_test_network_info() self.node_name = self.conn._nodename image_ref = self.glance.auto_disk_config_enabled_image['id'] @@ -93,6 +94,7 @@ def setUp(self, mock_register, mock_service): 'id': image_ref, 'disk_format': 'vmdk', 'size': int(metadata['size']), + 'owner': '', }) class FakeInstanceMetadata(object): @@ -123,9 +125,17 @@ def tearDown(self): super(ConfigDriveTestCase, self).tearDown() vmwareapi_fake.cleanup() + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') + @mock.patch('nova.utils.vm_needs_special_spawning') @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata', return_value='fake_metadata') - def _spawn_vm(self, fake_get_instance_meta, + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) + def _spawn_vm(self, fake_fetch_image_from_other_datastores, + fake_find_image_template_vm, fake_get_instance_meta, + mock_vm_special_spawning, mock_update_cluster_placement, injected_files=None, admin_password=None, block_device_info=None): diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 7f8dec8a619..7d76e5c2981 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -21,6 +21,7 @@ import collections import datetime +import fixtures from unittest import mock from eventlet import greenthread @@ -40,6 +41,8 @@ from nova.compute import task_states from nova.compute import vm_states import nova.conf +import nova.utils + from nova import context from nova import exception from nova.image import glance @@ -55,6 +58,7 @@ from nova.tests.unit.virt.vmwareapi import fake as vmwareapi_fake from nova.tests.unit.virt.vmwareapi import stubs from nova.virt import driver as v_driver +from nova.virt import fake from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import driver from nova.virt.vmwareapi import ds_util @@ -162,7 +166,8 @@ def _create_service(self, **kwargs): return objects.Service(**service_ref) @mock.patch.object(driver.VMwareVCDriver, '_register_openstack_extension') - def setUp(self, mock_register): + @mock.patch.object(objects.Service, 'get_by_compute_host') + def setUp(self, mock_svc, mock_register): super(VMwareAPIVMTestCase, self).setUp() ds_util.dc_cache_reset() vm_util.vm_refs_cache_reset() @@ -183,8 +188,10 @@ def setUp(self, mock_register): vmwareapi_fake.reset() self.glance = self.useFixture(nova_fixtures.GlanceFixture(self)) service = self._create_service(host=HOST) + mock_svc.return_value = service - self.conn = driver.VMwareVCDriver(None, False) + virtapi = fake.FakeComputeVirtAPI(mock.MagicMock()) + self.conn = driver.VMwareVCDriver(virtapi, False) self.assertFalse(service.disabled) self._set_exception_vars() self.node_name = self.conn._nodename @@ -204,9 +211,9 @@ def setUp(self, mock_register): 'id': image_ref, 'disk_format': 'vmdk', 'size': int(metadata['size']), + 'owner': '' }) self.fake_image_uuid = self.image.id - self.vnc_host = 'ha-host' # create compute node resource provider self.cn_rp = dict( @@ -223,6 +230,18 @@ def setUp(self, mock_register): self.pt.new_root(self.cn_rp['name'], self.cn_rp['uuid'], generation=0) self.pt.new_root(self.shared_rp['name'], self.shared_rp['uuid'], generation=0) + """ Monkey patch for vm_util.vm_needs_special_spawning + Currently set to return `True` since most of the methods are calling + vm_util.update_cluster_placement which calls _get_server_groups. This + is done in order to prevent mocking the tests using spawn. + """ + self.useFixture( + fixtures.MonkeyPatch( + 'nova.utils.vm_needs_special_spawning', + self._fake_vm_needs_special_spawning)) + + def _fake_vm_needs_special_spawning(self, *args, **kwargs): + return True def tearDown(self): super(VMwareAPIVMTestCase, self).tearDown() @@ -351,9 +370,17 @@ def _create_instance(self, node=None, set_image_ref=True, self.instance = fake_instance.fake_instance_obj( self.context, **values) - def _create_vm(self, node=None, num_instances=1, uuid=None, - flavor='m1.large', powered_on=True, - ephemeral=None, bdi=None, flavor_updates=None): + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) + def _create_vm(self, fake_fetch_image_from_other_datastores, + fake_find_image_template_vm, fake_is_volume_backed, + node=None, num_instances=1, uuid=None, flavor='m1.large', + powered_on=True, ephemeral=None, bdi=None, + flavor_updates=None): """Create and spawn the VM.""" if not node: node = self.node_name @@ -362,10 +389,14 @@ def _create_vm(self, node=None, num_instances=1, uuid=None, ephemeral=ephemeral, flavor_updates=flavor_updates) self.assertIsNone(vm_util.vm_ref_cache_get(self.uuid)) - self.conn.spawn(self.context, self.instance, self.image, - injected_files=[], admin_password=None, allocations={}, - network_info=self.network_info, - block_device_info=bdi) + with test.nested( + mock.patch.object(self.conn._vmops, 'update_cluster_placement', + return_value=None), + ): + self.conn.spawn(self.context, self.instance, self.image, + injected_files=[], admin_password=None, + allocations={}, network_info=self.network_info, + block_device_info=bdi) self._check_vm_record(num_instances=num_instances, powered_on=powered_on, uuid=uuid) @@ -409,7 +440,7 @@ def _check_vm_record(self, num_instances=1, powered_on=True, uuid=None): self.assertEqual(vm.get("summary.config.memorySizeMB"), self.type_data['memory_mb']) - self.assertEqual("ns0:VirtualE1000", + self.assertEqual("ns0:VirtualE1000e", vm.get("config.hardware.device").VirtualDevice[2].obj_name) if powered_on: # Check that the VM is running according to Nova @@ -543,8 +574,7 @@ def fake_attach_cdrom(vm_ref, instance, data_store_ref, @mock.patch.object(nova.virt.vmwareapi.images.VMwareImage, 'from_image') - def test_iso_disk_cdrom_attach_with_config_drive(self, - mock_from_image): + def test_iso_disk_cdrom_attach_with_config_drive(self, mock_from_image): img_props = images.VMwareImage( image_id=self.fake_image_uuid, file_size=80 * units.Gi, @@ -655,8 +685,7 @@ def test_destroy_with_attached_volumes(self, side_effect=vexc.ManagedObjectNotFoundException()) @mock.patch.object(vmops.VMwareVMOps, 'destroy') def test_destroy_with_attached_volumes_missing(self, - mock_destroy, - mock_power_off): + mock_destroy, mock_power_off): self._create_vm() connection_info = {'data': 'fake-data', 'serial': 'volume-fake-id'} bdm = [{'connection_info': connection_info, @@ -674,7 +703,7 @@ def test_destroy_with_attached_volumes_missing(self, side_effect=exception.NovaException()) @mock.patch.object(vmops.VMwareVMOps, 'destroy') def test_destroy_with_attached_volumes_with_exception( - self, mock_destroy, mock_detach_volume): + self, mock_destroy, mock_detach_volume): self._create_vm() connection_info = {'data': 'fake-data', 'serial': 'volume-fake-id'} bdm = [{'connection_info': connection_info, @@ -693,7 +722,7 @@ def test_destroy_with_attached_volumes_with_exception( side_effect=exception.DiskNotFound(message='oh man')) @mock.patch.object(vmops.VMwareVMOps, 'destroy') def test_destroy_with_attached_volumes_with_disk_not_found( - self, mock_destroy, mock_detach_volume): + self, mock_destroy, mock_detach_volume): self._create_vm() connection_info = {'data': 'fake-data', 'serial': 'volume-fake-id'} bdm = [{'connection_info': connection_info, @@ -800,9 +829,19 @@ def _fake_extend(instance, requested_size, name, dc_ref): vmwareapi_fake.assertPathExists(self, str(root)) self.assertEqual(1, fake_extend_virtual_disk[0].call_count) + @mock.patch.object(ds_util, 'get_datastore') @mock.patch.object(nova.virt.vmwareapi.images.VMwareImage, 'from_image') - def test_spawn_disk_extend_sparse(self, mock_from_image): + def test_spawn_disk_extend_sparse(self, mock_from_image, + mock_get_datastore): + fake_ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds1') + fake_ds = ds_obj.Datastore( + ref=fake_ds_ref, name='ds1', + capacity=10 * units.Gi, + freespace=10 * units.Gi) + + mock_get_datastore.return_value = fake_ds + img_props = images.VMwareImage( image_id=self.fake_image_uuid, file_size=units.Ki, @@ -1096,6 +1135,10 @@ def fake_call_method(module, method, *args, **kwargs): self._check_vm_info(info, power_state.RUNNING) self.assertTrue(self.exception) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=True) + @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + 'update_cluster_placement') @mock.patch.object(vm_util, 'relocate_vm') @mock.patch('nova.virt.vmwareapi.volumeops.VMwareVolumeOps.' 'attach_volume') @@ -1108,6 +1151,8 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, mock_get_res_pool_of_vm, mock_attach_volume, mock_relocate_vm, + mock_update_cluster_placement, + mock_is_volume_backed, set_image_ref=True): self._create_instance(set_image_ref=set_image_ref) @@ -1128,14 +1173,26 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, mock_attach_volume.assert_called_once_with(connection_info, self.instance, constants.DEFAULT_ADAPTER_TYPE) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) + @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + 'update_cluster_placement') @mock.patch('nova.virt.vmwareapi.volumeops.VMwareVolumeOps.' 'attach_volume') @mock.patch('nova.block_device.volume_in_mapping') @mock.patch('nova.virt.driver.block_device_info_get_mapping') + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) def test_spawn_attach_volume_iscsi(self, + mock_fetch_image_from_other_datastores, + mock_find_image_template_vm, mock_info_get_mapping, mock_block_volume_in_mapping, - mock_attach_volume): + mock_attach_volume, + mock_update_cluster_placement, + mock_is_volume_backed): self._create_instance() connection_info = self._test_vmdk_connection_info('iscsi') root_disk = [{'connection_info': connection_info, @@ -1224,7 +1281,9 @@ def _test_snapshot(self): self._check_vm_info(info, power_state.RUNNING) self.assertIsNone(func_call_matcher.match()) - def test_snapshot(self): + @mock.patch.object(vm_util, 'get_vmdk_info') + def test_snapshot(self, mock_get_vmdk_info): + mock_get_vmdk_info.return_value = self.get_fake_vmdk() self._create_vm() self._test_snapshot() @@ -1240,13 +1299,16 @@ def test_snapshot_non_existent(self): self.context, self.instance, "Test-Snapshot", lambda *args, **kwargs: None) + @mock.patch.object(vm_util, 'get_vmdk_info') @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps._delete_vm_snapshot') @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps._create_vm_snapshot') @mock.patch('nova.virt.vmwareapi.volumeops.VMwareVolumeOps.' 'attach_volume') def test_snapshot_delete_vm_snapshot(self, mock_attach_volume, mock_create_vm_snapshot, - mock_delete_vm_snapshot): + mock_delete_vm_snapshot, + mock_get_vmdk): + mock_get_vmdk.return_value = self.get_fake_vmdk() self._create_vm() fake_vm = self._get_vm_record() snapshot_ref = vmwareapi_fake.ManagedObjectReference( @@ -1258,12 +1320,26 @@ def test_snapshot_delete_vm_snapshot(self, mock_attach_volume, self._test_snapshot() - mock_create_vm_snapshot.assert_called_once_with(self.instance, - fake_vm.obj) mock_delete_vm_snapshot.assert_called_once_with(self.instance, fake_vm.obj, snapshot_ref) + def get_fake_vmdk(self): + ds_ref = vmwareapi_fake.ManagedObjectReference(value='fake-ref') + ds = ds_obj.Datastore(ds_ref, 'ds1') + device = vmwareapi_fake.DataObject() + backing = vmwareapi_fake.DataObject() + backing.datastore = ds.ref + device.backing = backing + device.key = 'fake-key' + vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', + 'fake-adapter', + 'fake-disk', + 1024, + device) + + return vmdk + def _snapshot_delete_vm_snapshot_exception(self, exception, call_count=1): self._create_vm() fake_vm = vmwareapi_fake.get_first_object_ref("VirtualMachine") @@ -1667,7 +1743,8 @@ def _test_get_vnc_console(self): opt_val = OptionValue(key='', value=5906) fake_vm.set(vm_util.VNC_CONFIG_KEY, opt_val) vnc_console = self.conn.get_vnc_console(self.context, self.instance) - self.assertEqual(self.vnc_host, vnc_console.host) + host = vmwareapi_fake.get_first_object('HostSystem') + self.assertEqual(host.name, vnc_console.host) self.assertEqual(5906, vnc_console.port) def test_get_vnc_console(self): @@ -2092,6 +2169,7 @@ def test_datastore_regex_configured(self): @mock.patch('nova.virt.vmwareapi.ds_util.get_available_datastores') def test_datastore_regex_configured_vcstate(self, mock_get_ds_ref): vcstate = self.conn._vc_state + vcstate._stats = {} # Clear cache to force new collection self.conn.get_available_resource(self.node_name) mock_get_ds_ref.assert_called_with( vcstate._session, vcstate._cluster, vcstate._datastore_regex) @@ -2099,20 +2177,29 @@ def test_datastore_regex_configured_vcstate(self, mock_get_ds_ref): def test_get_available_resource(self): stats = self.conn.get_available_resource(self.node_name) self.assertEqual(32, stats['vcpus']) + self.assertEqual(CONF.reserved_host_cpus, stats['vcpus_reserved']) self.assertEqual(1024, stats['local_gb']) self.assertEqual(1024 - 500, stats['local_gb_used']) self.assertEqual(2048, stats['memory_mb']) self.assertEqual(1000, stats['memory_mb_used']) + self.assertEqual(CONF.reserved_host_memory_mb, + stats['memory_mb_reserved']) self.assertEqual('VMware vCenter Server', stats['hypervisor_type']) self.assertEqual(5005000, stats['hypervisor_version']) self.assertEqual(self.node_name, stats['hypervisor_hostname']) - self.assertIsNone(stats['cpu_info']) + # TODO(xxxx): check cpu-info + # cpu_info = {'features': ['aes', 'avx'], + # 'vendor': 'Intel', + # 'topology': {'threads': 16, 'sockets': 2, 'cores': 8}, + # 'model': 'Intel(R) Xeon(R) CPU E5-4650 v4 @ 2.20GHz'} + # self.assertEqual(jsonutils.loads(stats['cpu_info']), cpu_info) self.assertEqual( [("i686", "vmware", "hvm"), ("x86_64", "vmware", "hvm")], stats['supported_instances']) @mock.patch('nova.virt.vmwareapi.ds_util.get_available_datastores') def test_update_provider_tree(self, mock_get_avail_ds): + self.assertEqual(CONF.reserved_host_memory_mb, 512) ds1 = ds_obj.Datastore(ref='fake-ref', name='datastore1', capacity=10 * units.Gi, freespace=3 * units.Gi) ds2 = ds_obj.Datastore(ref='fake-ref', name='datastore2', @@ -2124,6 +2211,8 @@ def test_update_provider_tree(self, mock_get_avail_ds): self.assertFalse(self.pt.has_inventory(self.node_name)) # now update it self.conn.update_provider_tree(self.pt, self.node_name) + vcstate = self.conn._vc_state + vcstate._stats = {} # Clear cache to force new collection expected = { orc.VCPU: { 'total': 32, @@ -2131,7 +2220,7 @@ def test_update_provider_tree(self, mock_get_avail_ds): 'min_unit': 1, 'max_unit': 16, 'step_size': 1, - 'allocation_ratio': 4.0, + 'allocation_ratio': CONF.initial_cpu_allocation_ratio, }, orc.MEMORY_MB: { 'total': 2048, @@ -2139,16 +2228,23 @@ def test_update_provider_tree(self, mock_get_avail_ds): 'min_unit': 1, 'max_unit': 1024, 'step_size': 1, - 'allocation_ratio': 1.0, + 'allocation_ratio': CONF.initial_ram_allocation_ratio, }, - orc.DISK_GB: { - 'total': 95, + orc.DISK_GB: { # Fix this + 'total': 1024.0, 'reserved': 0, 'min_unit': 1, - 'max_unit': 25, + 'max_unit': 500.0, 'step_size': 1, - 'allocation_ratio': 1.0, + 'allocation_ratio': CONF.initial_disk_allocation_ratio, }, + nova.utils.MEMORY_RESERVABLE_MB_RESOURCE: { + 'total': 2048 - 512, # 512 CONF.reserved_host_memory_mb + 'reserved': 0, + 'min_unit': 1, + 'max_unit': 1024, + 'step_size': 1, + } } inventory = self.pt.data(self.node_name).inventory self.assertEqual(expected, inventory) @@ -2314,9 +2410,13 @@ def test_detach_interface_with_exception(self, mock_get_device): def test_resize_to_smaller_disk(self): self._create_vm(flavor='m1.large') flavor = self._get_flavor_by_name('m1.small') - self.assertRaises(exception.InstanceFaultRollback, - self.conn.migrate_disk_and_power_off, self.context, - self.instance, 'fake_dest', flavor, None) + with test.nested( + mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False)): + self.assertRaises(exception.InstanceFaultRollback, + self.conn.migrate_disk_and_power_off, + self.context, self.instance, 'fake_dest', flavor, + None) def test_spawn_attach_volume_vmdk(self): self._spawn_attach_volume_vmdk() @@ -2341,25 +2441,165 @@ def test_datastore_dc_map(self): # currently there are 2 data stores self.assertEqual(2, len(ds_util._DS_DC_MAPPING)) - def test_pre_live_migration(self): - migrate_data = objects.VMwareLiveMigrateData() - migrate_data.cluster_name = 'fake-cluster' - migrate_data.datastore_regex = 'datastore1' - ret = self.conn.pre_live_migration(self.context, 'fake-instance', - 'fake-block-dev-info', 'fake-net-info', - 'fake-disk-info', migrate_data) - self.assertIs(migrate_data, ret) + def _create_live_migrate_data(self, source_compute=None, + dest_compute=None): + data = objects.migrate_data.VMwareLiveMigrateData() + + data.dest_cluster_ref = "cluster-0" + data.instance_already_migrated = False + data.is_same_vcenter = False + data.relocate_defaults = { + "relocate_spec": { + "pool": { + "_type": "Resgroup", + "value": "resgroup-0", + }, + "datastore": { + "_type": "Datastore", + "value": "datastore-0", + }, + "host": { + "_type": "Host", + "value": "host-0", + }, + "folder": { + "_type": "Folder", + "value": "folder-0", + } + }, + "service": { + "url": "https://otherhost.test", + "instance_uuid": "uuid", + "ssl_thumbprint": "sha1thumbprint", + "credentials": { + "_type": "ServiceLocatorNamePassword", + "username": "username", + "password": "password", + } + } + } + migration = objects.Migration( + source_compute=source_compute, + dest_compute=dest_compute, + ) + data.migration = migration + + return data + + def _create_block_device_info(self): + return {} + + def _create_placement_result(self, *args, **kwargs): + relocate_spec = mock.NonCallableMagicMock( + host=mock.sentinel.placement_host, + datastore=mock.sentinel.placement_datastore + ) + + recommendation = mock.NonCallableMagicMock( + reason="xvmotionPlacement", + rating=5, + action=[mock.NonCallableMagicMock(relocateSpec=relocate_spec)] + ) + + return mock.NonCallableMagicMock( + recommendations=[recommendation] + ) + + @mock.patch.object(vim_util, 'serialize_object') + @mock.patch.object(vmops.VMwareVMOps, 'place_vm') + def test_pre_live_migration(self, mock_place_vm, mock_serialize_object): + mock_place_vm.side_effect = self._create_placement_result + self._create_instance() + migrate_data = self._create_live_migrate_data() + block_device_info = self._create_block_device_info() + result = self.conn.pre_live_migration(self.context, + self.instance, block_device_info, 'fake_network_info', + 'fake_disk_info', migrate_data) + self.assertEqual(result, migrate_data) + + @mock.patch.object(vm_util, 'relocate_vm') + @mock.patch.object(vm_util, 'get_hardware_devices') + @mock.patch.object(vim_util, 'deserialize_object') + @mock.patch.object(driver.VMwareVCDriver, '_get_volume_mappings', + returns=[]) + def test_live_migration(self, get_volume_mappings, + deserialize_object, + get_hardware_devices, relocate_vm): + self._create_instance() + migrate_data = self._create_live_migrate_data() + + post_method = mock.Mock() + recover_method = mock.Mock() + get_hardware_devices.returns = [] + + with mock.patch.object(vm_util, 'get_vm_ref', + return_value=mock.sentinel.vm_ref): + + self.conn.live_migration(self.context, self.instance, + 'fake_dest', post_method, recover_method, + migrate_data=migrate_data) + + deserialize_object.assert_called() + get_hardware_devices.assert_called() + relocate_vm.assert_called() + post_method.assert_called() + recover_method.assert_not_called() + + @mock.patch.object(driver.VMwareVCDriver, '_get_volume_mappings', + returns=[]) + @mock.patch.object(vmops.VMwareVMOps, + 'live_migration', side_effect=test.TestingException) + def test_live_migration_failure_rollback(self, mock_live_migration, + get_volume_mappings): + self._create_instance() + migrate_data = self._create_live_migrate_data() + + post_method = mock.Mock() + recover_method = mock.Mock() + + exception = test.TestingException + + with test.nested( + mock.patch.object(vm_util, 'get_vm_ref', + return_value=mock.sentinel.vm_ref), + ) as (mock_vm_ref,): + + self.assertRaises(exception, self.conn.live_migration, + self.context, self.instance, + 'fake_dest', post_method, recover_method, + migrate_data=migrate_data) + + post_method.assert_not_called() + recover_method.assert_called() def test_rollback_live_migration_at_destination(self): - with mock.patch.object(self.conn, "destroy") as mock_destroy: - self.conn.rollback_live_migration_at_destination(self.context, - "instance", [], None) - mock_destroy.assert_called_once_with(self.context, - "instance", [], None) + self._create_instance() + block_device_info = self._create_block_device_info() + migrate_data = self._create_live_migrate_data() + with test.nested( + mock.patch.object(self.conn._volumeops, 'delete_shadow_vms'), + ) as (mock_delete_shadow_vms, ): + + self.conn.rollback_live_migration_at_destination( + self.context, self.instance, 'fake_network_info', + block_device_info, migrate_data=migrate_data) + mock_delete_shadow_vms.assert_called_once() def test_post_live_migration(self): - self.assertIsNone(self.conn.post_live_migration(self.context, - 'fake_instance', 'fake_block_device_info')) + self._create_instance() + migrate_data = self._create_live_migrate_data() + block_device_info = self._create_block_device_info() + with test.nested( + mock.patch.object(self.conn._volumeops, 'delete_shadow_vms'), + mock.patch.object(self.conn._vmops, + 'sync_instance_server_group') + ) as (mock_delete_shadow_vms, mock_sync_instance_server_group): + + self.conn.post_live_migration( + self.context, self.instance, block_device_info, + migrate_data) + mock_delete_shadow_vms.assert_called_once() + mock_sync_instance_server_group.assert_called_once() def test_get_instance_disk_info_is_implemented(self): # Ensure that the method has been implemented in the driver @@ -2408,15 +2648,39 @@ def test_warning_deprecated_version(self, mock_version, mock_warning): break self.assertTrue(version_arg_found) + def _mock_get_stats_from_cluster_per_host(self, + memory_mb_reserved=0): + return { + 'host1': { + 'available': True, + 'vcpus': 16, + 'vcpus_used': 0, + 'vcpus_reserved': 2, + 'memory_mb': 1024, + 'memory_mb_used': 0, + 'memory_mb_reserved': memory_mb_reserved, + 'cpu_info': {}, + }, + 'host2': { + 'available': True, + 'vcpus': 16, + 'vcpus_used': 0, + 'vcpus_reserved': 0, + 'memory_mb': 1024, + 'memory_mb_used': 0, + 'memory_mb_reserved': 0, + 'cpu_info': {}, + }, + } + @mock.patch.object(objects.Service, 'get_by_compute_host') def test_host_state_service_disabled(self, mock_service): service = self._create_service(disabled=False, host='fake-mini') mock_service.return_value = service - fake_stats = {'cpu': {'vcpus': 4}, - 'mem': {'total': '8194', 'free': '2048'}} + fake_stats = self._mock_get_stats_from_cluster_per_host() with test.nested( - mock.patch.object(vm_util, 'get_stats_from_cluster', + mock.patch.object(vm_util, 'get_stats_from_cluster_per_host', side_effect=[vexc.VimConnectionException('fake'), fake_stats, fake_stats]), mock.patch.object(service, 'save')) as (mock_stats, diff --git a/nova/tests/unit/virt/vmwareapi/test_ds_util.py b/nova/tests/unit/virt/vmwareapi/test_ds_util.py index 1716027afb0..66fea4dbd67 100644 --- a/nova/tests/unit/virt/vmwareapi/test_ds_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_ds_util.py @@ -289,9 +289,9 @@ def next_datastore(*args, **kwargs): return None with mock.patch.object(self.session, '_call_method', - side_effect=fake_call_method), \ + side_effect=fake_call_method), \ mock.patch('oslo_vmware.vim_util.continue_retrieval', - side_effect=next_datastore), \ + side_effect=next_datastore), \ mock.patch('oslo_vmware.vim_util.cancel_retrieval'): yield diff --git a/nova/tests/unit/virt/vmwareapi/test_images.py b/nova/tests/unit/virt/vmwareapi/test_images.py index 7ec57425474..96b4e6c9d80 100644 --- a/nova/tests/unit/virt/vmwareapi/test_images.py +++ b/nova/tests/unit/virt/vmwareapi/test_images.py @@ -171,7 +171,7 @@ def fake_extract(name): mock_get_vmdk_info.assert_called_once_with( session, mock.sentinel.vm_ref) session._call_method.assert_called_once_with( - session.vim, "UnregisterVM", mock.sentinel.vm_ref) + session.vim, "MarkAsTemplate", mock.sentinel.vm_ref) @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') @mock.patch('oslo_vmware.rw_handles.VmdkWriteHandle') @@ -217,7 +217,7 @@ def test_fetch_image_stream_optimized(self, mock_image_transfer.assert_called_once_with(mock_read_handle, mock_write_handle) session._call_method.assert_called_once_with( - session.vim, "UnregisterVM", mock.sentinel.vm_ref) + session.vim, "MarkAsTemplate", mock.sentinel.vm_ref) mock_get_vmdk_info.assert_called_once_with( session, mock.sentinel.vm_ref) @@ -226,6 +226,7 @@ def test_from_image_with_image_ref(self): raw_disk_size_in_bytes = raw_disk_size_in_gb * units.Gi mdata = {'size': raw_disk_size_in_bytes, 'disk_format': 'vmdk', + 'owner': '', 'properties': { "vmware_ostype": constants.DEFAULT_OS_TYPE, "vmware_adaptertype": constants.DEFAULT_ADAPTER_TYPE, @@ -262,6 +263,7 @@ def _image_build(self, image_lc_setting, global_lc_setting, mdata = {'size': raw_disk_size_in_btyes, 'disk_format': disk_format, + 'owner': '', 'properties': { "vmware_ostype": os_type, "vmware_adaptertype": adapter_type, @@ -343,7 +345,7 @@ def test_image_defaults(self): self.assertEqual('otherGuest', image.os_type) self.assertEqual('lsiLogic', image.adapter_type) self.assertEqual('preallocated', image.disk_type) - self.assertEqual('e1000', image.vif_model) + self.assertEqual('e1000e', image.vif_model) def test_use_vsphere_location(self): image = self._image_build(None, True, vsphere_location='vsphere://ok') diff --git a/nova/tests/unit/virt/vmwareapi/test_network_util.py b/nova/tests/unit/virt/vmwareapi/test_network_util.py index b3b5bb15eae..8e6d76a6ccf 100644 --- a/nova/tests/unit/virt/vmwareapi/test_network_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_network_util.py @@ -47,45 +47,21 @@ def _build_cluster_networks(self, networks): """Returns a set of results for a cluster network lookup. This is an example: - (ObjectContent){ - obj = - (obj){ - value = "domain-c7" - _type = "ClusterComputeResource" - } - propSet[] = - (DynamicProperty){ - name = "network" - val = - (ArrayOfManagedObjectReference){ - ManagedObjectReference[] = - (ManagedObjectReference){ - value = "network-54" - _type = "Network" - }, - (ManagedObjectReference){ - value = "dvportgroup-14" - _type = "DistributedVirtualPortgroup" - }, - } - }, + (ArrayOfManagedObjectReference){ + ManagedObjectReference[] = [ + (ManagedObjectReference){ + value = "network-54" + _type = "Network" + }, + (ManagedObjectReference){ + value = "dvportgroup-14" + _type = "DistributedVirtualPortgroup" + }, }] """ - - objects = [] - obj = ObjectContent(obj=vim_util.get_moref("domain-c7", - "ClusterComputeResource"), - propSet=[]) - value = fake.DataObject() - value.ManagedObjectReference = [] - for network in networks: - value.ManagedObjectReference.append(network) - - obj.propSet.append( - DynamicProperty(name='network', - val=value)) - objects.append(obj) - return ResultSet(objects=objects) + array = fake._create_array_of_type("ManagedObjectReference") + array.ManagedObjectReference = networks + return array def test_get_network_no_match(self): net_morefs = [vim_util.get_moref("dvportgroup-135", @@ -94,13 +70,14 @@ def test_get_network_no_match(self): "DistributedVirtualPortgroup")] networks = self._build_cluster_networks(net_morefs) - def mock_call_method(module, method, *args, **kwargs): - if method == 'get_object_properties': + def mock_call_method(module, method, mo_ref, property): + if method != 'get_object_property': + raise RuntimeError('Unexpected method') + if mo_ref == 'fake_cluster': return networks - if method == 'get_object_property': - result = fake.DataObject() - result.name = 'no-match' - return result + result = fake.DataObject() + result.name = 'no-match' + return result with mock.patch.object(self._session, '_call_method', mock_call_method): @@ -114,15 +91,16 @@ def _get_network_dvs_match(self, name): "DistributedVirtualPortgroup")] networks = self._build_cluster_networks(net_morefs) - def mock_call_method(module, method, *args, **kwargs): - if method == 'get_object_properties': + def mock_call_method(module, method, mo_ref, property): + if method != 'get_object_property': + raise RuntimeError('Unexpected method') + if mo_ref == 'fake_cluster': return networks - if method == 'get_object_property': - result = fake.DataObject() - result.name = name - result.key = 'fake_key' - result.distributedVirtualSwitch = 'fake_dvs' - return result + result = fake.DataObject() + result.name = name + result.key = 'fake_key' + result.distributedVirtualSwitch = 'fake_dvs' + return result with mock.patch.object(self._session, '_call_method', mock_call_method): @@ -141,11 +119,12 @@ def test_get_network_network_match(self): net_morefs = [vim_util.get_moref("network-54", "Network")] networks = self._build_cluster_networks(net_morefs) - def mock_call_method(module, method, *args, **kwargs): - if method == 'get_object_properties': + def mock_call_method(module, method, mo_ref, property): + if method != 'get_object_property': + raise RuntimeError('Unexpected method') + if mo_ref == 'fake_cluster': return networks - if method == 'get_object_property': - return 'fake_net' + return 'fake_net' with mock.patch.object(self._session, '_call_method', mock_call_method): diff --git a/nova/tests/unit/virt/vmwareapi/test_vim_util.py b/nova/tests/unit/virt/vmwareapi/test_vim_util.py index b3057a99ac2..42b56fe2169 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vim_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vim_util.py @@ -36,3 +36,9 @@ def test_get_inner_objects(self): self.vim, cluster_ref, 'datastore', 'Datastore', property) datastores = [oc.obj for oc in result.objects] self.assertEqual(expected_ds, datastores) + + def test_get_properties_for_an_empty_collection_of_objects(self): + result = vim_util.get_properties_for_a_collection_of_objects( + self.vim, "Foo", [], ["bar", "baz"]) + self.assertTrue(hasattr(result, "objects")) + self.assertTrue(hasattr(result.objects, "__iter__")) diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 38e7cf0edc4..24caab66718 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -13,8 +13,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +import builtins import collections +import io from unittest import mock from oslo_service import fixture as oslo_svc_fixture @@ -25,6 +26,7 @@ from oslo_vmware import pbm from oslo_vmware import vim_util as vutil +import nova.conf from nova import exception from nova.network import model as network_model from nova import test @@ -35,6 +37,8 @@ from nova.virt.vmwareapi import session as vmware_session from nova.virt.vmwareapi import vm_util +CONF = nova.conf.CONF + class partialObject(object): def __init__(self, path='fake-path'): @@ -55,10 +59,47 @@ def setUp(self): 'uuid': uuidutils.generate_uuid(), 'vcpus': 2, 'memory_mb': 2048}) + @staticmethod + def _expected_cpu_info(): + return { + 'features': ['aes', 'avx'], + 'model': 'Intel(R) Xeon(R) CPU E5-4650 v4 @ 2.20GHz', + 'topology': {'cores': 8, 'sockets': 2, 'threads': 16}, + 'vendor': 'Intel', + } + + @staticmethod + def _host_stats(): + return { + 'available': True, + 'vcpus': 4, + 'vcpus_used': 2, + 'vcpus_reserved': 1, + 'memory_mb': 4, + 'memory_mb_used': 2, + 'memory_mb_reserved': 1, + 'cpu_info': VMwareVMUtilTestCase._expected_cpu_info() + } + + def test_aggregate_stats_from_cluster(self): + a = self._host_stats() + b = self._host_stats() + b["cpu_info"]["model"] = "Different" + host_stats = {'a': a, 'b': b} + aggregated = vm_util.aggregate_stats_from_cluster(host_stats) + for k in a: + if k in ('available', 'cpu_info'): + continue + self.assertEqual(aggregated[k], sum(item[k] + for item in host_stats.values() + )) + self.assertEqual(aggregated["cpu_info"]["model"], "Mismatching values") + def _test_get_stats_from_cluster(self, connection_state="connected", maintenance_mode=False, failover_policy=None, - failover_hosts_filtered=0): + failover_hosts_filtered=0, + expected_stats=None): ManagedObjectRefs = [fake.ManagedObjectReference("HostSystem", "host1"), fake.ManagedObjectReference("HostSystem", @@ -71,13 +112,13 @@ def _test_get_stats_from_cluster(self, connection_state="connected", k = 'configuration.dasConfig.admissionControlPolicy' prop_dict[k] = failover_policy - hardware = fake.DataObject() - hardware.numCpuCores = 8 - hardware.numCpuThreads = 16 - hardware.vendor = "Intel" - hardware.cpuModel = "Intel(R) Xeon(R)" + hardware = fake.HostSystem.create_summary_hardware() hardware.memorySize = 4 * units.Gi + cpu_pkgs = fake.HostSystem.create_cpu_pkgs() + cpu_info = fake.HostSystem.create_hardware_cpu_info() + cap = fake.HostSystem.create_config_feature_capability() + runtime_host_1 = fake.DataObject() runtime_host_1.connectionState = "connected" runtime_host_1.inMaintenanceMode = False @@ -92,32 +133,43 @@ def _test_get_stats_from_cluster(self, connection_state="connected", runtime_host_2.connectionState = connection_state runtime_host_2.inMaintenanceMode = maintenance_mode - prop_list_host_1 = [fake.Prop(name="summary.hardware", val=hardware), + prop_list_host_1 = [fake.Prop(name="name", val="host-1.test"), + fake.Prop(name="summary.hardware", val=hardware), + fake.Prop(name="hardware.cpuPkg", val=cpu_pkgs), + fake.Prop(name="hardware.cpuInfo", val=cpu_info), + fake.Prop(name="config.featureCapability", + val=cap), fake.Prop(name="summary.runtime", val=runtime_host_1), fake.Prop(name="summary.quickStats", val=quickstats_1)] - prop_list_host_2 = [fake.Prop(name="summary.hardware", val=hardware), + prop_list_host_2 = [fake.Prop(name="name", val="host-2.test"), + fake.Prop(name="summary.hardware", val=hardware), + fake.Prop(name="hardware.cpuPkg", val=cpu_pkgs), + fake.Prop(name="hardware.cpuInfo", val=cpu_info), + fake.Prop(name="config.featureCapability", + val=cap), fake.Prop(name="summary.runtime", val=runtime_host_2), fake.Prop(name="summary.quickStats", val=quickstats_2)] fake_objects = { - 'host1': fake.ObjectContent("prop_list_host1", prop_list_host_1), - 'host2': fake.ObjectContent("prop_list_host1", prop_list_host_2) + 'host1': + fake.ObjectContent(ManagedObjectRefs[0], prop_list_host_1), + 'host2': + fake.ObjectContent(ManagedObjectRefs[1], prop_list_host_2) } def fake_call_method(*args): if "get_object_properties_dict" in args: return prop_dict - elif "get_properties_for_a_collection_of_objects" in args: + if "get_properties_for_a_collection_of_objects" in args: objects = fake.FakeRetrieveResult() for moref in args[3]: objects.add_object(fake_objects[moref.value]) return objects - else: - raise Exception('unexpected method call') + raise Exception('unexpected method call') session = fake.FakeSession() with mock.patch.object(session, '_call_method', fake_call_method): @@ -127,12 +179,19 @@ def fake_call_method(*args): else: num_hosts = 1 num_hosts -= failover_hosts_filtered - expected_stats = {'cpu': {'vcpus': num_hosts * 16, - 'max_vcpus_per_host': 16}, - 'mem': {'total': num_hosts * 4096, - 'free': num_hosts * 4096 - - num_hosts * 512, - 'max_mem_mb_per_host': 4096}} + if expected_stats is None: + expected_stats = { + 'cpu_info': self._expected_cpu_info(), + 'vcpus': num_hosts * 16, + 'vcpus_used': 0, + 'vcpus_reserved': 0, + 'max_vcpus_per_host': 16, + 'memory_mb': num_hosts * 4096, + 'memory_mb_used': num_hosts * 512, + 'memory_mb_reserved': 0, + 'max_mem_mb_per_host': 4096, + 'vm_reservable_memory_ratio': 1.0 + } self.assertEqual(expected_stats, result) def test_get_stats_from_cluster_hosts_connected_and_active(self): @@ -174,11 +233,168 @@ def test_get_stats_from_cluster_failover_resources_policy(self): self._test_get_stats_from_cluster(failover_policy=policy, failover_hosts_filtered=0) + @mock.patch('nova.virt.vmwareapi.vm_util._get_host_reservations_map') + def test_get_stats_from_cluster_reservations(self, mock_map): + CONF.set_override('hostgroup_reservations_json_file', '/some/path', + 'vmware') + mock_map.return_value = { + "host1": { + 'memory_mb': 256, + 'vcpus': 4}, + "host2": { + 'memory_mb': 512, + 'vcpus': 2}} + + expected = { + 'cpu_info': self._expected_cpu_info(), + 'vcpus': 2 * 16, + 'vcpus_used': 0, + 'vcpus_reserved': 4 + 2, # both hosts + 'max_vcpus_per_host': 16 - 2, # by host2 counts + 'memory_mb': 2 * 4096, + 'memory_mb_used': 2 * 512, + 'memory_mb_reserved': 512 + 256, # both + 'max_mem_mb_per_host': 4096 - 256, # host1 + 'vm_reservable_memory_ratio': 1.0 + } + self._test_get_stats_from_cluster(expected_stats=expected) + + def assertSubset(self, expected, full): + self.assertDictEqual(expected, { + key: full[key] for key in expected + }) + + def _create_stats(self, vcpus=32, memory_mb=4096): + return { + "vcpus": vcpus, + "memory_mb": memory_mb, + } + + def _create_reserved(self, vcpus_reserved=0, memory_mb_reserved=0): + return { + "vcpus_reserved": vcpus_reserved, + "memory_mb_reserved": memory_mb_reserved, + } + + def test_set_host_reservations_empty(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {} + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(0, 0) + self.assertSubset(expected, result) + + mapping = {'host2': {'vcpus': 5}} + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + self.assertSubset(expected, result) + + def test_set_host_reservations_with_default(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}} + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + + expected = self._create_reserved(2, 96) + self.assertSubset(expected, result) + + def test_set_host_reservations_with_some_default(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}, + 'host1': {'vcpus': 1}} + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(1, 96) + self.assertSubset(expected, result) + + mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}, + 'host1': {'memory_mb': 2048}} + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(2, 2048) + self.assertSubset(expected, result) + + def test_set_host_reservations_priority(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus_percent': 10, 'memory_percent': 50}, + 'host1': {'vcpus': 1}} + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(1, 2048) + self.assertSubset(expected, result) + + mapping['host1']['memory_mb'] = 96 + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(1, 96) + self.assertSubset(expected, result) + + def test_set_host_reservations_override(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus': 5}, + 'host1': {'vcpus_percent': 10}} + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(5, 0) + self.assertSubset(expected, result) + + mapping['host1']['vcpus'] = None + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(3, 0) + self.assertSubset(expected, result) + def test_get_host_ref_no_hosts_in_cluster(self): self.assertRaises(exception.NoValidHost, vm_util.get_host_ref, fake.FakeObjectRetrievalSession(""), 'fake_cluster') + def test_get_host_reservations_map_no_action(self): + self.assertEqual({}, vm_util._get_host_reservations_map()) + + @mock.patch.object(builtins, 'open') + def test_get_host_reservations_map_default_only(self, mock_open): + CONF.set_override('hostgroup_reservations_json_file', '/some/path', + 'vmware') + reservations = '{"__default__": {"vcpus": 2, "memory_mb": 512}, ' \ + '"foo": {"vcpus": 7}}' + response = io.BytesIO(reservations.encode('utf-8')) + mock_open.return_value = response + expected = {'__default__': {'vcpus': 2, 'memory_mb': 512}} + self.assertEqual(expected, vm_util._get_host_reservations_map()) + + @mock.patch.object(builtins, 'open') + def test_get_host_reservations_map(self, mock_open): + CONF.set_override('hostgroup_reservations_json_file', '/some/path', + 'vmware') + reservations = '{"__default__": {"vcpus": 2, "memory_mb": 512}, ' \ + '"foo": {"vcpus": 7, "vcpus_percent": 10, ' \ + '"memory_mb": 1024}}' + response = io.BytesIO(reservations.encode('utf-8')) + mock_open.return_value = response + + class Group(object): + def __init__(self, name, host=None): + self.name = name + if host is not None: + self.host = host + + host1 = fake.ManagedObjectReference('HostSystem', 'host1') + host2 = fake.ManagedObjectReference('HostSystem', 'host2') + host3 = fake.ManagedObjectReference('HostSystem', 'host3') + + groups = [ + Group('vmgroup'), + Group('hostgroup1', host=[host1]), + Group('foo', host=[host2, host3])] + + expected = {'__default__': {'vcpus': 2, 'memory_mb': 512}, + 'host2': {'vcpus': 7, 'vcpus_percent': 10, + 'memory_mb': 1024}, + 'host3': {'vcpus': 7, 'vcpus_percent': 10, + 'memory_mb': 1024}} + self.assertEqual(expected, vm_util._get_host_reservations_map(groups)) + def test_get_resize_spec(self): vcpus = 2 memory_mb = 2048 @@ -196,6 +412,14 @@ def test_get_resize_spec(self): cpuAllocation.shares.level = 'normal' cpuAllocation.shares.shares = 0 expected.cpuAllocation = cpuAllocation + memoryAllocation = fake_factory.create('ns0:ResourceAllocationInfo') + memoryAllocation.reservation = 0 + memoryAllocation.limit = -1 + memoryAllocation.shares = fake_factory.create('ns0:SharesInfo') + memoryAllocation.shares.level = 'normal' + memoryAllocation.shares.shares = 0 + expected.memoryAllocation = memoryAllocation + expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -204,7 +428,9 @@ def test_get_resize_spec_with_limits(self): memory_mb = 2048 cpu_limits = vm_util.Limits(limit=7, reservation=6) - extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits) + memory_limits = vm_util.Limits(reservation=127) + extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits, + memory_limits=memory_limits) fake_factory = fake.FakeFactory() result = vm_util.get_vm_resize_spec(fake_factory, vcpus, memory_mb, extra_specs) @@ -218,6 +444,14 @@ def test_get_resize_spec_with_limits(self): cpuAllocation.shares.level = 'normal' cpuAllocation.shares.shares = 0 expected.cpuAllocation = cpuAllocation + memoryAllocation = fake_factory.create('ns0:ResourceAllocationInfo') + memoryAllocation.reservation = 127 + memoryAllocation.limit = -1 + memoryAllocation.shares = fake_factory.create('ns0:SharesInfo') + memoryAllocation.shares.level = 'normal' + memoryAllocation.shares.shares = 0 + expected.memoryAllocation = memoryAllocation + expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -691,6 +925,7 @@ def test_get_vm_create_spec(self): extra_specs) expected = self._create_vm_config_spec() + self.assertEqual(expected, result) def test_get_vm_create_spec_with_serial_port(self): @@ -707,6 +942,7 @@ def test_get_vm_create_spec_with_serial_port(self): serial_port_spec = vm_util.create_serial_port_spec(fake_factory) expected = self._create_vm_config_spec() expected.deviceChange = [serial_port_spec] + self.assertEqual(expected, result) def test_get_vm_create_spec_with_allocations(self): @@ -758,6 +994,7 @@ def test_get_vm_create_spec_with_allocations(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) + self.assertEqual(expected, result) def test_get_vm_create_spec_with_limit(self): @@ -1003,6 +1240,21 @@ def test_get_vm_create_spec_with_firmware(self): expected.tools.beforeGuestStandby = True self.assertEqual(expected, result) + def test_get_vm_create_spec_with_default_hw_version(self): + CONF.set_override('default_hw_version', 'vmx-13', + 'vmware') + extra_specs = vm_util.ExtraSpecs() + fake_factory = fake.FakeFactory() + result = vm_util.get_vm_create_spec(fake_factory, + self._instance, + 'fake-datastore', [], + extra_specs) + + expected = self._create_vm_config_spec() + expected.version = 'vmx-13' + + self.assertEqual(expected, result) + def test_create_vm(self): def fake_call_method(module, method, *args, **kwargs): @@ -1847,6 +2099,7 @@ def test_get_vm_create_spec_with_cores_per_socket(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) + self.assertEqual(expected, result) def test_get_vm_create_spec_with_memory_allocations(self): @@ -1898,6 +2151,7 @@ def test_get_vm_create_spec_with_memory_allocations(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) + self.assertEqual(expected, result) def test_get_swap(self): diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 4842c226230..0f3e58bb38d 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -25,17 +25,21 @@ from oslo_vmware import vim_util as vutil from nova.compute import power_state +import nova.conf from nova import context from nova import exception from nova.network import model as network_model from nova import objects from nova import test +from nova.tests.unit import fake_block_device from nova.tests.unit import fake_flavor from nova.tests.unit import fake_instance from nova.tests.unit.virt.vmwareapi import fake as vmwareapi_fake from nova.tests.unit.virt.vmwareapi import stubs +from nova import utils from nova import version from nova.virt import hardware +from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import images @@ -44,6 +48,9 @@ from nova.virt.vmwareapi import vim_util from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops +from nova.virt.vmwareapi import volumeops + +CONF = nova.conf.CONF class DsPathMatcher(object): @@ -54,7 +61,7 @@ def __eq__(self, ds_path_param): return str(ds_path_param) == self.expected_ds_path_str -class VMwareVMOpsTestCase(test.NoDBTestCase): +class VMwareVMOpsTestCase(test.TestCase): def setUp(self): super(VMwareVMOpsTestCase, self).setUp() ds_util.dc_cache_reset() @@ -107,11 +114,14 @@ def setUp(self): root_gb=10, ephemeral_gb=0, swap=0, extra_specs={}) self._instance.flavor = self._flavor - - self._vmops = vmops.VMwareVMOps(self._session, self._virtapi, None, + self._volumeops = volumeops.VMwareVolumeOps(self._session) + self._vmops = vmops.VMwareVMOps(self._session, self._virtapi, + self._volumeops, + mock.Mock, cluster=cluster.obj) self._cluster = cluster - self._image_meta = objects.ImageMeta.from_dict({'id': self._image_id}) + self._image_meta = objects.ImageMeta.from_dict({'id': self._image_id, + 'owner': ''}) subnet_4 = network_model.Subnet(cidr='192.168.0.1/24', dns=[network_model.IP('192.168.0.1')], gateway= @@ -187,7 +197,8 @@ def test_get_machine_id_str(self): self.assertEqual('DE:AD:BE:EF:00:00;;;;;#', result) def _setup_create_folder_mocks(self): - ops = vmops.VMwareVMOps(mock.Mock(), mock.Mock(), mock.Mock()) + ops = vmops.VMwareVMOps(mock.Mock(), mock.Mock(), mock.Mock(), + mock.Mock()) base_name = 'folder' ds_name = "datastore" ds_ref = vmwareapi_fake.ManagedObjectReference(value=1) @@ -213,7 +224,8 @@ def test_create_folder_if_missing_exception(self, mock_mkdir): mock_mkdir.assert_called_with(ops._session, path, dc) def test_get_valid_vms_from_retrieve_result(self): - ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock()) + ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock(), + mock.Mock(), cluster=self._cluster.obj) fake_objects = vmwareapi_fake.FakeRetrieveResult() for x in range(0, 3): vm = vmwareapi_fake.VirtualMachine() @@ -225,7 +237,8 @@ def test_get_valid_vms_from_retrieve_result(self): self.assertEqual(3, len(vms)) def test_get_valid_vms_from_retrieve_result_with_invalid(self): - ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock()) + ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock(), + mock.Mock(), cluster=self._cluster.obj) fake_objects = vmwareapi_fake.FakeRetrieveResult() valid_vm = vmwareapi_fake.VirtualMachine() valid_vm.set('config.extraConfig["nvp.vm-uuid"]', @@ -355,7 +368,8 @@ def mock_call_method(module, method, *args, **kwargs): def _test_get_datacenter_ref_and_name(self, ds_ref_exists=False): instance_ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds-1') - _vcvmops = vmops.VMwareVMOps(self._session, None, None) + _vcvmops = vmops.VMwareVMOps(self._session, None, None, None, + cluster=self._cluster.obj) result = vmwareapi_fake.FakeRetrieveResult() if ds_ref_exists: ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds-1') @@ -392,9 +406,14 @@ def test_get_datacenter_ref_and_name_with_no_datastore(self): @mock.patch.object(vm_util, 'reconfigure_vm') @mock.patch.object(vm_util, 'power_on_instance') @mock.patch.object(ds_obj, 'get_datastore_by_ref') - def test_rescue(self, mock_get_ds_by_ref, mock_power_on, mock_reconfigure, - mock_get_boot_spec, mock_find_rescue, - mock_get_vm_ref, mock_disk_copy, + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) + def test_rescue(self, mock_fetch_image_from_other_datastores, + mock_find_image_template_vm, mock_get_ds_by_ref, + mock_power_on, mock_reconfigure, mock_get_boot_spec, + mock_find_rescue, mock_get_vm_ref, mock_disk_copy, mock_power_off, mock_glance): _volumeops = mock.Mock() self._vmops._volumeops = _volumeops @@ -588,40 +607,109 @@ def test_clean_shutdown_no_vmwaretools(self): vmware_tools_status="toolsNotOk", succeeds=False) - def _test_finish_migration(self, power_on=True, resize_instance=False): + def _test_finish_migration(self, power_on=True, resize_instance=False, + migration=None, relocate_fails=False): with test.nested( mock.patch.object(self._vmops, '_resize_create_ephemerals_and_swap'), mock.patch.object(self._vmops, "_update_instance_progress"), mock.patch.object(vm_util, "power_on_instance"), mock.patch.object(vm_util, "get_vm_ref", - return_value='fake-ref') + return_value='fake-ref'), + mock.patch.object(vmops.VMwareVMOps, + "_remove_ephemerals_and_swap"), + mock.patch.object(vm_util, 'get_vmdk_info'), + mock.patch.object(vmops.VMwareVMOps, "_resize_vm"), + mock.patch.object(vmops.VMwareVMOps, "_resize_disk"), + mock.patch.object(vmops.VMwareVMOps, "_relocate_vm"), + mock.patch.object(vmops.VMwareVMOps, "_detach_volumes"), + mock.patch.object(vmops.VMwareVMOps, "_attach_volumes"), + mock.patch.object(vmops.VMwareVMOps, + "update_cluster_placement"), + mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) ) as (fake_resize_create_ephemerals_and_swap, - fake_update_instance_progress, fake_power_on, fake_get_vm_ref): - self._vmops.finish_migration(context=self._context, - migration=None, - instance=self._instance, - disk_info=None, - network_info=None, - block_device_info=None, - resize_instance=resize_instance, - image_meta=None, - power_on=power_on) - fake_resize_create_ephemerals_and_swap.assert_called_once_with( - 'fake-ref', self._instance, None) - if power_on: - fake_power_on.assert_called_once_with(self._session, - self._instance, - vm_ref='fake-ref') + fake_update_instance_progress, fake_power_on, fake_get_vm_ref, + fake_remove_ephemerals_and_swap, fake_get_vmdk_info, + fake_resize_vm, fake_resize_disk, fake_relocate_vm, + fake_detach_volumes, fake_attach_volumes, + fake_update_cluster_placement, fake_is_volume_backed): + migration = migration or objects.Migration(dest_compute="nova", + source_compute="nova") + if relocate_fails: + fake_relocate_vm.side_effect = test.TestingException + vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', + 'fake-adapter', + 'fake-disk', + self._instance.flavor.root_gb * units.Gi, + 'fake-device') + fake_get_vmdk_info.return_value = vmdk + vm_ref_calls = [mock.call(self._session, self._instance)] + try: + self._vmops.finish_migration(context=self._context, + migration=migration, + instance=self._instance, + disk_info=None, + network_info=None, + block_device_info=None, + resize_instance=resize_instance, + image_meta=None, + power_on=power_on) + except test.TestingException: + pass + relocate_failed = False + if migration.dest_compute != migration.source_compute: + fake_relocate_vm.\ + assert_called_once_with('fake-ref', self._context, + self._instance, None, None) + fake_detach_volumes.assert_called_once_with(self._instance, + None) + fake_attach_volumes.assert_called_once_with(self._instance, + None, + vmdk.adapter_type) + if not relocate_fails: + fake_update_cluster_placement.assert_called_once_with( + self._context, self._instance) + else: + fake_update_cluster_placement.assert_not_called() + relocate_failed = True + + if not relocate_failed: + fake_resize_create_ephemerals_and_swap.assert_called_once_with( + 'fake-ref', self._instance, None) + fake_remove_ephemerals_and_swap.assert_called_once_with( + 'fake-ref') + if power_on: + fake_power_on.assert_called_once_with(self._session, + self._instance, + vm_ref='fake-ref') + else: + self.assertFalse(fake_power_on.called) + + fake_resize_vm.assert_called_once_with(self._context, + self._instance, + 'fake-ref', + self._instance.flavor, + mock.ANY) + if resize_instance: + fake_resize_disk.\ + assert_called_once_with(self._instance, + 'fake-ref', + vmdk, + self._instance.flavor) + + calls = [mock.call(self._context, self._instance, step=i, + total_steps=vmops.RESIZE_TOTAL_STEPS) + for i in range(2, 7)] + fake_update_instance_progress.assert_has_calls(calls) else: - self.assertFalse(fake_power_on.called) - - calls = [ - mock.call(self._context, self._instance, step=5, - total_steps=vmops.RESIZE_TOTAL_STEPS), - mock.call(self._context, self._instance, step=6, - total_steps=vmops.RESIZE_TOTAL_STEPS)] - fake_update_instance_progress.assert_has_calls(calls) + fake_resize_create_ephemerals_and_swap.assert_not_called() + fake_remove_ephemerals_and_swap.assert_not_called() + fake_power_on.assert_not_called() + fake_resize_vm.assert_not_called() + fake_resize_disk.assert_not_called() + fake_update_instance_progress.assert_not_called() + fake_get_vm_ref.assert_has_calls(vm_ref_calls) def test_finish_migration_power_on(self): self._test_finish_migration(power_on=True, resize_instance=False) @@ -632,6 +720,17 @@ def test_finish_migration_power_off(self): def test_finish_migration_power_on_resize(self): self._test_finish_migration(power_on=True, resize_instance=True) + def test_finish_migration_another_cluster(self, relocate_fails=False): + self._test_finish_migration(power_on=True, + resize_instance=False, + relocate_fails=relocate_fails, + migration=objects.Migration( + dest_compute="dest", + source_compute="source")) + + def test_finish_migration_relocate_fails(self): + self.test_finish_migration_another_cluster(relocate_fails=True) + @mock.patch.object(vmops.VMwareVMOps, '_create_swap') @mock.patch.object(vmops.VMwareVMOps, '_create_ephemeral') @mock.patch.object(ds_obj, 'get_datastore_by_ref', @@ -683,6 +782,7 @@ def test_resize_create_ephemerals_no_root(self): vmdk = vm_util.VmdkInfo(None, None, None, 0, None) self._test_resize_create_ephemerals(vmdk, None) + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(vmops.VMwareVMOps, '_resize_create_ephemerals_and_swap') @mock.patch.object(vmops.VMwareVMOps, '_remove_ephemerals_and_swap') @@ -698,7 +798,13 @@ def test_resize_create_ephemerals_no_root(self): @mock.patch.object(vm_util, 'power_off_instance') @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake-ref') @mock.patch.object(vm_util, 'power_on_instance') - def _test_finish_revert_migration(self, fake_power_on, + @mock.patch.object(vmops.VMwareVMOps, '_detach_volumes') + @mock.patch.object(vmops.VMwareVMOps, '_attach_volumes') + @mock.patch.object(vmops.VMwareVMOps, '_relocate_vm') + @mock.patch.object(vmops.VMwareVMOps, 'list_instances') + def _test_finish_revert_migration(self, fake_list_instances, + fake_relocate_vm, fake_attach_volumes, + fake_detach_volumes, fake_power_on, fake_get_vm_ref, fake_power_off, fake_resize_spec, fake_reconfigure_vm, fake_get_browser, @@ -707,7 +813,9 @@ def _test_finish_revert_migration(self, fake_power_on, fake_remove_ephemerals_and_swap, fake_resize_create_ephemerals_and_swap, fake_get_extra_specs, - power_on): + fake_update_cluster_placement, + power_on, instances_list=None, + relocate_fails=False): """Tests the finish_revert_migration method on vmops.""" datastore = ds_obj.Datastore(ref='fake-ref', name='fake') device = vmwareapi_fake.DataObject() @@ -715,7 +823,7 @@ def _test_finish_revert_migration(self, fake_power_on, backing.datastore = datastore.ref device.backing = backing vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', - 'fake-adapter', + mock.sentinel.adapter_type, 'fake-disk', 'fake-capacity', device) @@ -732,14 +840,21 @@ def _test_finish_revert_migration(self, fake_power_on, self._vmops._volumeops = mock.Mock() mock_attach_disk = self._vmops._volumeops.attach_disk_to_vm mock_detach_disk = self._vmops._volumeops.detach_disk_from_vm + instances_list = instances_list or [self._instance.uuid] + fake_list_instances.return_value = instances_list + same_cluster = self._instance.uuid in instances_list + if relocate_fails: + fake_relocate_vm.side_effect = test.TestingException() + try: + self._vmops.finish_revert_migration(self._context, + instance=self._instance, + network_info=None, + block_device_info=None, + power_on=power_on) + except test.TestingException: + pass - self._vmops.finish_revert_migration(self._context, - instance=self._instance, - network_info=None, - block_device_info=None, - power_on=power_on) - fake_get_vm_ref.assert_called_once_with(self._session, - self._instance) + vm_ref_calls = [mock.call(self._session, self._instance)] fake_power_off.assert_called_once_with(self._session, self._instance, 'fake-ref') @@ -787,13 +902,28 @@ def _test_finish_revert_migration(self, fake_power_on, '[fake] uuid/original.vmdk', '[fake] uuid/root.vmdk') mock_attach_disk.assert_called_once_with( - 'fake-ref', self._instance, 'fake-adapter', 'fake-disk', + 'fake-ref', self._instance, vmdk.adapter_type, 'fake-disk', '[fake] uuid/root.vmdk', disk_io_limits=extra_specs.disk_io_limits) fake_remove_ephemerals_and_swap.assert_called_once_with('fake-ref') fake_resize_create_ephemerals_and_swap.assert_called_once_with( 'fake-ref', self._instance, None) - if power_on: + if not same_cluster: + fake_detach_volumes.assert_called_once_with(self._instance, + None) + fake_relocate_vm.\ + assert_called_once_with('fake-ref', self._context, + self._instance, None) + fake_attach_volumes.assert_called_once_with(self._instance, + None, + vmdk.adapter_type) + if not relocate_fails: + fake_update_cluster_placement.assert_called_once_with( + self._context, self._instance) + else: + fake_update_cluster_placement.assert_not_called() + fake_get_vm_ref.assert_has_calls(vm_ref_calls) + if power_on and not relocate_fails: fake_power_on.assert_called_once_with(self._session, self._instance) else: @@ -892,37 +1022,32 @@ def mock_call_method(module, method, *args, **kwargs): mock_get_ds.assert_called_once_with(self._session, 'cluster_ref', None) - @mock.patch.object(vm_util, 'relocate_vm') - @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake_vm') - @mock.patch.object(vm_util, 'get_cluster_ref_by_name', - return_value='fake_cluster') - @mock.patch.object(vm_util, 'get_res_pool_ref', return_value='fake_pool') - @mock.patch.object(vmops.VMwareVMOps, '_find_datastore_for_migration') - @mock.patch.object(vmops.VMwareVMOps, '_find_esx_host', - return_value='fake_host') - def test_live_migration(self, mock_find_host, mock_find_datastore, - mock_get_respool, mock_get_cluster, mock_get_vm, - mock_relocate): - post_method = mock.MagicMock() - migrate_data = objects.VMwareLiveMigrateData() - migrate_data.cluster_name = 'fake-cluster' - migrate_data.datastore_regex = 'ds1|ds2' - mock_find_datastore.return_value = ds_obj.Datastore('ds_ref', 'ds') - with mock.patch.object(self._session, '_call_method', - return_value='hardware-devices'): - self._vmops.live_migration( - self._context, self._instance, 'fake-host', - post_method, None, False, migrate_data) - mock_get_vm.assert_called_once_with(self._session, self._instance) - mock_get_cluster.assert_called_once_with(self._session, 'fake-cluster') - mock_find_datastore.assert_called_once_with(self._instance, 'fake_vm', - 'fake_cluster', mock.ANY) - mock_find_host.assert_called_once_with('fake_cluster', 'ds_ref') - mock_relocate.assert_called_once_with(self._session, 'fake_vm', - 'fake_pool', 'ds_ref', 'fake_host', - devices=[]) - post_method.assert_called_once_with(self._context, self._instance, - 'fake-host', False, migrate_data) + def test_finish_revert_migration_another_cluster(self, + relocate_fails=False): + instances_list = ["fake_uuid_foo_bar"] + self._test_finish_revert_migration(power_on=True, + instances_list=instances_list, + relocate_fails=relocate_fails) + + def test_finish_revert_migration_relocate_fails(self): + self.test_finish_revert_migration_another_cluster(relocate_fails=True) + + @mock.patch.object(volumeops.VMwareVolumeOps, 'attach_volume') + def test_attach_volumes(self, fake_attach_volume): + block_device_info = { + 'block_device_mapping': [ + {'boot_index': -1, 'connection_info': {'id': 'c'}}, + {'boot_index': 1, 'connection_info': {'id': 'b'}}, + {'boot_index': 0, 'connection_info': {'id': 'a'}}, + ] + } + self._vmops._attach_volumes(self._instance, block_device_info, + mock.sentinel.adapter_type) + fake_attach_volume.assert_has_calls([ + mock.call({'id': 'a'}, self._instance, mock.sentinel.adapter_type), + mock.call({'id': 'b'}, self._instance, mock.sentinel.adapter_type), + mock.call({'id': 'c'}, self._instance, mock.sentinel.adapter_type), + ]) @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @@ -938,17 +1063,118 @@ def test_resize_vm(self, fake_resize_spec, fake_reconfigure, memory_mb=1024, vcpus=2, extra_specs={}) - self._vmops._resize_vm(self._context, self._instance, 'vm-ref', flavor, + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() + self._vmops._resize_vm(self._context, instance, 'vm-ref', flavor, None) fake_get_metadata.assert_called_once_with(self._context, - self._instance, + instance, flavor=flavor) fake_resize_spec.assert_called_once_with( self._session.vim.client.factory, 2, 1024, extra_specs, - metadata=self._metadata) + metadata=self._metadata) fake_reconfigure.assert_called_once_with(self._session, 'vm-ref', 'fake-spec') + @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') + @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') + @mock.patch.object(vmops.VMwareVMOps, '_clean_up_after_special_spawning') + @mock.patch.object(vm_util, 'reconfigure_vm') + @mock.patch.object(vm_util, 'get_vm_resize_spec', + return_value='fake-spec') + @mock.patch.object(cluster_util, 'update_cluster_drs_vm_override') + def test_resize_vm_bigvm_upsize(self, fake_drs_override, fake_resize_spec, + fake_reconfigure, + fake_cleanup_after_special_spawning, + fake_get_extra_specs, fake_get_metadata): + extra_specs = vm_util.ExtraSpecs() + fake_get_extra_specs.return_value = extra_specs + fake_get_metadata.return_value = self._metadata + flavor = objects.Flavor(name='bigvm-test', + memory_mb=CONF.bigvm_mb, + vcpus=2, + extra_specs={}) + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() + self._vmops._resize_vm(self._context, instance, 'vm-ref', flavor, + None) + behavior = constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED + fake_drs_override.assert_called_once_with(self._session, + self._cluster.obj, + 'vm-ref', + operation='add', + behavior=behavior) + expected = (self._context, int(flavor.memory_mb), flavor) + fake_cleanup_after_special_spawning.assert_called_once_with(*expected) + + @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') + @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') + @mock.patch.object(vmops.VMwareVMOps, '_clean_up_after_special_spawning') + @mock.patch.object(vm_util, 'reconfigure_vm') + @mock.patch.object(vm_util, 'get_vm_resize_spec', + return_value='fake-spec') + @mock.patch.object(cluster_util, 'update_cluster_drs_vm_override') + def test_resize_vm_bigvm_downsize(self, fake_drs_override, + fake_resize_spec, fake_reconfigure, + fake_cleanup_after_special_spawning, + fake_get_extra_specs, fake_get_metadata): + extra_specs = vm_util.ExtraSpecs() + fake_get_extra_specs.return_value = extra_specs + fake_get_metadata.return_value = self._metadata + flavor = objects.Flavor(name='m1.small', + memory_mb=1024, + vcpus=2, + extra_specs={}) + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() + instance.old_flavor.memory_mb = CONF.bigvm_mb + self._vmops._resize_vm(self._context, instance, 'vm-ref', flavor, + None) + fake_drs_override.assert_called_once_with(self._session, + self._cluster.obj, + 'vm-ref', + operation='remove') + expected = (self._context, int(flavor.memory_mb), flavor) + fake_cleanup_after_special_spawning.assert_called_once_with(*expected) + + def test_reserve_all_memory_for_memory_reserved_flavor(self): + self.flags(group='vmware', reserve_all_memory=False) + self._instance.flavor.extra_specs.update({ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY: + self._instance.flavor.memory_mb}) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertEqual(specs.memory_limits.reservation, + self._instance.flavor.memory_mb) + + def test_reserve_no_memory_for_memory_unreserved_flavor(self): + self.flags(group='vmware', reserve_all_memory=False) + self._instance.flavor.extra_specs.pop( + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY, None) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertIsNone(specs.memory_limits.reservation) + + def test_global_reserve_all_memory_overrides_flavor_setting(self): + self.flags(group='vmware', reserve_all_memory=True) + self._instance.flavor.extra_specs.update({ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY: + self._instance.flavor.memory_mb // 2}) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertEqual(specs.memory_limits.reservation, + self._instance.flavor.memory_mb) # ie NOT memory_mb/2 + + def test_reserve_max_flavor_memory_for_memory_reserved_flavor(self): + self.flags(group='vmware', reserve_all_memory=False) + self._instance.flavor.extra_specs.update({ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY: + self._instance.flavor.memory_mb + 1000}) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertEqual(specs.memory_limits.reservation, + self._instance.flavor.memory_mb) + @mock.patch.object(vmops.VMwareVMOps, '_extend_virtual_disk') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(ds_util, 'disk_move') @@ -975,18 +1201,20 @@ def test_resize_disk(self, fake_disk_copy, fake_disk_move, extra_specs = vm_util.ExtraSpecs() fake_get_extra_specs.return_value = extra_specs + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() flavor = fake_flavor.fake_flavor_obj(self._context, root_gb=self._instance.flavor.root_gb + 1) - self._vmops._resize_disk(self._instance, 'fake-ref', vmdk, flavor) + self._vmops._resize_disk(instance, 'fake-ref', vmdk, flavor) fake_get_dc_ref_and_name.assert_called_once_with(datastore.ref) fake_disk_copy.assert_called_once_with( self._session, dc_info.ref, '[fake] uuid/root.vmdk', '[fake] uuid/resized.vmdk') mock_detach_disk.assert_called_once_with('fake-ref', - self._instance, + instance, device) fake_extend.assert_called_once_with( - self._instance, flavor['root_gb'] * units.Mi, + instance, flavor['root_gb'] * units.Mi, '[fake] uuid/resized.vmdk', dc_info.ref) calls = [ mock.call(self._session, dc_info.ref, @@ -998,7 +1226,7 @@ def test_resize_disk(self, fake_disk_copy, fake_disk_move, fake_disk_move.assert_has_calls(calls) mock_attach_disk.assert_called_once_with( - 'fake-ref', self._instance, 'fake-adapter', 'fake-disk', + 'fake-ref', instance, 'fake-adapter', 'fake-disk', '[fake] uuid/root.vmdk', disk_io_limits=extra_specs.disk_io_limits) @@ -1073,17 +1301,15 @@ def test_migrate_disk_and_power_off_disk_shrink(self): self._test_migrate_disk_and_power_off, flavor_root_gb=self._instance.flavor.root_gb - 1) - @mock.patch.object(vmops.VMwareVMOps, "_remove_ephemerals_and_swap") + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vm_util, 'get_vmdk_info') - @mock.patch.object(vmops.VMwareVMOps, "_resize_disk") - @mock.patch.object(vmops.VMwareVMOps, "_resize_vm") @mock.patch.object(vm_util, 'power_off_instance') @mock.patch.object(vmops.VMwareVMOps, "_update_instance_progress") @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake-ref') def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, - fake_power_off, fake_resize_vm, - fake_resize_disk, fake_get_vmdk_info, - fake_remove_ephemerals_and_swap, + fake_power_off, fake_get_vmdk_info, + fake_is_volume_backed, flavor_root_gb): vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', 'fake-adapter', @@ -1096,8 +1322,7 @@ def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, self._vmops.migrate_disk_and_power_off(self._context, self._instance, None, - flavor, - None) + flavor) fake_get_vm_ref.assert_called_once_with(self._session, self._instance) @@ -1105,71 +1330,98 @@ def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, fake_power_off.assert_called_once_with(self._session, self._instance, 'fake-ref') - fake_resize_vm.assert_called_once_with(self._context, self._instance, - 'fake-ref', flavor, mock.ANY) - fake_resize_disk.assert_called_once_with(self._instance, 'fake-ref', - vmdk, flavor) calls = [mock.call(self._context, self._instance, step=i, total_steps=vmops.RESIZE_TOTAL_STEPS) - for i in range(4)] + for i in range(2)] fake_progress.assert_has_calls(calls) + @mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid') + @mock.patch.object(vmops.VMwareVMOps, '_resize_create_ephemerals_and_swap') @mock.patch.object(vmops.VMwareVMOps, "_remove_ephemerals_and_swap") @mock.patch.object(vm_util, 'get_vmdk_info') @mock.patch.object(vmops.VMwareVMOps, "_resize_disk") @mock.patch.object(vmops.VMwareVMOps, "_resize_vm") - @mock.patch.object(vm_util, 'power_off_instance') + @mock.patch.object(vm_util, 'power_on_instance') @mock.patch.object(vmops.VMwareVMOps, "_update_instance_progress") @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake-ref') - def test_migrate_disk_and_power_off_root_block_device(self, - fake_get_vm_ref, fake_progress, - fake_power_off, fake_resize_vm, - fake_resize_disk, fake_get_vmdk_info, - fake_remove_ephemerals_and_swap): + def test_finish_migration_root_block_device(self, fake_get_vm_ref, + fake_progress, fake_power_on, + fake_resize_vm, + fake_resize_disk, + fake_get_vmdk_info, + fake_remove_eph_and_swap, + fake_resize_create_eph_swap, + fake_bdm_get_by_instance_uuid): # shrinking the root-disk should be ignored - flavor_root_gb = self._instance.flavor.root_gb - 1 - - self._instance.image_ref = None - connection_info1 = {'data': 'fake-data1', 'serial': 'volume-fake-id1'} - connection_info2 = {'data': 'fake-data2', 'serial': 'volume-fake-id2'} - connection_info3 = {'data': 'fake-data3', 'serial': 'volume-fake-id3'} - bdm = [{'boot_index': 0, - 'connection_info': connection_info1, - 'disk_bus': constants.ADAPTER_TYPE_IDE}, - {'boot_index': 1, - 'connection_info': connection_info2, - 'disk_bus': constants.DEFAULT_ADAPTER_TYPE}, - {'boot_index': 2, - 'connection_info': connection_info3, - 'disk_bus': constants.ADAPTER_TYPE_LSILOGICSAS}] - bdi = {'block_device_mapping': bdm} - - vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', - 'fake-adapter', - 'fake-disk', - self._instance.flavor.root_gb * units.Gi, - 'fake-device') - fake_get_vmdk_info.return_value = vmdk - flavor = fake_flavor.fake_flavor_obj(self._context, - root_gb=flavor_root_gb) - self._vmops.migrate_disk_and_power_off(self._context, - self._instance, - None, - flavor, - bdi) - + bdms = objects.block_device.block_device_make_list_from_dicts( + self._context, [ + fake_block_device.FakeDbBlockDeviceDict( + {'id': 1, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/sda', 'tag': "db", + 'volume_id': uuids.volume_1, + 'boot_index': 0}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 2, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/hda', 'tag': "nfvfunc1", + 'volume_id': uuids.volume_2}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 3, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/sdb', 'tag': "nfvfunc2", + 'volume_id': uuids.volume_3}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 4, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/hdb', + 'volume_id': uuids.volume_4}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 5, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/vda', 'tag': "nfvfunc3", + 'volume_id': uuids.volume_5}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 6, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/vdb', 'tag': "nfvfunc4", + 'volume_id': uuids.volume_6}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 7, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/vdc', 'tag': "nfvfunc5", + 'volume_id': uuids.volume_7}), + ] + ) + fake_bdm_get_by_instance_uuid.return_value = bdms + + migration = objects.Migration(source_compute="nova", + dest_compute="nova") + self._vmops.finish_migration(context=self._context, + migration=migration, + instance=self._instance, + disk_info=None, + network_info=None, + block_device_info=None, + resize_instance=True, + image_meta=self._image_meta, + power_on=True) fake_get_vm_ref.assert_called_once_with(self._session, self._instance) - - fake_power_off.assert_called_once_with(self._session, - self._instance, - 'fake-ref') + fake_resize_create_eph_swap.assert_called_once_with( + 'fake-ref', self._instance, None) + fake_remove_eph_and_swap.assert_called_once_with('fake-ref') + fake_power_on.assert_called_once_with(self._session, self._instance, + vm_ref='fake-ref') fake_resize_vm.assert_called_once_with(self._context, self._instance, - 'fake-ref', flavor, mock.ANY) + 'fake-ref', + self._instance.flavor, + self._image_meta) + fake_get_vmdk_info.assert_not_called() fake_resize_disk.assert_not_called() calls = [mock.call(self._context, self._instance, step=i, total_steps=vmops.RESIZE_TOTAL_STEPS) - for i in range(4)] + for i in range(2, 7)] fake_progress.assert_has_calls(calls) @mock.patch.object(vutil, 'get_inventory_path', return_value='fake_path') @@ -1206,22 +1458,28 @@ def test_prepare_for_spawn_invalid_ram(self): self.assertRaises(exception.InstanceUnacceptable, self._vmops.prepare_for_spawn, instance) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=True) @mock.patch('nova.image.glance.API.get') @mock.patch.object(vmops.LOG, 'debug') @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_if_missing') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.lockutils, 'lock') - def test_spawn_mask_block_device_info_password(self, mock_lock, - mock_build_virtual_machine, mock_get_vm_config_info, - mock_fetch_image_if_missing, mock_debug, mock_glance): + @mock.patch.object(ds_util, 'get_datastore') + def test_spawn_mask_block_device_info_password(self, mock_get_datastore, + mock_lock, mock_update_cluster_placement, + mock_build_virtual_machine, mock_get_vm_config_info, + mock_fetch_image_if_missing, mock_debug, mock_glance, + mock_is_volume_backed): # Very simple test that just ensures block_device_info auth_password # is masked when logged; the rest of the test just fails out early. data = {'auth_password': 'scrubme'} bdm = [{'boot_index': 0, 'disk_bus': constants.DEFAULT_ADAPTER_TYPE, 'connection_info': {'data': data}}] bdi = {'block_device_mapping': bdm} - + mock_get_datastore.return_value = self._ds self.password_logged = False # Tests that the parameters to the to_xml method are sanitized for @@ -1235,11 +1493,26 @@ def fake_debug(*args, **kwargs): self.flags(flat_injected=False) self.flags(enabled=False, group='vnc') - mock_vi = mock.Mock() - mock_vi.root_gb = 1 - mock_vi.ii.file_size = 2 * units.Gi - mock_vi.instance.flavor.root_gb = 1 - mock_get_vm_config_info.return_value = mock_vi + extra_specs = vm_util.ExtraSpecs() + flavor_fits_image = False + file_size = 10 * units.Gi if flavor_fits_image else 5 * units.Gi + image_info = images.VMwareImage( + image_id=self._image_id, + file_size=file_size, + linked_clone=False) + + cache_root_folder = self._ds.build_path("vmware_base", self._image_id) + mock_imagecache = mock.Mock() + mock_imagecache.get_image_cache_folder.return_value = cache_root_folder + dc_info = ds_util.DcInfo( + ref=self._cluster, name='fake_dc', + vmFolder=vmwareapi_fake.ManagedObjectReference( + name='Folder', + value='fake_vm_folder')) + vi = vmops.VirtualMachineInstanceConfigInfo( + self._instance, image_info, + self._ds, dc_info, mock_imagecache, extra_specs) + mock_get_vm_config_info.return_value = vi # Call spawn(). We don't care what it does as long as it generates # the log message, which we check below. @@ -1275,6 +1548,8 @@ def _get_metadata(self, is_image_used=True): 'image_id': uuids.image if is_image_used else None, 'version': version.version_string_with_package()}) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vm_util, 'rename_vm') @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') @@ -1284,18 +1559,21 @@ def _get_metadata(self, is_image_used=True): @mock.patch( 'nova.virt.vmwareapi.imagecache.ImageCacheManager.enlist_image') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_non_root_block_device(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, enlist_image, fetch_image, use_disk_image, power_on_instance, create_folders, - rename_vm): + rename_vm, + is_volume_backed_instance): self._instance.flavor = self._flavor extra_specs = get_extra_specs.return_value connection_info1 = {'data': 'fake-data1', 'serial': 'volume-fake-id1'} @@ -1328,6 +1606,7 @@ def test_spawn_non_root_block_device(self, from_image, get_vm_config_info.assert_called_once_with(self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, self._get_metadata()) enlist_image.assert_called_once_with(image_info.image_id, @@ -1340,21 +1619,26 @@ def test_spawn_non_root_block_device(self, from_image, connection_info2, self._instance, constants.DEFAULT_ADAPTER_TYPE) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=True) @mock.patch.object(vm_util, 'rename_vm') @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') @mock.patch('nova.virt.vmwareapi.vm_util.power_on_instance') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_with_no_image_and_block_devices(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, power_on_instance, create_folders, - rename_vm): + rename_vm, + is_volume_backed_instance): self._instance.image_ref = None self._instance.flavor = self._flavor extra_specs = get_extra_specs.return_value @@ -1391,6 +1675,7 @@ def test_spawn_with_no_image_and_block_devices(self, from_image, get_vm_config_info.assert_called_once_with(self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, self._get_metadata(is_image_used=False)) volumeops.attach_root_volume.assert_called_once_with( @@ -1403,19 +1688,24 @@ def test_spawn_with_no_image_and_block_devices(self, from_image, connection_info3, self._instance, constants.ADAPTER_TYPE_LSILOGICSAS) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') @mock.patch('nova.virt.vmwareapi.vm_util.power_on_instance') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_unsupported_hardware(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, power_on_instance, - create_folders): + create_folders, + is_volume_backed_instance): self._instance.image_ref = None self._instance.flavor = self._flavor extra_specs = get_extra_specs.return_value @@ -1444,6 +1734,7 @@ def test_spawn_unsupported_hardware(self, from_image, get_vm_config_info.assert_called_once_with( self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, self._get_metadata(is_image_used=False)) @@ -1464,9 +1755,11 @@ def test_get_ds_browser(self): @mock.patch.object( vmops.VMwareVMOps, '_sized_image_exists', return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_extend_virtual_disk') + @mock.patch.object(vmops.VMwareVMOps, '_extend_if_required') @mock.patch.object(vm_util, 'copy_virtual_disk') def _test_use_disk_image_as_linked_clone(self, mock_copy_virtual_disk, + mock_extend_if_required, mock_extend_virtual_disk, mock_sized_image_exists, flavor_fits_image=False): @@ -1497,11 +1790,10 @@ def _test_use_disk_image_as_linked_clone(self, str(vi.cache_image_path), str(sized_cached_image_ds_loc)) - if not flavor_fits_image: - mock_extend_virtual_disk.assert_called_once_with( - self._instance, vi.root_gb * units.Mi, - str(sized_cached_image_ds_loc), - self._dc_info.ref) + mock_extend_if_required.assert_called_once_with( + self._dc_info, + vi.ii, + vi.instance, str(sized_cached_image_ds_loc)) mock_attach_disk_to_vm.assert_called_once_with( "fake_vm_ref", self._instance, vi.ii.adapter_type, @@ -1517,9 +1809,11 @@ def test_use_disk_image_as_linked_clone_flavor_fits_image(self): self._test_use_disk_image_as_linked_clone(flavor_fits_image=True) @mock.patch.object(vmops.VMwareVMOps, '_extend_virtual_disk') + @mock.patch.object(vmops.VMwareVMOps, '_extend_if_required') @mock.patch.object(vm_util, 'copy_virtual_disk') def _test_use_disk_image_as_full_clone(self, mock_copy_virtual_disk, + mock_extend_if_required, mock_extend_virtual_disk, flavor_fits_image=False): extra_specs = vm_util.ExtraSpecs() @@ -1548,10 +1842,10 @@ def _test_use_disk_image_as_full_clone(self, str(vi.cache_image_path), fake_path) - if not flavor_fits_image: - mock_extend_virtual_disk.assert_called_once_with( - self._instance, vi.root_gb * units.Mi, - fake_path, self._dc_info.ref) + mock_extend_if_required.assert_called_once_with( + self._dc_info, + vi.ii, + vi.instance, fake_path) mock_attach_disk_to_vm.assert_called_once_with( "fake_vm_ref", self._instance, vi.ii.adapter_type, @@ -1636,8 +1930,13 @@ def _verify_spawn_method_calls(self, mock_call_method, extras=None): recorded_methods = [c[1][1] for c in mock_call_method.mock_calls] self.assertEqual(expected_methods, recorded_methods) + @mock.patch('nova.utils.vm_needs_special_spawning', + return_value=False) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch( 'nova.virt.vmwareapi.vmops.VMwareVMOps._update_vnic_index') @mock.patch( @@ -1674,11 +1973,13 @@ def _test_spawn(self, mock_get_datastore, mock_configure_config_drive, mock_update_vnic_index, + mock_update_cluster_placement, mock_create_folders, + mock_is_volume_backed, + mock_special_spawning, block_device_info=None, extra_specs=None, config_drive=False): - if extra_specs is None: extra_specs = vm_util.ExtraSpecs() @@ -1687,6 +1988,7 @@ def _test_spawn(self, 'id': self._image_id, 'disk_format': 'vmdk', 'size': image_size, + 'owner': '' } image = objects.ImageMeta.from_dict(image) image_info = images.VMwareImage( @@ -1717,10 +2019,18 @@ def _test_spawn(self, mock.patch.object(self._vmops, '_get_extra_specs', return_value=extra_specs), mock.patch.object(self._vmops, '_get_instance_metadata', - return_value='fake-metadata') + return_value='fake-metadata'), + mock.patch.object(ds_util, 'file_size', return_value=0), + mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + '_find_image_template_vm', + return_value=None), + mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + '_fetch_image_from_other_datastores', + return_value=None) ) as (_wait_for_task, _call_method, _generate_uuid, _fetch_image, _get_img_svc, _get_inventory_path, _get_extra_specs, - _get_instance_metadata): + _get_instance_metadata, file_size, _find_image_template_vm, + _fetch_image_from_other_datastores): self._vmops.spawn(self._context, self._instance, image, injected_files='fake_files', admin_password='password', @@ -1740,6 +2050,7 @@ def _test_spawn(self, extra_specs, constants.DEFAULT_OS_TYPE, profile_spec=None, + vm_name=None, metadata='fake-metadata') mock_create_vm.assert_called_once_with( self._session, @@ -1819,11 +2130,14 @@ def _test_spawn(self, mock_update_vnic_index.assert_called_once_with( self._context, self._instance, network_info) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(ds_util, 'get_datastore') @mock.patch.object(vmops.VMwareVMOps, 'get_datacenter_ref_and_name') def _test_get_spawn_vm_config_info(self, mock_get_datacenter_ref_and_name, mock_get_datastore, + mock_is_volume_backed, image_size_bytes=0): image_info = images.VMwareImage( image_id=self._image_id, @@ -1910,6 +2224,8 @@ def test_spawn_with_block_device_info_swap(self): 'device_name': '/dev/sdb'}} self._test_spawn(block_device_info=block_device_info) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vm_util, 'rename_vm') @mock.patch('nova.virt.vmwareapi.vm_util.power_on_instance') @mock.patch.object(vmops.VMwareVMOps, '_create_and_attach_thin_disk') @@ -1918,19 +2234,22 @@ def test_spawn_with_block_device_info_swap(self): @mock.patch( 'nova.virt.vmwareapi.imagecache.ImageCacheManager.enlist_image') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_with_ephemerals_and_swap(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, enlist_image, fetch_image, use_disk_image, create_and_attach_thin_disk, power_on_instance, - rename_vm): + rename_vm, + is_volume_backed_instance): self._instance.flavor = objects.Flavor(vcpus=1, memory_mb=512, name="m1.tiny", root_gb=1, ephemeral_gb=1, swap=512, @@ -1969,6 +2288,7 @@ def test_spawn_with_ephemerals_and_swap(self, from_image, get_vm_config_info.assert_called_once_with(self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, metadata) enlist_image.assert_called_once_with(image_info.image_id, vi.datastore, vi.dc_info.ref) @@ -2105,14 +2425,17 @@ def test_create_swap_with_bdi(self): def test_create_swap_with_no_bdi(self): self._test_create_swap_from_instance(None) + @mock.patch('nova.utils.vm_needs_special_spawning') @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') - def test_build_virtual_machine(self, mock_create_folder): + def test_build_virtual_machine(self, mock_create_folder, + mock_special_spawning): image = images.VMwareImage(image_id=self._image_id) extra_specs = vm_util.ExtraSpecs() vm_ref = self._vmops.build_virtual_machine(self._instance, + self._context, image, self._dc_info, self._ds, self.network_info, @@ -2148,7 +2471,7 @@ def test_build_virtual_machine(self, mock_create_folder): # Test that the VM's network is configured as specified devices = vm.get('config.hardware.device').VirtualDevice for device in devices: - if device.obj_name != 'ns0:VirtualE1000': + if device.obj_name != 'ns0:VirtualE1000e': continue self.assertEqual(self._network_values['address'], device.macAddress) @@ -2240,6 +2563,8 @@ def _validate_extra_specs(self, expected, actual): actual.cpu_limits.shares_level) self.assertEqual(expected.cpu_limits.shares_share, actual.cpu_limits.shares_share) + self.assertEqual(expected.hw_version, + actual.hw_version) def _validate_flavor_extra_specs(self, flavor_extra_specs, expected): # Validate that the extra specs are parsed correctly @@ -2407,6 +2732,19 @@ def test_extra_specs_vif_shares_with_invalid_level(self): self.assertRaises(exception.InvalidInput, self._validate_flavor_extra_specs, flavor_extra_specs, extra_specs) + def test_extra_specs_hw_version_override(self): + CONF.set_override('default_hw_version', 'vmx-13', + 'vmware') + flavor_extra_specs = {'vmware:hw_version': 'vmx-14'} + extra_specs = vm_util.ExtraSpecs(hw_version='vmx-14') + self._validate_flavor_extra_specs(flavor_extra_specs, extra_specs) + + def test_extra_specs_hw_version_override_empty(self): + CONF.set_override('default_hw_version', 'vmx-13', + 'vmware') + extra_specs = vm_util.ExtraSpecs(hw_version='vmx-13') + self._validate_flavor_extra_specs({}, extra_specs) + def _make_vm_config_info(self, is_iso=False, is_sparse_disk=False, vsphere_location=None): disk_type = (constants.DISK_TYPE_SPARSE if is_sparse_disk @@ -2439,7 +2777,13 @@ def _make_vm_config_info(self, is_iso=False, is_sparse_disk=False, @mock.patch.object(vmops.VMwareVMOps, '_cache_flat_image') @mock.patch.object(vmops.VMwareVMOps, '_delete_datastore_file') @mock.patch.object(vmops.VMwareVMOps, '_update_image_size') + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) def _test_fetch_image_if_missing(self, + mock_fetch_image_from_other_datastores, + mock_find_image_template_vm, mock_update_image_size, mock_delete_datastore_file, mock_cache_flat_image, @@ -2596,9 +2940,15 @@ def test_fetch_image_as_file_exception(self, cookies='Fake-CookieJar') @mock.patch.object(images, 'fetch_image_stream_optimized', - return_value=123) - def test_fetch_image_as_vapp(self, mock_fetch_image): + return_value=(123, '123')) + @mock.patch.object(vmops.VMwareVMOps, '_get_project_folder') + @mock.patch.object(vmops.VMwareVMOps, '_get_image_template_vm_name', + return_value='fake-name') + def test_fetch_image_as_vapp(self, mock_template_vm_name, + mock_get_project_folder, + mock_fetch_image): vi = self._make_vm_config_info() + mock_get_project_folder.return_value = vi.dc_info.vmFolder image_ds_loc = mock.Mock() image_ds_loc.parent.basename = 'fake-name' self._vmops._fetch_image_as_vapp(self._context, vi, image_ds_loc) @@ -2612,12 +2962,23 @@ def test_fetch_image_as_vapp(self, mock_fetch_image): self._vmops._root_resource_pool) self.assertEqual(vi.ii.file_size, 123) - @mock.patch.object(images, 'fetch_image_ova', return_value=123) - def test_fetch_image_as_ova(self, mock_fetch_image): + @mock.patch.object(images, 'fetch_image_ova', return_value=(123, '123')) + @mock.patch.object(vmops.VMwareVMOps, '_get_project_folder') + @mock.patch.object(vmops.VMwareVMOps, '_get_image_template_vm_name', + return_value='fake-name') + def test_fetch_image_as_ova(self, mock_template_vm_name, + mock_get_project_folder, + mock_fetch_image): vi = self._make_vm_config_info() + mock_get_project_folder.return_value = vi.dc_info.vmFolder image_ds_loc = mock.Mock() image_ds_loc.parent.basename = 'fake-name' self._vmops._fetch_image_as_ova(self._context, vi, image_ds_loc) + + mock_template_vm_name.assert_called_once_with( + vi.ii.image_id, vi.datastore.name + ) + mock_fetch_image.assert_called_once_with( self._context, vi.instance, @@ -2914,7 +3275,7 @@ def test_attach_interface(self, mock_get_vm_ref, self._vmops._network_api = _network_api vif_info = vif.get_vif_dict(self._session, self._cluster, - 'VirtualE1000', self._network_values) + 'VirtualE1000e', self._network_values) extra_specs = vm_util.ExtraSpecs() mock_extra_specs.return_value = extra_specs self._vmops.attach_interface(self._context, self._instance, @@ -3018,7 +3379,7 @@ def test_attach_interface_with_limits(self, mock_get_vm_ref, self._vmops._network_api = _network_api vif_info = vif.get_vif_dict(self._session, self._cluster, - 'VirtualE1000', self._network_values) + 'VirtualE1000e', self._network_values) vif_limits = vm_util.Limits(shares_level='custom', shares_share=40) extra_specs = vm_util.ExtraSpecs(vif_limits=vif_limits) diff --git a/nova/tests/unit/volume/test_cinder.py b/nova/tests/unit/volume/test_cinder.py index 5c5914487c5..9b6f322d5d8 100644 --- a/nova/tests/unit/volume/test_cinder.py +++ b/nova/tests/unit/volume/test_cinder.py @@ -670,6 +670,10 @@ def test_detach(self, mock_cinderclient): @mock.patch('nova.volume.cinder.cinderclient') def test_detach_no_attachment_id(self, mock_cinderclient): + """This behaves like the multiattach case. It searches the + attachment_id in the cinder attachments, because we ran into problems + with "ghost attachments" because of MessagingTimeout on attach. + """ attachment = {'server_id': 'fake_uuid', 'attachment_id': 'fakeid' } @@ -683,7 +687,7 @@ def test_detach_no_attachment_id(self, mock_cinderclient): self.api.detach(self.ctx, 'id1', instance_uuid='fake_uuid') mock_cinderclient.assert_called_with(self.ctx, microversion=None) - mock_volumes.detach.assert_called_once_with('id1', None) + mock_volumes.detach.assert_called_once_with('id1', 'fakeid') @mock.patch('nova.volume.cinder.cinderclient') def test_detach_no_attachment_id_multiattach(self, mock_cinderclient): @@ -702,6 +706,52 @@ def test_detach_no_attachment_id_multiattach(self, mock_cinderclient): mock_cinderclient.assert_called_with(self.ctx, microversion=None) mock_volumes.detach.assert_called_once_with('id1', 'fakeid') + @mock.patch('nova.volume.cinder.cinderclient') + def test_detach_no_attachment_id_ghost_attachment_no_others( + self, mock_cinderclient): + """If we have no attachment_id and we can't find an attachment for the + instance_uuid in Cinder and we have not attachments reported by Cinder, + we don't do a detach call without attachment_id. + We call without attachment_id, because we need to reset the state of + the volume in Cinder. Since there's no other attachment reported, we + can't delete an attachment for another instance in Cinder. + """ + mock_volumes = mock.MagicMock() + mock_cinderclient.return_value = mock.MagicMock(version='2', + volumes=mock_volumes) + mock_cinderclient.return_value.volumes.get.return_value = \ + FakeVolume('id1', attachments=[]) + + self.api.detach(self.ctx, 'id1', instance_uuid='fake_uuid') + + mock_cinderclient.assert_called_with(self.ctx, microversion=None) + mock_volumes.detach.assert_called_once_with('id1', None) + + @mock.patch('nova.volume.cinder.cinderclient') + def test_detach_no_attachment_id_ghost_attachment_with_others( + self, mock_cinderclient): + """If we have no attachment_id and we can't find an attachment for the + instance_uuid in Cinder and we have attachments reported by Cinder, we + don't do a detach call. The reason is, that the attachment probably + didn't work because of MessagingTimeout on reserving the block device + name. Then we only have a block device mapping without connection_info, + which also has updated_at set to NULL. + """ + attachment = {'server_id': 'fake_uuid2', + 'attachment_id': 'fakeid' + } + + mock_volumes = mock.MagicMock() + mock_cinderclient.return_value = mock.MagicMock(version='2', + volumes=mock_volumes) + mock_cinderclient.return_value.volumes.get.return_value = \ + FakeVolume('id1', attachments=[attachment]) + + self.api.detach(self.ctx, 'id1', instance_uuid='fake_uuid') + + mock_cinderclient.assert_called_with(self.ctx, microversion=None) + mock_volumes.detach.assert_not_called() + @mock.patch('nova.volume.cinder.cinderclient') def test_detach_internal_server_error(self, mock_cinderclient): mock_cinderclient.return_value.volumes.detach.side_effect = ( diff --git a/nova/utils.py b/nova/utils.py index 429cf964fcc..6c8de175fdc 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -80,6 +80,13 @@ QUOTA_SEPARATE_KEY = 'quota:separate' QUOTA_INSTANCE_ONLY_KEY = 'quota:instance_only' +# Custom resource for reservable memory +MEMORY_RESERVABLE_MB_RESOURCE = 'CUSTOM_MEMORY_RESERVABLE_MB' +MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY = \ + 'resources:' + MEMORY_RESERVABLE_MB_RESOURCE + +BIGVM_EXCLUSIVE_TRAIT = 'CUSTOM_HANA_EXCLUSIVE_HOST' + _FILE_CACHE = {} _SERVICE_TYPES = service_types.ServiceTypes() @@ -135,13 +142,6 @@ def generate_random_string(size=8): return ''.join([random.choice(characters) for _x in range(size)]) -# Default symbols to use for passwords. Avoids visually confusing characters. -# ~6 bits per symbol -DEFAULT_PASSWORD_SYMBOLS = ('23456789', # Removed: 0,1 - 'ABCDEFGHJKLMNPQRSTUVWXYZ', # Removed: I, O - 'abcdefghijkmnopqrstuvwxyz') # Removed: l - - def last_completed_audit_period(unit=None, before=None): """This method gives you the most recently *completed* audit period. @@ -232,7 +232,7 @@ def last_completed_audit_period(unit=None, before=None): return (begin, end) -def generate_password(length=None, symbolgroups=DEFAULT_PASSWORD_SYMBOLS): +def generate_password(length=None, symbolgroups=None): """Generate a random password from the supplied symbol groups. At least one symbol from each group will be included. Unpredictable @@ -244,12 +244,19 @@ def generate_password(length=None, symbolgroups=DEFAULT_PASSWORD_SYMBOLS): if length is None: length = CONF.password_length + if symbolgroups is None: + symbolgroups = CONF.password_symbol_groups + r = random.SystemRandom() # NOTE(jerdfelt): Some password policies require at least one character # from each group of symbols, so start off with one random character # from each symbol group - password = [r.choice(s) for s in symbolgroups] + # NOTE(fwiesel): And some policies require even more of them, so + # do it as often as configured in DEFAULT.password_all_group_samples + password = [r.choice(s) + for s in symbolgroups * CONF.password_all_group_samples + if s] # If length < len(symbolgroups), the leading characters will only # be from the first length groups. Try our best to not be predictable # by shuffling and then truncating. @@ -1292,3 +1299,28 @@ def is_baremetal_host(host_state): def is_baremetal_flavor(flavor): return 'capabilities:cpu_arch' in flavor.extra_specs + + +def is_big_vm(memory_mb, flavor): + # small VMs are not big + if memory_mb < CONF.bigvm_mb: + return False + + # baremetal instances are not big + if is_baremetal_flavor(flavor): + return False + + return True + + +def vm_needs_special_spawning(memory_mb, flavor): + if is_big_vm(memory_mb, flavor): + return True + + if is_baremetal_flavor(flavor): + return False + + if flavor.extra_specs.get('spawn_on_free_host', 'false').lower() == 'true': + return True + + return False diff --git a/nova/virt/driver.py b/nova/virt/driver.py index a268d435d11..941c9a37ec3 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -278,6 +278,7 @@ class ComputeDriver(object): "has_imagecache": False, "supports_evacuate": False, "supports_migrate_to_same_host": False, + "resource_scheduling": False, "supports_attach_interface": False, "supports_device_tagging": False, "supports_tagged_attach_interface": False, @@ -314,6 +315,7 @@ class ComputeDriver(object): "supports_image_type_vhdx": False, "supports_image_type_vmdk": False, "supports_image_type_ploop": False, + "driver_specific_device_name": False } # Indicates if this driver will rebalance nodes among compute service @@ -2010,6 +2012,13 @@ def cleanup_lingering_instance_resources(self, instance): """ return True + def sync_server_group(self, context, sg_uuid): + """Sync that specific server-group into the backend + + If the driver manages multiple HVs in a cluster, this method will be + called if a customer changed a server-group for this host via API. + """ + def load_compute_driver(virtapi, compute_driver=None): """Load a compute driver module. diff --git a/nova/virt/fake.py b/nova/virt/fake.py index c0aa5ba13a3..f4c9cb86e91 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -704,6 +704,11 @@ def update_compute_provider_status(self, context, rp_uuid, enabled): pass +class FakeComputeVirtAPI(FakeVirtAPI): + def __init__(self, compute): + self._compute = compute + + class SmallFakeDriver(FakeDriver): # The api samples expect specific cpu memory and disk sizes. In order to # allow the FakeVirt driver to be used outside of the unit tests, provide diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 361522e8e30..7faff7b018e 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -441,6 +441,7 @@ def __init__(self, virtapi, read_only=False): self.capabilities = { "has_imagecache": True, + "resource_scheduling": False, "supports_evacuate": True, "supports_migrate_to_same_host": False, "supports_attach_interface": True, @@ -470,6 +471,7 @@ def __init__(self, virtapi, read_only=False): self.image_backend.backend().SUPPORTS_LUKS, "supports_ephemeral_encryption_luks": self.image_backend.backend().SUPPORTS_LUKS, + "driver_specific_device_name": True, } super(LibvirtDriver, self).__init__(virtapi) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py new file mode 100644 index 00000000000..c925604d9cc --- /dev/null +++ b/nova/virt/vmwareapi/cluster_util.py @@ -0,0 +1,414 @@ +# Copyright (c) 2013 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_log import log as logging +from oslo_vmware import vim_util + +from nova import exception +from nova.i18n import _ +from nova import utils + +LOG = logging.getLogger(__name__) + + +def reconfigure_cluster(session, cluster, config_spec): + reconfig_task = session._call_method( + session.vim, "ReconfigureComputeResource_Task", + cluster, spec=config_spec, + modify=True) + session.wait_for_task(reconfig_task) + + +def create_vm_group(client_factory, name, vm_refs): + """Create a ClusterVmGroup object + """ + group = client_factory.create('ns0:ClusterVmGroup') + group.name = name + group.vm = vm_refs + + return group + + +def create_host_group(client_factory, name, host_refs, group=None): + """Create a ClusterHostGroup object + + :param:group: if given, update the ClusterHostGroup object instead of + creating a new one + """ + group = group or client_factory.create('ns0:ClusterHostGroup') + group.name = name + + if hasattr(group, 'host'): + group.host += host_refs + else: + group.host = host_refs + + return group + + +def create_group_spec(client_factory, group, operation): + """Create a ClusterGroupSpec object""" + if operation not in ('add', 'edit', 'remove'): + msg = 'Invalid operation for ClusterGroupSpec: {}'.format(operation) + raise exception.ValidationError(msg) + + group_spec = client_factory.create('ns0:ClusterGroupSpec') + group_spec.operation = operation + group_spec.info = group + if operation == 'remove': + group_spec.removeKey = group.name + + return group_spec + + +def _create_host_group_spec(client_factory, name, host_refs, operation="add", + group=None): + group = group or client_factory.create('ns0:ClusterHostGroup') + group.name = name + + if hasattr(group, 'host'): + group.host += host_refs + else: + group.host = host_refs + + group_spec = client_factory.create('ns0:ClusterGroupSpec') + group_spec.operation = operation + group_spec.info = group + return group_spec + + +def _get_vm_group(cluster_config, group_name): + if not hasattr(cluster_config, 'group'): + return None + for group in cluster_config.group: + if group.name == group_name: + return group + return None + + +def fetch_cluster_groups(session, cluster_ref=None, cluster_config=None, + group_type=None): + """Fetch all groups of a cluster + + The cluster can be identified by a cluster_ref or by an explicit + cluster_config. If identified by cluster_ref, we fetch the cluster_config. + + If the caller only needs either HostGroup or VmGroup, group_type can be set + to 'host' or 'vm' respectively. + """ + if group_type not in (None, 'vm', 'host'): + msg = 'Invalid group_type {}'.format(group_type) + raise exception.ValidationError(msg) + + if (cluster_config, cluster_ref) == (None, None): + msg = 'Either cluster_config or cluster_ref must be given.' + raise exception.ValidationError(msg) + + if cluster_config is None: + cluster_config = session._call_method( + vim_util, "get_object_property", cluster_ref, "configurationEx") + + groups = {} + for group in getattr(cluster_config, 'group', []): + if group_type == 'vm': + if not vim_util.is_vim_instance(group, 'ClusterVmGroup'): + continue + elif group_type == 'host': + if not vim_util.is_vim_instance(group, 'ClusterHostGroup'): + continue + + groups[group.name] = group + + return groups + + +def fetch_cluster_rules(session, cluster_ref=None, cluster_config=None): + """Fetch all DRS rules of a cluster + + The cluster can be identified by a cluster_ref or by an explicit + cluster_config. If identified by cluster_ref, we fetch the cluster_config. + """ + if (cluster_config, cluster_ref) == (None, None): + msg = 'Either cluster_config or cluster_ref must be given.' + raise exception.ValidationError(msg) + + if cluster_config is None: + cluster_config = session._call_method( + vim_util, "get_object_property", cluster_ref, "configurationEx") + + return {r.name: r for r in getattr(cluster_config, 'rule', [])} + + +def delete_vm_group(session, cluster, vm_group): + """Add delete impl for removing group if deleted vm is the + last vm in a vm group + """ + client_factory = session.vim.client.factory + + group_spec = create_group_spec(client_factory, vm_group, "remove") + + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.groupSpec = [group_spec] + + reconfigure_cluster(session, cluster, config_spec) + + +@utils.synchronized('vmware-vm-group-policy') +def update_vm_group_membership(session, cluster, vm_group_name, vm_ref, + remove=False): + """Updates cluster for vm placement using DRS + + If remove is set to false (default), add the VM to a Vm-group with + the given name and cluster. Create it if missing. + Otherwise, remove the vm from the group. The group itself won't get removed + even if there are no instances left. + + The assumption is that an administrator defines rules with other means + on that group. + """ + cluster_config = session._call_method( + vim_util, "get_object_property", cluster, "configurationEx") + + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + + group = _get_vm_group(cluster_config, vm_group_name) + + if not group: + if remove: + return + group = create_vm_group(client_factory, vm_group_name, [vm_ref]) + operation = "add" + elif not remove: + if not hasattr(group, "vm"): + group.vm = [vm_ref] + else: + group.vm.append(vm_ref) + operation = "edit" + else: # group and remove + if not hasattr(group, "vm"): + return + filtered_vms = [] + found = False + vm_ref_value = vim_util.get_moref_value(vm_ref) + for ref in group.vm: + if vim_util.get_moref_value(ref) == vm_ref_value: + found = True + else: + filtered_vms.append(ref) + if not found: + return + group.vm = filtered_vms + operation = "edit" + + group_spec = create_group_spec(client_factory, group, operation) + config_spec.groupSpec = [group_spec] + + reconfigure_cluster(session, cluster, config_spec) + + +def create_vm_rule(client_factory, name, vm_refs, policy='affinity', + rule=None): + """Create a ClusterAffinityRuleSpec or ClusterAntiAffinityRuleSpec object + + :param:policy: Defines with of the object types is created. "affinity" and + "soft-affinity" map to ClusterAffinityRuleSpec, + "anti-affinity" and "soft-anti-affinity" map to + ClusterAntiAffinityRuleSpec. Ignored if "rule" is given + :param:rule: if given, don't create any object, but instead update the + given object's attributes. "policy" is ignored here. + """ + if rule is None: + if policy in ('affinity', 'soft-affinity'): + obj_type = 'ns0:ClusterAffinityRuleSpec' + elif policy in ('anti-affinity', 'soft-anti-affinity'): + obj_type = 'ns0:ClusterAntiAffinityRuleSpec' + else: + msg = 'Policy {} is not supported.'.format(policy) + raise exception.ValidationError(msg) + + rule = client_factory.create(obj_type) + + rule.name = name + rule.enabled = True + rule.vm = vm_refs + + return rule + + +def create_rule_spec(client_factory, rule, operation='add'): + """Create a ClusterRuleSpec object""" + rule_spec = client_factory.create('ns0:ClusterRuleSpec') + rule_spec.operation = operation + rule_spec.info = rule + if operation == 'remove': + rule_spec.removeKey = rule.key + return rule_spec + + +def _create_cluster_rules_spec(client_factory, name, vm_refs, + policy='affinity', operation="add", + rule=None): + if operation == 'edit': + vm_refs = vm_refs + rule.vm + rule = create_vm_rule(client_factory, name, vm_refs, policy, rule) + return create_rule_spec(client_factory, rule, operation) + + +def _create_cluster_group_rules_spec(client_factory, name, vm_group_name, + host_group_name, policy='affinity', + rule=None): + rules_info = client_factory.create('ns0:ClusterVmHostRuleInfo') + rules_info.name = name + rules_info.enabled = True + rules_info.mandatory = True + rules_info.vmGroupName = vm_group_name + if policy == 'affinity': + rules_info.affineHostGroupName = host_group_name + elif policy == 'anti-affinity': + rules_info.antiAffineHostGroupName = host_group_name + else: + msg = _('%s policy is not supported.') % policy + raise exception.ValidationError(msg) + + if rule is not None: + rules_info.key = rule.key + rules_info.ruleUuid = rule.ruleUuid + + operation = 'add' if rule is None else 'edit' + return create_rule_spec(client_factory, rules_info, operation) + + +def add_rule(session, cluster_ref, rule): + """Add the given DRS rule to the given cluster""" + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.rulesSpec = [create_rule_spec(client_factory, rule, 'add')] + reconfigure_cluster(session, cluster_ref, config_spec) + + +def get_rule(session, cluster_ref, rule_name): + """Get a DRS rule from the cluster by name""" + cluster_config = session._call_method( + vim_util, "get_object_property", cluster_ref, "configurationEx") + return _get_rule(cluster_config, rule_name) + + +def _get_rule(cluster_config, rule_name): + if not hasattr(cluster_config, 'rule'): + return + for rule in cluster_config.rule: + if rule.name == rule_name: + return rule + + +def get_rules_by_prefix(session, cluster_ref, rule_prefix): + """Get all DRS rules starting with the given prefix + + Useful, if you don't know the policy of a server-group the rule was created + for. + """ + cluster_config = session._call_method( + vim_util, "get_object_property", cluster_ref, "configurationEx") + + return [rule for rule in getattr(cluster_config, 'rule', []) + if rule.name.startswith(rule_prefix)] + + +def delete_rule(session, cluster_ref, rule): + """Delete the given DRS rule from the given cluster""" + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.rulesSpec = [create_rule_spec(client_factory, rule, 'remove')] + reconfigure_cluster(session, cluster_ref, config_spec) + + +def update_rule(session, cluster_ref, rule): + """Update the already modified DRS rule in the cluster""" + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.rulesSpec = [create_rule_spec(client_factory, rule, 'edit')] + reconfigure_cluster(session, cluster_ref, config_spec) + + +def is_drs_enabled(session, cluster): + """Check if DRS is enabled on a given cluster""" + drs_config = session._call_method(vim_util, "get_object_property", cluster, + "configuration.drsConfig") + if drs_config and hasattr(drs_config, 'enabled'): + return drs_config.enabled + + return False + + +def update_cluster_drs_vm_override(session, cluster, vm_ref, operation='add', + behavior=None, enabled=True): + """Add/Update `ClusterDrsVmConfigSpec` for a VM. + + `behavior` can be any `DrsBehaviour` as string. + + `behavior` and `enabled` are only used if `operation` is `add`. + """ + if operation not in ('add', 'remove'): + msg = _('%s operation for ClusterDrsVmConfigSpec not supported.') + raise exception.ValidationError(msg % operation) + + client_factory = session.vim.client.factory + + drs_vm_spec = client_factory.create('ns0:ClusterDrsVmConfigSpec') + drs_vm_spec.operation = operation + + if operation == 'add': + if behavior is None: + msg = _('behavior cannot be unset for operation "add"') + raise exception.ValidationError(msg) + drs_vm_info = client_factory.create('ns0:ClusterDrsVmConfigInfo') + drs_vm_info.behavior = behavior + drs_vm_info.enabled = enabled + drs_vm_info.key = vm_ref + + drs_vm_spec.info = drs_vm_info + + elif operation == 'remove': + drs_vm_spec.removeKey = vm_ref + + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.drsVmConfigSpec = [drs_vm_spec] + + reconfigure_cluster(session, cluster, config_spec) + + +def fetch_cluster_drs_vm_overrides(session, cluster_ref=None, + cluster_config=None): + """Fetch all enabled DRS overrides of the cluster + + The cluster can be identified by a cluster_ref or by an explicit + cluster_config. If identified by cluster_ref, we fetch the cluster_config. + + Returns a dict (key, behavior) where key is the VM moref value and behavior + a `DrsBehaviour` as string. + """ + if (cluster_config, cluster_ref) == (None, None): + msg = 'Either cluster_config or cluster_ref must be given.' + raise exception.ValidationError(msg) + + if cluster_config is None: + cluster_config = session._call_method( + vim_util, "get_object_property", cluster_ref, "configurationEx") + + overrides = getattr(cluster_config, 'drsVmConfig', []) + return {vim_util.get_moref_value(o.key): o.behavior for o in overrides + if o.enabled} diff --git a/nova/virt/vmwareapi/constants.py b/nova/virt/vmwareapi/constants.py index 5e74d879d48..f75ac167ce8 100644 --- a/nova/virt/vmwareapi/constants.py +++ b/nova/virt/vmwareapi/constants.py @@ -46,13 +46,16 @@ DATASTORE_TYPE_VSAN = 'vsan' DATASTORE_TYPE_VVOL = 'VVOL' -DEFAULT_VIF_MODEL = network_model.VIF_MODEL_E1000 +DEFAULT_VIF_MODEL = network_model.VIF_MODEL_E1000E DEFAULT_OS_TYPE = "otherGuest" DEFAULT_ADAPTER_TYPE = "lsiLogic" DEFAULT_DISK_TYPE = DISK_TYPE_PREALLOCATED DEFAULT_DISK_FORMAT = DISK_FORMAT_VMDK DEFAULT_CONTAINER_FORMAT = CONTAINER_FORMAT_BARE +DRS_BEHAVIOR_FULLY_AUTOMATED = 'fullyAutomated' +DRS_BEHAVIOR_PARTIALLY_AUTOMATED = 'partiallyAutomated' + IMAGE_VM_PREFIX = "OSTACK_IMG" SNAPSHOT_VM_PREFIX = "OSTACK_SNAP" @@ -69,6 +72,10 @@ EXTENSION_KEY = 'org.openstack.compute' EXTENSION_TYPE_INSTANCE = 'instance' +# extension and type used by vCLS VMs created by DRS beginning from vSphere 7 +VCLS_EXTENSION_KEY = 'com.vmware.vim.eam' +VCLS_EXTENSION_TYPE_AGENT = 'cluster-agent' + # The max number of devices that can be connected to one adapter # One adapter has 16 slots but one reserved for controller SCSI_MAX_CONNECT_NUMBER = 15 @@ -234,3 +241,9 @@ POWER_STATES = {'poweredOff': power_state.SHUTDOWN, 'poweredOn': power_state.RUNNING, 'suspended': power_state.SUSPENDED} + + +# Prefix used in the name of DRS groups and rules created by the driver to +# distinguish between what has been automatically created and what is +# admin-created. +DRS_PREFIX = 'NOVA_' diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 69b38f576d8..e8394d5db5a 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -18,14 +18,18 @@ """ A connection to the VMware vCenter platform. """ - +import contextlib +from operator import attrgetter import os +import random import re +import time import urllib.request import os_resource_classes as orc import os_traits as ot from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_utils import excutils from oslo_utils import units from oslo_utils import versionutils as v_utils @@ -43,12 +47,16 @@ from nova.i18n import _ from nova import objects import nova.privsep.path +from nova import utils +from nova.virt import block_device from nova.virt import driver +from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants -from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import host +from nova.virt.vmwareapi.rpc import VmwareRpcService from nova.virt.vmwareapi import session +from nova.virt.vmwareapi import special_spawning from nova.virt.vmwareapi import vim_util as nova_vim_util from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops @@ -61,6 +69,9 @@ TIME_BETWEEN_API_CALL_RETRIES = 1.0 MAX_CONSOLE_BYTES = 100 * units.Ki +UUID_RE = re.compile(r'[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-' + r'[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') + class VMwareVCDriver(driver.ComputeDriver): """The VC host connection object.""" @@ -69,6 +80,7 @@ class VMwareVCDriver(driver.ComputeDriver): "has_imagecache": True, "supports_evacuate": False, "supports_migrate_to_same_host": True, + "resource_scheduling": True, "supports_attach_interface": True, "supports_multiattach": False, "supports_trusted_certs": False, @@ -106,8 +118,8 @@ def __init__(self, virtapi, scheme="https"): super(VMwareVCDriver, self).__init__(virtapi) if (CONF.vmware.host_ip is None or - CONF.vmware.host_username is None or - CONF.vmware.host_password is None): + CONF.vmware.host_username is None or + CONF.vmware.host_password is None): raise Exception(_("Must specify host_ip, host_username and " "host_password to use vmwareapi.VMwareVCDriver")) @@ -141,19 +153,25 @@ def __init__(self, virtapi, scheme="https"): self._create_nodename(vim_util.get_moref_value(self._cluster_ref)) self._volumeops = volumeops.VMwareVolumeOps(self._session, self._cluster_ref) + self._vc_state = host.VCState(self._session, + self._nodename, + self._cluster_ref, + self._datastore_regex) self._vmops = vmops.VMwareVMOps(self._session, virtapi, self._volumeops, + self._vc_state, self._cluster_ref, datastore_regex=self._datastore_regex) - self._vc_state = host.VCState(self._session, - self._nodename, - self._cluster_ref, - self._datastore_regex) - + self.capabilities['resource_scheduling'] = \ + cluster_util.is_drs_enabled(self._session, self._cluster_ref) # Register the OpenStack extension self._register_openstack_extension() + virtapi._compute.additional_endpoints.extend([ + special_spawning._SpecialVmSpawningServer(self), + VmwareRpcService(self)]) + def _check_min_version(self): min_version = v_utils.convert_version_to_int(constants.MIN_VC_VERSION) next_min_ver = v_utils.convert_version_to_int( @@ -166,7 +184,7 @@ def _check_min_version(self): 'vCenter version %(min_version)s or greater.') % { 'version': vc_version, 'min_version': constants.MIN_VC_VERSION}) - elif v_utils.convert_version_to_int(vc_version) < next_min_ver: + if v_utils.convert_version_to_int(vc_version) < next_min_ver: LOG.warning('Running Nova with a VMware vCenter version less ' 'than %(version)s is deprecated. The required ' 'minimum version of vCenter will be raised to ' @@ -206,6 +224,10 @@ def init_host(self, host): if vim is None: self._session._create_session() + self._vmops.set_compute_host(host) + LOG.debug("Starting green server-group sync-loop thread") + utils.spawn(self._server_group_sync_loop, host) + def cleanup_host(self, host): self._session.logout() @@ -267,8 +289,7 @@ def migrate_disk_and_power_off(self, context, instance, dest, """ # TODO(PhilDay): Add support for timeout (clean shutdown) return self._vmops.migrate_disk_and_power_off(context, instance, - dest, flavor, - block_device_info) + dest, flavor) def confirm_migration(self, context, migration, instance, network_info): """Confirms a resize, destroying the source VM.""" @@ -289,65 +310,6 @@ def finish_migration(self, context, migration, instance, disk_info, network_info, image_meta, resize_instance, block_device_info, power_on) - def pre_live_migration(self, context, instance, block_device_info, - network_info, disk_info, migrate_data): - return migrate_data - - def post_live_migration_at_source(self, context, instance, network_info): - pass - - def post_live_migration_at_destination(self, context, instance, - network_info, - block_migration=False, - block_device_info=None): - pass - - def cleanup_live_migration_destination_check(self, context, - dest_check_data): - pass - - def live_migration(self, context, instance, dest, - post_method, recover_method, block_migration=False, - migrate_data=None): - """Live migration of an instance to another host.""" - self._vmops.live_migration(context, instance, dest, post_method, - recover_method, block_migration, - migrate_data) - - def check_can_live_migrate_source(self, context, instance, - dest_check_data, block_device_info=None): - cluster_name = dest_check_data.cluster_name - cluster_ref = vm_util.get_cluster_ref_by_name(self._session, - cluster_name) - if cluster_ref is None: - msg = (_("Cannot find destination cluster %s for live migration") % - cluster_name) - raise exception.MigrationPreCheckError(reason=msg) - res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref) - if res_pool_ref is None: - msg = _("Cannot find destination resource pool for live migration") - raise exception.MigrationPreCheckError(reason=msg) - return dest_check_data - - def check_can_live_migrate_destination(self, context, instance, - src_compute_info, dst_compute_info, - block_migration=False, - disk_over_commit=False): - # the information that we need for the destination compute node - # is the name of its cluster and datastore regex - data = objects.VMwareLiveMigrateData() - data.cluster_name = CONF.vmware.cluster_name - data.datastore_regex = CONF.vmware.datastore_regex - return data - - def rollback_live_migration_at_destination(self, context, instance, - network_info, - block_device_info, - destroy_disks=True, - migrate_data=None): - """Clean up destination node after a failed live migration.""" - self.destroy(context, instance, network_info, block_device_info) - def get_instance_disk_info(self, instance, block_device_info=None): pass @@ -409,25 +371,17 @@ def _create_nodename(self, mo_id): return '%s.%s' % (mo_id, self._vcenter_uuid) - def _get_available_resources(self, host_stats): - return {'vcpus': host_stats['vcpus'], - 'memory_mb': host_stats['host_memory_total'], - 'local_gb': host_stats['disk_total'], - 'vcpus_used': 0, - 'memory_mb_used': host_stats['host_memory_total'] - - host_stats['host_memory_free'], - 'local_gb_used': host_stats['disk_used'], - 'hypervisor_type': host_stats['hypervisor_type'], - 'hypervisor_version': host_stats['hypervisor_version'], - 'hypervisor_hostname': host_stats['hypervisor_hostname'], - # The VMWare driver manages multiple hosts, so there are - # likely many different CPU models in use. As such it is - # impossible to provide any meaningful info on the CPU - # model of the "host" - 'cpu_info': None, - 'supported_instances': host_stats['supported_instances'], - 'numa_topology': None, - } + def _get_available_resources(self, host_stats, nodename): + stats = host_stats[nodename].copy() + stats["memory_mb_reserved"] = min(stats["memory_mb"], + stats["memory_mb_reserved"] + + CONF.reserved_host_memory_mb) + stats["vcpus_reserved"] = min(stats["vcpus"], + stats["vcpus_reserved"] + + CONF.reserved_host_cpus) + stats["cpu_info"] = jsonutils.dumps(stats['cpu_info'], sort_keys=True) + + return stats def get_available_resource(self, nodename): """Retrieve resource info. @@ -438,8 +392,8 @@ def get_available_resource(self, nodename): :returns: dictionary describing resources """ - host_stats = self._vc_state.get_host_stats(refresh=True) - stats_dict = self._get_available_resources(host_stats) + host_stats = self._vc_state.get_host_stats() + stats_dict = self._get_available_resources(host_stats, nodename) return stats_dict def get_available_nodes(self, refresh=False): @@ -447,6 +401,15 @@ def get_available_nodes(self, refresh=False): This driver supports only one compute node. """ + # get_available_nodes is called at the beginning of polling + # the resources of all the nodes via + # get_inventory & get_available_resource + # We follow here the same pattern as in the ironic driver and use this + # function call as an indicator of a new polling cycle and refresh + # the host stats cached in _vc_state by calling... + self._vc_state.get_host_stats(refresh=True) + # In the following calls to get_inventory and get_available_resource + # for each node, we then return the cached data return [self._nodename] def update_provider_tree(self, provider_tree, nodename, allocations=None): @@ -496,55 +459,67 @@ def update_provider_tree(self, provider_tree, nodename, allocations=None): :raises: ReshapeFailed if the requested tree reshape fails for whatever reason. """ - # NOTE(cdent): This is a side-effecty method, we are changing the - # the provider tree in place (on purpose). + stats = self.get_available_resource(nodename) + result = {} + + # NOTE(yikun): If the inv record does not exists, the allocation_ratio + # will use the CONF.xxx_allocation_ratio value if xxx_allocation_ratio + # is set, and fallback to use the initial_xxx_allocation_ratio + # otherwise. inv = provider_tree.data(nodename).inventory ratios = self._get_allocation_ratios(inv) - stats = vm_util.get_stats_from_cluster(self._session, - self._cluster_ref) - datastores = ds_util.get_available_datastores(self._session, - self._cluster_ref, - self._datastore_regex) - total_disk_capacity = sum([ds.capacity for ds in datastores]) - max_free_space = max([ds.freespace for ds in datastores]) - reserved_disk_gb = compute_utils.convert_mb_to_ceil_gb( - CONF.reserved_host_disk_mb) - result = { - orc.VCPU: { - 'total': stats['cpu']['vcpus'], - 'reserved': CONF.reserved_host_cpus, + + local_gb = stats["local_gb"] + local_gb_max_free = stats.get("local_gb_max_free", "local_gb") + if local_gb > 0 and local_gb_max_free > 0: + reserved_disk_gb = compute_utils.convert_mb_to_ceil_gb( + CONF.reserved_host_disk_mb) + result[orc.DISK_GB] = { + 'total': local_gb, + 'reserved': reserved_disk_gb, + 'min_unit': 1, + 'max_unit': local_gb_max_free, + 'step_size': 1, + 'allocation_ratio': ratios[orc.DISK_GB], + } + + vcpus = stats["vcpus"] + max_vcpus = stats.get("max_vcpus_per_host", vcpus) + if vcpus > 0 and max_vcpus > 0: + reserved_vcpus = stats["vcpus_reserved"] + result[orc.VCPU] = { + 'total': vcpus, + 'reserved': reserved_vcpus, 'min_unit': 1, - 'max_unit': stats['cpu']['max_vcpus_per_host'], + 'max_unit': max_vcpus, 'step_size': 1, 'allocation_ratio': ratios[orc.VCPU], - }, - orc.MEMORY_MB: { - 'total': stats['mem']['total'], - 'reserved': CONF.reserved_host_memory_mb, + } + + memory_mb = stats["memory_mb"] + reserved_memory_mb = stats["memory_mb_reserved"] + max_memory_mb = stats.get("max_mem_mb_per_host", memory_mb) + if memory_mb > 0 and max_memory_mb > 0: + result[orc.MEMORY_MB] = { + 'total': memory_mb, + 'reserved': reserved_memory_mb, 'min_unit': 1, - 'max_unit': stats['mem']['max_mem_mb_per_host'], + 'max_unit': max_memory_mb, 'step_size': 1, 'allocation_ratio': ratios[orc.MEMORY_MB], - }, - } - - # If a sharing DISK_GB provider exists in the provider tree, then our - # storage is shared, and we should not report the DISK_GB inventory in - # the compute node provider. - # TODO(cdent): We don't do this yet, in part because of the issues - # in bug #1784020, but also because we can represent all datastores - # as shared providers and should do once update_provider_tree is - # working well. - if provider_tree.has_sharing_provider(orc.DISK_GB): - LOG.debug('Ignoring sharing provider - see bug #1784020') - result[orc.DISK_GB] = { - 'total': total_disk_capacity // units.Gi, - 'reserved': reserved_disk_gb, - 'min_unit': 1, - 'max_unit': max_free_space // units.Gi, - 'step_size': 1, - 'allocation_ratio': ratios[orc.DISK_GB], - } + } + + available_memory_mb = memory_mb - reserved_memory_mb + reserved_reservable_memory = int(available_memory_mb * + (1 - stats.get("vm_reservable_memory_ratio", 0))) + if available_memory_mb > 0: + result[utils.MEMORY_RESERVABLE_MB_RESOURCE] = { + 'total': available_memory_mb, + 'reserved': reserved_reservable_memory, + 'min_unit': 1, + 'max_unit': max_memory_mb, + 'step_size': 1, + } provider_tree.update_inventory(nodename, result) @@ -577,6 +552,11 @@ def attach_volume(self, context, connection_info, instance, mountpoint, def detach_volume(self, context, connection_info, instance, mountpoint, encryption=None): """Detach volume storage to VM instance.""" + if not self._vmops.is_instance_in_resource_pool(instance): + volume_id = block_device.get_volume_id(connection_info) + LOG.debug("Not detaching %s, vm is in different cluster", + volume_id, instance=instance) + return True # NOTE(claudiub): if context parameter is to be used in the future, # the _detach_instance_volumes method will have to be updated as well. return self._volumeops.detach_volume(connection_info, instance) @@ -762,3 +742,348 @@ def attach_interface(self, context, instance, image_meta, vif): def detach_interface(self, context, instance, vif): """Detach an interface from the instance.""" self._vmops.detach_interface(context, instance, vif) + + def sync_server_group(self, context, sg_uuid): + self._vmops.sync_server_group(context, sg_uuid) + + def _server_group_sync_loop(self, compute_host): + """Retrieve all groups from the cluster and from the DB and call + self.sync_server_group() for them. + + We always wait a little to prohibit overwhelming the cluster. + """ + context = nova_context.get_admin_context() + + while CONF.vmware.server_group_sync_loop_spacing >= 0: + LOG.debug('Starting server-group sync-loop') + try: + + InstanceList = objects.instance.InstanceList + instance_uuids = set(i.uuid for i in InstanceList.get_by_host( + context, compute_host, expected_attrs=[])) + + InstanceGroupList = objects.instance_group.InstanceGroupList + ig_list = InstanceGroupList.get_by_instance_uuids( + context, instance_uuids) + + sg_uuids = set(ig.uuid for ig in ig_list) + + cluster_rules = cluster_util.fetch_cluster_rules( + self._session, cluster_ref=self._cluster_ref) + for rule_name in cluster_rules: + if not rule_name.startswith(constants.DRS_PREFIX): + # not managed by us + continue + + # should look like + # NOVA_d7134f26-f2e2-42ed-b5f1-4092962e84d5-affinity + # NOVA_d7134f26-f2e2-42ed-b5f1-4092962e84d5-soft-anti-affinity + # NOVA_d7134f26-f2e2-42ed-b5f1-4092962e84d5-soft-affinity + m = UUID_RE.search(rule_name) + if not m: + continue + + sg_uuids.add(m.group(0)) + + for sg_uuid in sg_uuids: + spacing = \ + CONF.vmware.server_group_sync_loop_max_group_spacing + sleep_time = random.uniform(0.5, spacing) + time.sleep(sleep_time) + self._vmops.sync_server_group(context, sg_uuid) + except Exception as e: + LOG.exception("Finished server-group sync-loop with error: %s", + e) + else: + LOG.debug('Finished server-group sync-loop') + + time.sleep(CONF.vmware.server_group_sync_loop_spacing) + + def check_can_live_migrate_destination(self, context, instance, + src_compute_info, dst_compute_info, + block_migration=False, + disk_over_commit=False): + """Check if it is possible to execute live migration.""" + data = objects.migrate_data.VMwareLiveMigrateData() + + data.instance_already_migrated = ( + self.instance_exists(instance) and # In this vcenter + self._vmops.is_instance_in_resource_pool(instance)) + + url = "https://{}:{}".format(CONF.vmware.host_ip, + CONF.vmware.host_port) + + data.cluster_name = self._cluster_name + data.dest_cluster_ref = vim_util.get_moref_value(self._cluster_ref) + data.datastore_regex = CONF.vmware.datastore_regex + client_factory = self._session.vim.client.factory + + service = vm_util.create_service_locator(client_factory, + url, self._vcenter_uuid, + vm_util.create_service_locator_name_password(client_factory, + username=CONF.vmware.host_username, + password=CONF.vmware.host_password)) + + data.relocate_defaults = { + "service": nova_vim_util.serialize_object(service, typed=True) + } + + return data + + def cleanup_live_migration_destination_check(self, context, + dest_check_data): + """Do required cleanup on dest host after check_can_live_migrate calls + """ + + def check_can_live_migrate_source(self, context, instance, + dest_check_data, block_device_info=None): + """Check if it is possible to execute live migration.""" + + client_factory = self._session.vim.client.factory + defaults = dest_check_data.relocate_defaults + service_locator = nova_vim_util.deserialize_object(client_factory, + defaults["service"], + "ServiceLocator") + + dest_check_data.is_same_vcenter = ( + service_locator.instanceUuid == self._vcenter_uuid) + + if dest_check_data.instance_already_migrated: + return dest_check_data + + if dest_check_data.is_same_vcenter: + # Drop the service-credentials, no need to check the volumes, + # as we do not need to mess around with them + defaults.pop("service") + dest_check_data.relocate_defaults = defaults + + # Remove the DRS constraints so we can actually move it. + # We have to do it in the check, because the next step + # is the pre_live_migration and that is on the destination + # host + self._vmops.update_cluster_placement(context, instance, + remove=True) + else: + # Validate that we have all necessary information for the + # _old_ volume attachments + self._get_checked_volumes(context, instance, [ + 'volume', # For deleting the old shadow-vms + ]) + + return dest_check_data + + def pre_live_migration(self, context, instance, block_device_info, + network_info, disk_info, migrate_data): + """Prepare an instance for live migration (on the destination host)""" + if migrate_data.instance_already_migrated: + return migrate_data + + return self._pre_live_migration(context, instance, + block_device_info, network_info, disk_info, migrate_data) + + def _pre_live_migration(self, context, instance, block_device_info, + network_info, disk_info, migrate_data): + result = self._vmops.place_vm(context, instance) + + if hasattr(result, 'drsFault'): + LOG.error("Placement Error: %s", nova_vim_util.serialize_object( + result.drsFault, typed=True), instance=instance) + + if (not hasattr(result, 'recommendations') or + not result.recommendations): + raise exception.MigrationError( + reason="PlaceVM did not give any recommendations") + + rs = sorted([r for r in result.recommendations + if r.reason == "xvmotionPlacement" and + r.action], + key=attrgetter("rating")) + if not rs: + raise exception.MigrationError( + reason="Did not get any xvmotionPlacement") + + relocate_spec = rs[0].action[0].relocateSpec + + # Should never happen, but if it does we rather want an error + # here, than sometime down the line + if not relocate_spec.host: + raise exception.MigrationError( + reason="No host with enough resources") + + # Samere here: Should never happen + if not relocate_spec.datastore: + raise exception.MigrationError( + reason="No datastore with enough resources") + + # relocate_defaults are serialized/deserialized on put/get + defaults = migrate_data.relocate_defaults + spec = nova_vim_util.serialize_object(relocate_spec, typed=True) + defaults["relocate_spec"] = spec + # Writing the values back + migrate_data.relocate_defaults = defaults + + return migrate_data + + def _get_checked_volumes(self, context, instance, required_values, + exc=exception.MigrationError): + volumes = self._get_volume_mappings(context, instance) + for key, volume in volumes.items(): + for v in required_values: + if v not in volume: + message = "Missing {} for volume {} (Device {})".format( + v, volume.get("volume_id"), key + ) + raise exc(message) + return volumes + + def live_migration(self, context, instance, dest, + post_method, recover_method, block_migration=False, + migrate_data=None): + """Live migration of an instance to another host.""" + if not migrate_data: + LOG.error("live_migration() called without migration_data" + " - cannot continue operations", instance=instance) + recover_method(context, instance, dest, migrate_data) + raise ValueError("Missing migrate_data") + + if migrate_data.instance_already_migrated: + LOG.info("Recovering migration", instance=instance) + post_method(context, instance, dest, block_migration, migrate_data) + return + + try: + # We require the target-datastore for all volume-attachment + required_volume_attributes = ["datastore_ref"] + + target = self._vmops.api_for_migration(migrate_data.migration) + if not migrate_data.is_same_vcenter: + # For the shadow vm fixup after migration + required_volume_attributes.append('volume') + + # Validate that we have all necessary information for the + # new volume attachments + # This cannot be done in pre-check, as the new volume attachments + # are created prior the live-migration + volumes = self._get_checked_volumes(context, instance, + required_volume_attributes) + if 'vifs' not in migrate_data or not migrate_data.vifs: + migrate_data.vif_infos = [] + else: + dest_network_info = [ + vif.get_dest_vif() + for vif in migrate_data.vifs + ] + vif_model = None # Doesn't matter as we won't change the type + migrate_data.vif_infos = target.get_vif_info(context, + vif_model=vif_model, network_info=dest_network_info) + self._vmops.live_migration(instance, migrate_data, volumes) + LOG.info("Migration operation completed", instance=instance) + post_method(context, instance, dest, block_migration, migrate_data) + except Exception: + LOG.exception("Failed due to an exception", instance=instance) + with excutils.save_and_reraise_exception(): + # We are still in the task-state migrating, so cannot + # recover the DRS settings. We rely on the sync to do that + LOG.debug("Calling live migration recover_method " + "for instance: %s", instance["name"], + instance=instance) + recover_method(context, instance, dest, migrate_data) + + def _get_volume_mappings(self, context, instance): + """Returns a mapping for all the devices with volumes to + their connection_info.data, where possible. + It is up to the caller to verify that the required information + is available. + It uses the _current_ block-device mapping. + """ + volume_infos = [] + for bdm in objects.BlockDeviceMappingList.get_by_instance_uuid( + context, instance.uuid): + if not bdm.is_volume: + continue + # If we have volume_id, `map_volumes_to_devices` will map it to + # a device. + data = {"volume_id": bdm.volume_id} + try: + connection_info = jsonutils.loads(bdm.connection_info) + data.update(connection_info.get("data", {})) + except ValueError as e: + message = ("Could not parse connection_info for volume {}." + " Reason: {}" + ).format(bdm.volume_id, e) + LOG.warning(message, instance=instance) + + # Normalize the datastore reference + # As it depends on the caller, if actually need the + # value, we do not raise an error here + datastore = data.get("datastore") or \ + data.get("ds_ref_val") + if datastore: + data["datastore_ref"] = vim_util.get_moref(datastore, + "Datastore") + volume_infos.append(data) + return self._volumeops.map_volumes_to_devices(instance, volume_infos) + + def rollback_live_migration_at_destination(self, context, instance, + network_info, + block_device_info, + destroy_disks=True, + migrate_data=None): + """Clean up destination node after a failed live migration.""" + LOG.info("rollback_live_migration_at_destination %s", + block_device_info, instance=instance) + if not migrate_data.is_same_vcenter: + self._volumeops.delete_shadow_vms(block_device_info, instance) + + @contextlib.contextmanager + def _error_out_instance_on_exception(self, instance, message): + try: + yield + except Exception as error: + LOG.exception("Failed to %s, setting to ERROR state", + message, + instance=instance, error=error) + instance.vm_state = vm_states.ERROR + instance.save() + + def post_live_migration(self, context, instance, block_device_info, + migrate_data=None): + """Post operation of live migration at source host.""" + if not migrate_data.is_same_vcenter: + with self._error_out_instance_on_exception(instance, + "delete shadow vms"): + self._volumeops.delete_shadow_vms(block_device_info, instance) + + with self._error_out_instance_on_exception(instance, + "sync server groups"): + self._vmops.sync_instance_server_group(context, instance) + + def post_live_migration_at_source(self, context, instance, network_info): + # We have to clean up our VM-moref cache, so that when a VM with + # volume attachments comes back to this compute node after being moved + # away, the moref is not incorrect/stale and the VMware API, on volume + # attachment creation, doesn't complain that the VM + # "has already been deleted or has not been completely created". + vm_util.vm_ref_cache_delete(instance.uuid) + + def post_live_migration_at_destination(self, context, instance, + network_info, + block_migration=False, + block_device_info=None): + """Post operation of live migration at destination host.""" + with self._error_out_instance_on_exception(instance, + "disable drs"): + self._vmops.disable_drs_if_needed(instance) + + with self._error_out_instance_on_exception(instance, + "update cluster placement"): + self._vmops.update_cluster_placement(context, instance) + + with self._error_out_instance_on_exception(instance, + "fixup shadow vms"): + volumes = self._get_volume_mappings(context, instance) + LOG.debug("Fixing shadow vms %s", volumes, instance=instance) + self._volumeops.fixup_shadow_vms(instance, volumes) + + self._vmops._clean_up_after_special_spawning( + context, instance.memory_mb, instance.flavor) diff --git a/nova/virt/vmwareapi/ds_util.py b/nova/virt/vmwareapi/ds_util.py index 7ff353845f1..391fb535a91 100644 --- a/nova/virt/vmwareapi/ds_util.py +++ b/nova/virt/vmwareapi/ds_util.py @@ -51,7 +51,7 @@ def _select_datastore(session, datastores, best_match, datastore_regex=None, """Find the most preferable datastore in a given RetrieveResult object. :param session: vmwareapi session - :param datastores: an iterator to the objects of RetrieveResult + :param datastores: an iterator to the results of the datastore list :param best_match: the current best match for datastore :param datastore_regex: an optional regular expression to match names :param storage_policy: storage policy for the datastore @@ -62,12 +62,10 @@ def _select_datastore(session, datastores, best_match, datastore_regex=None, if storage_policy: matching_ds = _filter_datastores_matching_storage_policy( session, datastores, storage_policy) - if not matching_ds: - return best_match else: matching_ds = datastores - # matching_ds is actually a RetrieveResult object from vSphere API call + # data_stores is actually a RetrieveResult object from vSphere API call for obj_content in matching_ds: # the propset attribute "need not be set" by returning API if not hasattr(obj_content, 'propSet'): @@ -141,7 +139,6 @@ def get_datastore(session, cluster, datastore_regex=None, datastore_regex, storage_policy, allowed_ds_types) - if best_match: return best_match @@ -149,12 +146,11 @@ def get_datastore(session, cluster, datastore_regex=None, raise exception.DatastoreNotFound( _("Storage policy %s did not match any datastores") % storage_policy) - elif datastore_regex: + if datastore_regex: raise exception.DatastoreNotFound( _("Datastore regex %s did not match any datastores") % datastore_regex.pattern) - else: - raise exception.DatastoreNotFound() + raise exception.DatastoreNotFound() def _get_allowed_datastores(datastores, datastore_regex): @@ -173,12 +169,22 @@ def _get_allowed_datastores(datastores, datastore_regex): freespace=propdict['summary.freeSpace'])) -def get_available_datastores(session, cluster=None, datastore_regex=None): +def get_available_datastores(session, cluster=None, + datastore_regex=None, dc_ref=None): """Get the datastore list and choose the first local storage.""" - ds = session._call_method(vutil, - "get_object_property", - cluster, - "datastore") + if cluster: + ds = session._call_method(vutil, + "get_object_property", + cluster, + "datastore") + elif dc_ref: + ds = session._call_method(vutil, + "get_object_property", + dc_ref, + "datastore") + else: + return [] + if not ds: return [] datastore_mors = ds.ManagedObjectReference @@ -429,7 +435,7 @@ def _filter_datastores_matching_storage_policy(session, datastores, storage_policy): """Get datastores matching the given storage policy. - :param datastores: an iterator over objects of a RetrieveResult + :param datastores: an iterator over datastore results :param storage_policy: the storage policy name :return: an iterator to datastores conforming to the given storage policy """ diff --git a/nova/virt/vmwareapi/host.py b/nova/virt/vmwareapi/host.py index 8efc59bb61e..fb3fb65a73c 100644 --- a/nova/virt/vmwareapi/host.py +++ b/nova/virt/vmwareapi/host.py @@ -16,6 +16,7 @@ """ Management class for host-related functions (start, reboot, etc). """ +from copy import deepcopy from oslo_log import log as logging from oslo_utils import units @@ -34,33 +35,44 @@ CONF = nova.conf.CONF LOG = logging.getLogger(__name__) +SERVICE_DISABLED_REASON = 'set by vmwareapi host_state' + def _get_ds_capacity_and_freespace(session, cluster=None, datastore_regex=None): capacity = 0 freespace = 0 + max_freespace = 0 try: for ds in ds_util.get_available_datastores(session, cluster, datastore_regex): capacity += ds.capacity freespace += ds.freespace + max_freespace = max(max_freespace, ds.freespace) except exception.DatastoreNotFound: pass - return capacity, freespace + return capacity, freespace, max_freespace class VCState(object): """Manages information about the vCenter cluster""" - def __init__(self, session, host_name, cluster, datastore_regex): + def __init__(self, session, cluster_node_name, cluster, datastore_regex): super(VCState, self).__init__() self._session = session - self._host_name = host_name + self._cluster_node_name = cluster_node_name self._cluster = cluster self._datastore_regex = datastore_regex self._stats = {} - self._auto_service_disabled = False + ctx = context.get_admin_context() + try: + service = objects.Service.get_by_compute_host(ctx, CONF.host) + self._auto_service_disabled = service.disabled \ + and service.disabled_reason == SERVICE_DISABLED_REASON + except exception.ComputeHostNotFound: + # this can happened on newly-added hosts + self._auto_service_disabled = False about_info = self._session._call_method(vim_util, "get_about_info") self._hypervisor_type = about_info.name self._hypervisor_version = versionutils.convert_version_to_int( @@ -79,47 +91,63 @@ def update_status(self): """Update the current state of the cluster.""" data = {} try: - capacity, freespace = _get_ds_capacity_and_freespace(self._session, - self._cluster, self._datastore_regex) + capacity, free, max_free = _get_ds_capacity_and_freespace( + self._session, self._cluster, self._datastore_regex) # Get cpu, memory stats from the cluster - stats = vm_util.get_stats_from_cluster(self._session, - self._cluster) + per_host_stats = vm_util.get_stats_from_cluster_per_host( + self._session, self._cluster) except (vexc.VimConnectionException, vexc.VimAttributeException) as ex: # VimAttributeException is thrown when vpxd service is down LOG.warning("Failed to connect with %(node)s. " "Error: %(error)s", - {'node': self._host_name, 'error': ex}) + {'node': self._cluster_node_name, 'error': ex}) self._set_host_enabled(False) return data - data["vcpus"] = stats['cpu']['vcpus'] - data["disk_total"] = capacity // units.Gi - data["disk_available"] = freespace // units.Gi - data["disk_used"] = data["disk_total"] - data["disk_available"] - data["host_memory_total"] = stats['mem']['total'] - data["host_memory_free"] = stats['mem']['free'] - data["hypervisor_type"] = self._hypervisor_type - data["hypervisor_version"] = self._hypervisor_version - data["hypervisor_hostname"] = self._host_name - data["supported_instances"] = [ - (obj_fields.Architecture.I686, - obj_fields.HVType.VMWARE, - obj_fields.VMMode.HVM), - (obj_fields.Architecture.X86_64, - obj_fields.HVType.VMWARE, - obj_fields.VMMode.HVM)] + local_gb = capacity / units.Gi + local_gb_used = (capacity - free) / units.Gi + local_gb_max_free = max_free / units.Gi + + defaults = { + "local_gb": local_gb, + "local_gb_used": local_gb_used, + "local_gb_max_free": local_gb_max_free, + "supported_instances": [ + (obj_fields.Architecture.I686, + obj_fields.HVType.VMWARE, + obj_fields.VMMode.HVM), + (obj_fields.Architecture.X86_64, + obj_fields.HVType.VMWARE, + obj_fields.VMMode.HVM)], + "numa_topology": None, + } + + for host, stats in per_host_stats.items(): + data[host] = self._merge_stats(host, stats, defaults) + + cluster_stats = vm_util.aggregate_stats_from_cluster(per_host_stats) + cluster_stats["hypervisor_type"] = self._hypervisor_type + cluster_stats["hypervisor_version"] = self._hypervisor_version + data[self._cluster_node_name] = self._merge_stats( + self._cluster_node_name, cluster_stats, defaults) self._stats = data if self._auto_service_disabled: self._set_host_enabled(True) return data + def _merge_stats(self, host, stats, defaults): + result = deepcopy(defaults) + result["hypervisor_hostname"] = host + result.update(stats) + return result + def _set_host_enabled(self, enabled): """Sets the compute host's ability to accept new instances.""" ctx = context.get_admin_context() service = objects.Service.get_by_compute_host(ctx, CONF.host) service.disabled = not enabled - service.disabled_reason = 'set by vmwareapi host_state' + service.disabled_reason = SERVICE_DISABLED_REASON service.save() self._auto_service_disabled = service.disabled diff --git a/nova/virt/vmwareapi/images.py b/nova/virt/vmwareapi/images.py index e983fe5fada..4bcf1ffca74 100644 --- a/nova/virt/vmwareapi/images.py +++ b/nova/virt/vmwareapi/images.py @@ -17,19 +17,19 @@ Utility functions for Image transfer and manipulation. """ +from lxml import etree import os import tarfile -from lxml import etree from oslo_config import cfg from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import encodeutils from oslo_utils import strutils from oslo_utils import units +from oslo_vmware import exceptions as vexc from oslo_vmware import rw_handles - from nova import exception from nova.i18n import _ from nova.image import glance @@ -62,7 +62,8 @@ def __init__(self, image_id, file_type=constants.DEFAULT_DISK_FORMAT, linked_clone=None, vsphere_location=None, - vif_model=constants.DEFAULT_VIF_MODEL): + vif_model=constants.DEFAULT_VIF_MODEL, + owner=None): """VMwareImage holds values for use in building VMs. image_id (str): uuid of the image @@ -75,6 +76,7 @@ def __init__(self, image_id, linked_clone (bool): use linked clone, or don't vsphere_location (str): image location in datastore or None vif_model (str): virtual machine network interface + owner (str): uuid of the owning project """ self.image_id = image_id self.file_size = file_size @@ -84,6 +86,7 @@ def __init__(self, image_id, self.disk_type = disk_type self.file_type = file_type self.vsphere_location = vsphere_location + self.owner = owner # NOTE(vui): This should be removed when we restore the # descriptor-based validation. @@ -142,7 +145,8 @@ def from_image(cls, context, image_id, image_meta): 'image_id': image_id, 'linked_clone': linked_clone, 'container_format': container_format, - 'vsphere_location': get_vsphere_location(context, image_id) + 'vsphere_location': get_vsphere_location(context, image_id), + 'owner': image_meta.owner } if image_meta.obj_attr_is_set('size'): @@ -183,14 +187,20 @@ def from_image(cls, context, image_id, image_meta): def get_vsphere_location(context, image_id): """Get image location in vsphere or None.""" # image_id can be None if the instance is booted using a volume. - if image_id: + if not image_id: + return None + + try: metadata = IMAGE_API.get(context, image_id, include_locations=True) - locations = metadata.get('locations') - if locations: - for loc in locations: - loc_url = loc.get('url') - if loc_url and loc_url.startswith('vsphere://'): - return loc_url + except exception.ImageNotFound: + return None + + locations = metadata.get('locations') + if locations: + for loc in locations: + loc_url = loc.get('url') + if loc_url and loc_url.startswith('vsphere://'): + return loc_url return None @@ -347,23 +357,100 @@ def fetch_image_stream_optimized(context, instance, session, vm_name, read_iter = IMAGE_API.download(context, image_ref) read_handle = rw_handles.ImageReadHandle(read_iter) - write_handle = rw_handles.VmdkWriteHandle(session, - session._host, - session._port, - res_pool_ref, - vm_folder_ref, - vm_import_spec, - file_size) - image_transfer(read_handle, write_handle) - - imported_vm_ref = write_handle.get_imported_vm() + imported_vm_ref = _import_image(session, read_handle, vm_import_spec, + vm_name, vm_folder_ref, res_pool_ref, + file_size) LOG.info("Downloaded image file data %(image_ref)s", {'image_ref': instance.image_ref}, instance=instance) vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) - session._call_method(session.vim, "UnregisterVM", imported_vm_ref) - LOG.info("The imported VM was unregistered", instance=instance) - return vmdk.capacity_in_bytes + vm_util.mark_vm_as_template(session, instance, imported_vm_ref) + return vmdk.capacity_in_bytes, vmdk.path + + +def _import_image(session, read_handle, vm_import_spec, vm_name, vm_folder_ref, + res_pool_ref, file_size): + # retry in order to handle conflicts in case of parallel execution + # (multiple agents or previously failed import of the same image + max_attempts = 3 + imported_vm_ref = None + for i in range(max_attempts): + try: + write_handle = rw_handles.VmdkWriteHandle(session, + session._host, + session._port, + res_pool_ref, + vm_folder_ref, + vm_import_spec, + file_size) + image_transfer(read_handle, write_handle) + imported_vm_ref = write_handle.get_imported_vm() + + break + except vexc.DuplicateName: + LOG.debug("Handling name duplication during import of VM %s", + vm_name) + vm_ref = vm_util.get_vm_ref_from_name(session, vm_name) + waited_for_ongoing_import = _wait_for_import_task(session, vm_ref) + if waited_for_ongoing_import: + imported_vm_ref = vm_ref + break + else: + try: + destroy_task = session._call_method(session.vim, + "Destroy_Task", + vm_ref) + session._wait_for_task(destroy_task) + vm_util.vm_ref_cache_delete(vm_name) + except vexc.ManagedObjectNotFoundException: + # another agent destroyed the VM in the meantime + pass + + if not imported_vm_ref: + raise vexc.VMwareDriverException("Could not import image" + " %s within %d attempts." + % (vm_name, max_attempts)) + return imported_vm_ref + + +def _wait_for_import_task(session, vm_ref): + client_factory = session.vim.client.factory + task_filter_spec = client_factory.create('ns0:TaskFilterSpec') + task_filter_spec.entity = client_factory.create( + 'ns0:TaskFilterSpecByEntity') + task_filter_spec.entity.entity = vm_ref + task_filter_spec.entity.recursion = "self" + task_filter_spec.state = ["queued", "running"] + + waited = False + task_collector = None + + try: + task_collector = session._call_method( + session.vim, "CreateCollectorForTasks", + session.vim.service_content.taskManager, filter=task_filter_spec) + + while True: + page_tasks = session._call_method(session.vim, "ReadNextTasks", + task_collector, maxCount=10) + if len(page_tasks) == 0: + break + + for ti in page_tasks: + if ti.descriptionId == "ResourcePool.ImportVAppLRO": + try: + session._wait_for_task(ti.task) + waited = True + except vexc.VimException as e: + LOG.debug("Awaiting previous import on VM %s failed " + "with %s", vm_ref, e) + + return waited + finally: + if task_collector: + session._call_method(session.vim, + "DestroyCollector", task_collector) + return waited def get_vmdk_name_from_ovf(xmlstr): @@ -411,25 +498,17 @@ def fetch_image_ova(context, instance, session, vm_name, ds_name, elif vmdk_name and tar_info.name.startswith(vmdk_name): # Actual file name is .XXXXXXX extracted = tar.extractfile(tar_info) - write_handle = rw_handles.VmdkWriteHandle( - session, - session._host, - session._port, - res_pool_ref, - vm_folder_ref, - vm_import_spec, - file_size) - image_transfer(extracted, write_handle) + imported_vm_ref = _import_image(session, extracted, + vm_import_spec, vm_name, + vm_folder_ref, res_pool_ref, + file_size) + LOG.info("Downloaded OVA image file %(image_ref)s", {'image_ref': instance.image_ref}, instance=instance) - imported_vm_ref = write_handle.get_imported_vm() vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) - session._call_method(session.vim, "UnregisterVM", - imported_vm_ref) - LOG.info("The imported VM was unregistered", - instance=instance) - return vmdk.capacity_in_bytes + vm_util.mark_vm_as_template(session, instance, imported_vm_ref) + return vmdk.capacity_in_bytes, vmdk.path raise exception.ImageUnacceptable( reason=_("Extracting vmdk from OVA failed."), image_id=image_ref) diff --git a/nova/virt/vmwareapi/network_util.py b/nova/virt/vmwareapi/network_util.py index a6a84bfdb3e..581146c219c 100644 --- a/nova/virt/vmwareapi/network_util.py +++ b/nova/virt/vmwareapi/network_util.py @@ -23,7 +23,6 @@ from oslo_vmware import vim_util as vutil from nova.virt.vmwareapi import vim_util -from nova.virt.vmwareapi import vm_util LOG = logging.getLogger(__name__) @@ -59,57 +58,45 @@ def _get_network_obj(session, network_objects, network_name): """ network_obj = {} - # network_objects is actually a RetrieveResult object from vSphere API call - for obj_content in network_objects: - # the propset attribute "need not be set" by returning API - if not hasattr(obj_content, 'propSet'): - continue - prop_dict = vm_util.propset_dict(obj_content.propSet) - network_refs = prop_dict.get('network') - if network_refs: - network_refs = network_refs.ManagedObjectReference - for network in network_refs: - # Get network properties - if network._type == 'DistributedVirtualPortgroup': - props = session._call_method(vutil, - "get_object_property", - network, - "config") - # NOTE(asomya): This only works on ESXi if the port binding - # is set to ephemeral - net_name = _get_name_from_dvs_name(props.name) - if network_name in net_name: - network_obj['type'] = 'DistributedVirtualPortgroup' - network_obj['dvpg'] = props.key - dvs_props = session._call_method(vutil, - "get_object_property", - props.distributedVirtualSwitch, - "uuid") - network_obj['dvsw'] = dvs_props - return network_obj - else: - props = session._call_method(vutil, - "get_object_property", - network, - "summary.name") - if props == network_name: - network_obj['type'] = 'Network' - network_obj['name'] = network_name - return network_obj + for network in vim_util.get_array_items(network_objects): + # Get network properties + if network._type == 'DistributedVirtualPortgroup': + props = session._call_method(vutil, + "get_object_property", + network, + "config") + # NOTE(asomya): This only works on ESXi if the port binding + # is set to ephemeral + net_name = _get_name_from_dvs_name(props.name) + if network_name in net_name: + network_obj['type'] = 'DistributedVirtualPortgroup' + network_obj['dvpg'] = props.key + dvs_props = session._call_method(vutil, + "get_object_property", props.distributedVirtualSwitch, + "uuid") + network_obj['dvsw'] = dvs_props + return network_obj + else: + props = session._call_method(vutil, + "get_object_property", + network, + "summary.name") + if props == network_name: + network_obj['type'] = 'Network' + network_obj['name'] = network_name + return network_obj def get_network_with_the_name(session, network_name="vmnet0", cluster=None): """Gets reference to the network whose name is passed as the argument. """ - vm_networks = session._call_method(vim_util, - 'get_object_properties', - None, cluster, - 'ClusterComputeResource', ['network']) - - with vutil.WithRetrieval(session.vim, vm_networks) as network_objs: - network_obj = _get_network_obj(session, network_objs, network_name) - if network_obj: - return network_obj + network_objs = session._call_method(vutil, + "get_object_property", + cluster, + "network") - LOG.debug("Network %s not found on cluster!", network_name) + network_obj = _get_network_obj(session, network_objs, network_name) + if not network_obj: + LOG.debug("Network %s not found on cluster!", network_name) + return network_obj diff --git a/nova/virt/vmwareapi/rpc.py b/nova/virt/vmwareapi/rpc.py new file mode 100644 index 00000000000..368b865c578 --- /dev/null +++ b/nova/virt/vmwareapi/rpc.py @@ -0,0 +1,61 @@ +# Copyright (c) 2021 SAP SE +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +RPC server and client for communicating with other Vmareapi drivers directly. +This is the gateway which allows us gathering VMWare related information from +other hosts and perform cross vCenter operations. +""" +import oslo_messaging as messaging + +from nova.compute import rpcapi +from nova import profiler +from nova import rpc + + +_NAMESPACE = 'vmwareapi' +_VERSION = '5.0' + + +@profiler.trace_cls("rpc") +class VmwareRpcApi(object): + def __init__(self, server): + target = messaging.Target(topic=rpcapi.RPC_TOPIC, version=_VERSION, + namespace=_NAMESPACE) + self._router = rpc.ClientRouter(rpc.get_client(target)) + self._server = server + + def _call(self, fname, ctxt, version=None, **kwargs): + version = version or _VERSION + cctxt = self._router.client(ctxt).prepare(server=self._server, + version=version) + return cctxt.call(ctxt, fname, **kwargs) + + def get_vif_info(self, ctxt, vif_model=None, network_info=None): + return self._call('get_vif_info', ctxt, vif_model=vif_model, + network_info=network_info,) + + +@profiler.trace_cls("rpc") +class VmwareRpcService(object): + target = messaging.Target(version=_VERSION, namespace=_NAMESPACE) + + def __init__(self, driver): + self._driver = driver + + def get_vif_info(self, ctxt, vif_model=None, network_info=None): + return self._driver._vmops.get_vif_info(ctxt, vif_model=vif_model, + network_info=network_info, + ) diff --git a/nova/virt/vmwareapi/special_spawning.py b/nova/virt/vmwareapi/special_spawning.py new file mode 100644 index 00000000000..fff6c469f8c --- /dev/null +++ b/nova/virt/vmwareapi/special_spawning.py @@ -0,0 +1,360 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# Copyright (c) 2012 VMware, Inc. +# Copyright (c) 2011 Citrix Systems, Inc. +# Copyright 2011 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from operator import itemgetter + +from oslo_log import log as logging +import oslo_messaging as messaging +from oslo_vmware import vim_util as vutil + +from nova.compute import rpcapi +import nova.conf +from nova import exception +from nova import profiler +from nova import rpc +from nova.virt.vmwareapi import cluster_util +from nova.virt.vmwareapi import constants +from nova.virt.vmwareapi import vim_util +from nova.virt.vmwareapi.vm_util import propset_dict + +LOG = logging.getLogger(__name__) +CONF = nova.conf.CONF +FREE_HOST_STATE_DONE = 'done' +FREE_HOST_STATE_ERROR = 'error' +FREE_HOST_STATE_STARTED = 'started' + +BIGVM_RESOURCE = 'CUSTOM_BIGVM' + + +@profiler.trace_cls("rpc") +class SpecialVmSpawningInterface(object): + """RPC client foundr calling _SpecialVmSpawningServer""" + + def __init__(self): + super(SpecialVmSpawningInterface, self).__init__() + target = messaging.Target(topic=rpcapi.RPC_TOPIC, version='5.0') + self.router = rpc.ClientRouter(rpc.get_client(target)) + + def remove_host_from_hostgroup(self, ctxt, compute_host_name): + version = '5.0' + cctxt = self.router.client(ctxt).prepare( + server=compute_host_name, version=version) + return cctxt.call(ctxt, 'remove_host_from_hostgroup') + + def free_host(self, ctxt, compute_host_name): + version = '5.0' + cctxt = self.router.client(ctxt).prepare( + server=compute_host_name, version=version) + return cctxt.call(ctxt, 'free_host') + + +class _SpecialVmSpawningServer(object): + """Additional RPC interface for handling special spawning needs.""" + + target = messaging.Target(version='5.0') + + def __init__(self, driver): + self._driver = driver + self._session = driver._session + self._cluster = driver._cluster_ref + self._vmops = driver._vmops + + def _get_group(self, cluster_config=None): + """Return the hostgroup or None if not found.""" + if cluster_config is None: + cluster_config = self._session._call_method( + vutil, "get_object_property", self._cluster, "configurationEx") + if not cluster_config: + # that should never happen. we should not proceed with whatever + # called us + msg = 'Cluster {} must have an attribute "configurationEx".' + raise exception.ValidationError(msg.format(self._cluster)) + + group_ret = getattr(cluster_config, 'group', None) + if not group_ret: + return None + + hg_name = CONF.vmware.bigvm_deployment_free_host_hostgroup_name + if not hg_name: + raise exception.ValidationError('Function for special spawning ' + 'were called, but the setting ' + '"bigvm_deployment_free_host_hostgroup_name" is unconfigured.') + + for group in group_ret: + # we are only interested in one special group + if group.name == hg_name: + return group + + def _get_hosts_in_cluster(self, cluster_ref): + """Return a list of HostSystem morefs belonging to the cluster""" + result = self._session._call_method( + vim_util, 'get_inner_objects', cluster_ref, 'host', 'HostSystem') + with vutil.WithRetrieval(self._session.vim, result) as objects: + return [obj.obj for obj in objects] + + def _get_vms_on_host(self, host_ref): + """Return a list of VMs uuids with their memory size and state""" + vm_data = [] + vm_ret = self._session._call_method(vutil, + "get_object_property", + host_ref, + "vm") + # if there are no VMs on the host, we don't need to look further + if not vm_ret: + return vm_data + + vm_mors = vm_ret.ManagedObjectReference + result = self._session._call_method(vutil, + "get_properties_for_a_collection_of_objects", + "VirtualMachine", vm_mors, + ["config.instanceUuid", "runtime.powerState", + "config.hardware.memoryMB", "config.managedBy"]) + with vutil.WithRetrieval(self._session.vim, result) as objects: + for obj in objects: + vm_props = propset_dict(obj.propSet) + # sometimes, the vCenter finds a file it thinks is a VM and it + # doesn't even have a config attribute ... instead of crashing + # with a KeyError, we assume this VM is not running and totally + # doesn't matter as nova also will not be able to handle it + if 'config.instanceUuid' not in vm_props: + continue + + vm_data.append(( + vm_props['config.instanceUuid'], + vm_props['config.hardware.memoryMB'], + vm_props['runtime.powerState'], + vm_props.get('config.managedBy'), + vutil.get_moref_value(obj.obj))) + return vm_data + + def remove_host_from_hostgroup(self, context): + """Search for the host in the special spawning hostgroup and remove + that group, because emptying it seems not to work. + """ + group = self._get_group() + # no group -> nothing to do + if group is None: + return True + + # if there are no hosts in the group, we're done + if not getattr(group, 'host', None): + return True + + client_factory = self._session.vim.client.factory + + group_spec = cluster_util.create_group_spec(client_factory, group, + 'remove') + # if there are rules using this hostgroup, we also have to remove that + # one, because that rule will not update inside of DRS as it's invalid + # if we remove the group + rule_specs = [] + rules = cluster_util.fetch_cluster_rules(self._session, self._cluster) + for rule in rules: + attrs = ('antiAffineHostGroupName', 'affineHostGroupName') + if all(getattr(rule, attr, None) != group.name for attr in attrs): + continue + rule_spec = \ + cluster_util.create_rule_spec(client_factory, rule, 'remove') + rule_specs.append(rule_spec) + + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.groupSpec = [group_spec] + config_spec.rulesSpec = rule_specs + cluster_util.reconfigure_cluster(self._session, self._cluster, + config_spec) + + return True + + def free_host(self, context): + """Find a host that doesn't have a bigvm and put it into the special + hostgroup. If that's already the case, return whether there are running + VMs left on the host, i.e. the process is finished. + """ + cluster_config = self._session._call_method( + vutil, "get_object_property", self._cluster, "configurationEx") + + # get the group + group = self._get_group(cluster_config) + + failover_hosts = [] + policy = cluster_config.dasConfig.admissionControlPolicy + if policy and hasattr(policy, 'failoverHosts'): + failover_hosts = set(vutil.get_moref_value(h) + for h in policy.failoverHosts) + + if group is None or not getattr(group, 'host', None): + # find a host to free + + # retrieve all hosts of the cluster + host_objs = {vutil.get_moref_value(h): h + for h in self._get_hosts_in_cluster(self._cluster)} + vms_per_host = {h: [] for h in host_objs} + + # get all the vms in a cluster, because we need to find a host + # without big VMs. + props = ['config.hardware.memoryMB', 'runtime.host', + 'runtime.powerState', + 'summary.quickStats.hostMemoryUsage'] + cluster_vms = self._vmops._list_instances_in_cluster(props) + + for vm_uuid, vm_props in cluster_vms: + props = (vm_props.get('config.hardware.memoryMB', 0), + vm_props.get('runtime.powerState', 'poweredOff'), + vm_props.get('summary.quickStats.hostMemoryUsage', 0)) + # every host_obj is different, even though the value, which + # really matters, is the same + host_obj = vm_props.get('runtime.host') + if not host_obj: + continue + + host = vutil.get_moref_value(host_obj) + vms_per_host.setdefault(host, []). \ + append(props) + + # filter for hosts without big VMs + vms_per_host = {h: vms for h, vms in vms_per_host.items() + if all(mem < CONF.bigvm_mb + for mem, state, used_mem in vms)} + + if not vms_per_host: + LOG.warning('No suitable host found for freeing a host for ' + 'spawning (bigvm filter).') + return FREE_HOST_STATE_ERROR + + # filter hosts which are failover hosts + vms_per_host = {h: vms for h, vms in vms_per_host.items() + if h not in failover_hosts} + + if not vms_per_host: + LOG.warning('No suitable host found for freeing a host for ' + 'spawning (failover host filter).') + return FREE_HOST_STATE_ERROR + + # filter hosts which are in a wrong state + result = self._session._call_method(vim_util, + "get_properties_for_a_collection_of_objects", + "HostSystem", + [host_objs[h] for h in vms_per_host], + ['summary.runtime']) + host_states = {} + with vutil.WithRetrieval(self._session.vim, result) as objects: + for obj in objects: + host_props = propset_dict(obj.propSet) + runtime_summary = host_props['summary.runtime'] + moref_value = vutil.get_moref_value(obj.obj) + host_states[moref_value] = ( + runtime_summary.inMaintenanceMode is False and + runtime_summary.connectionState == "connected") + + vms_per_host = {h: vms for h, vms in vms_per_host.items() + if host_states[h]} + + if not vms_per_host: + LOG.warning('No suitable host found for freeing a host for ' + 'spawning (host state filter).') + return FREE_HOST_STATE_ERROR + + mem_per_host = {h: sum(used_mem for mem, state, used_mem in vms) + for h, vms in vms_per_host.items()} + + # take the one with least memory used + host, _ = sorted(mem_per_host.items(), key=itemgetter(1))[0] + host_ref = host_objs[host] + + client_factory = self._session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + + # we need to either create the group from scratch or at least add a + # host to it + operation = 'add' if group is None else 'edit' + group = cluster_util.create_host_group(client_factory, + CONF.vmware.bigvm_deployment_free_host_hostgroup_name, + [host_ref], group) + group_spec = cluster_util.create_group_spec(client_factory, + group, operation) + config_spec.groupSpec = [group_spec] + + # create the appropriate rule for VMs to leave the host + rule_name = '{}_anti-affinity'.format( + CONF.vmware.bigvm_deployment_free_host_hostgroup_name) + rule = cluster_util._get_rule(cluster_config, rule_name) + rule_spec = cluster_util._create_cluster_group_rules_spec( + client_factory, rule_name, + CONF.vmware.special_spawning_vm_group, + CONF.vmware.bigvm_deployment_free_host_hostgroup_name, + 'anti-affinity', rule) + config_spec.rulesSpec = [rule_spec] + + cluster_util.reconfigure_cluster(self._session, self._cluster, + config_spec) + else: + if len(group.host) > 1: + LOG.warning('Found more than 1 host in spawning hostgroup.') + host_ref = group.host[0] + + # check if the host is still suitable + if vutil.get_moref_value(host_ref) in failover_hosts: + LOG.warning('Host destined for spawning became a failover ' + 'host.') + return FREE_HOST_STATE_ERROR + + runtime_summary = self._session._call_method( + vutil, "get_object_property", host_ref, 'summary.runtime') + if (runtime_summary.inMaintenanceMode is True or + runtime_summary.connectionState != "connected"): + LOG.warning('Host destined for spawning was set to ' + 'maintenance or became disconnected.') + return FREE_HOST_STATE_ERROR + + # filter the VMs on the host, so we don't look at the non-movable + # DRS-created and -owned VMs + vcls_identifier = (constants.VCLS_EXTENSION_KEY, + constants.VCLS_EXTENSION_TYPE_AGENT) + vms_on_host = [ + (u, state, ref) + for u, h, state, m, ref in self._get_vms_on_host(host_ref) + if not m or (m.extensionKey, m.type) != vcls_identifier] + + # ignore partiallyAutomated VMs. They should not be big VMs as we chose + # a host appropriately, so they should be large VMs, which we tolerate + # next to a big VM as they get moved by the nanny + drs_overrides = cluster_util.fetch_cluster_drs_vm_overrides( + self._session, cluster_config=cluster_config) + wanted_drs_override_behaviors = ( + constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED,) + vms_on_host = [ + (u, state) for u, state, ref in vms_on_host + if drs_overrides.get(ref) not in wanted_drs_override_behaviors] + + # check if there are running VMs on that host + vms_on_host = [u for u, state in vms_on_host + if state != 'poweredOff'] + if vms_on_host: + # check if DRS is enabled, so freeing up can work + if cluster_config.drsConfig.defaultVmBehavior != \ + constants.DRS_BEHAVIOR_FULLY_AUTOMATED: + LOG.error('DRS set to %(actual)s, expected %(expected)s.', + {'actual': cluster_config.drsConfig.defaultVmBehavior, + 'expected': constants.DRS_BEHAVIOR_FULLY_AUTOMATED}) + return FREE_HOST_STATE_ERROR + + LOG.debug('Freeing up %(host)s for spawning in progress.', + {'host': vutil.get_moref_value(host_ref)}) + return FREE_HOST_STATE_STARTED + + LOG.info('Done freeing up %(host)s for spawning.', + {'host': vutil.get_moref_value(host_ref)}) + return FREE_HOST_STATE_DONE diff --git a/nova/virt/vmwareapi/vim_util.py b/nova/virt/vmwareapi/vim_util.py index 699660de40f..fe4962c2181 100644 --- a/nova/virt/vmwareapi/vim_util.py +++ b/nova/virt/vmwareapi/vim_util.py @@ -25,6 +25,11 @@ CONF = nova.conf.CONF +class EmptyRetrieveResult(object): + def __init__(self): + self.objects = [] + + def object_to_dict(obj, list_depth=1): """Convert Suds object into serializable format. @@ -51,30 +56,6 @@ def object_to_dict(obj, list_depth=1): return d -def get_object_properties(vim, collector, mobj, type, properties): - """Gets the properties of the Managed object specified.""" - client_factory = vim.client.factory - if mobj is None: - return None - usecoll = collector - if usecoll is None: - usecoll = vim.service_content.propertyCollector - property_filter_spec = client_factory.create('ns0:PropertyFilterSpec') - property_spec = client_factory.create('ns0:PropertySpec') - property_spec.all = (properties is None or len(properties) == 0) - property_spec.pathSet = properties - property_spec.type = type - object_spec = client_factory.create('ns0:ObjectSpec') - object_spec.obj = mobj - object_spec.skip = False - property_filter_spec.propSet = [property_spec] - property_filter_spec.objectSet = [object_spec] - options = client_factory.create('ns0:RetrieveOptions') - options.maxObjects = CONF.vmware.maximum_objects - return vim.RetrievePropertiesEx(usecoll, specSet=[property_filter_spec], - options=options) - - def get_objects(vim, type, properties_to_collect=None, all=False): """Gets the list of objects of the type specified.""" return vutil.get_objects(vim, type, CONF.vmware.maximum_objects, @@ -136,7 +117,7 @@ def get_properties_for_a_collection_of_objects(vim, type, """ client_factory = vim.client.factory if len(obj_list) == 0: - return [] + return EmptyRetrieveResult() prop_spec = get_prop_spec(client_factory, type, properties) lst_obj_specs = [] for obj in obj_list: @@ -168,3 +149,91 @@ def get_array_items(array_obj): if hasattr(array_obj, attr_name): return getattr(array_obj, attr_name) return array_obj + + +# To be moved to oslo_vmware +def _get_child_type(sxtype, name): + if not sxtype: + return None + child, _ = sxtype.get_child(name) + return child.type[0] + + +# To be moved to oslo_vmware +def serialize_object(obj, typed=False, expected_type=None): + if obj is None: + return None + + if (obj.__class__.__name__ == 'ManagedObjectReference' and + not obj.value): + # Special case for suds, which creates "empty" references + # as a value object with an empty string + return None + + d = {} + if not typed or "sxtype" not in obj.__metadata__: + sxtype = None + else: + sxtype = obj.__metadata__["sxtype"] + if expected_type and sxtype.name != expected_type: + d["class"] = sxtype.name + + for k, v in dict(obj).items(): + if hasattr(v, "__keylist__"): + expected_type = _get_child_type(sxtype, k) + ser = serialize_object(v, + typed=typed, + expected_type=expected_type) + if not typed or ser: + d[k] = ser + elif isinstance(v, list): + expected_type = _get_child_type(sxtype, k) + ser = [] + for item in v: + if hasattr(item, "__keylist__"): + ser.append( + serialize_object(item, typed=typed, + expected_type=expected_type)) + else: + ser.append(item) + if not typed or ser: + d[k] = ser + else: + if not typed or v is not None: + d[k] = v + return d + + +def deserialize_object(factory, ser, typename): + if ser is None: + return ser + + if typename == "ManagedObjectReference": + # Special case: It is a simpleContent object (the only in the wsdl) + return vutil.get_moref(ser["value"], ser["_type"]) + + typename = ser.get("class", typename) + qtypename = "ns0:{}".format(typename) + obj = factory.create(qtypename) + try: + sxtype = obj.__metadata__["sxtype"] + except AttributeError: + sxtype = factory.resolver.find(qtypename) + + for child, _ in sxtype.children(): + v = ser.get(child.name, None) + + if isinstance(v, list): + if not v or not isinstance(v[0], dict): + deser = v + else: + type_ = child.type[0] + deser = [deserialize_object(factory, i, type_) for i in v] + elif isinstance(v, dict): + type_ = child.type[0] + deser = deserialize_object(factory, v, type_) + else: + deser = v + + setattr(obj, child.name, deser) + return obj diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index a11da9cffb0..e6dc9ac22a1 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -20,16 +20,22 @@ import collections import copy +import hashlib +import operator +import socket +import ssl +from urllib.parse import urlparse from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_service import loopingcall from oslo_utils import excutils from oslo_utils import units +from oslo_utils.versionutils import convert_version_to_int from oslo_vmware import exceptions as vexc from oslo_vmware import pbm from oslo_vmware import vim_util as vutil - import nova.conf from nova import exception from nova.i18n import _ @@ -65,6 +71,8 @@ # unnecessary communication with the backend. _VM_REFS_CACHE = {} +_HOST_RESERVATIONS_DEFAULT_KEY = '__default__' + class Limits(object): @@ -107,7 +115,7 @@ def __init__(self, cpu_limits=None, hw_version=None, self.memory_limits = memory_limits or Limits() self.disk_io_limits = disk_io_limits or Limits() self.vif_limits = vif_limits or Limits() - self.hw_version = hw_version + self.hw_version = hw_version or CONF.vmware.default_hw_version self.storage_policy = storage_policy self.cores_per_socket = cores_per_socket self.hv_enabled = hv_enabled @@ -115,6 +123,114 @@ def __init__(self, cpu_limits=None, hw_version=None, self.hw_video_ram = hw_video_ram +class HistoryCollectorItems: + + def __init__(self, session, history_collector, read_page_method, + reverse_page_order=False, max_page_size=10, + page_sort_key_func=None): + self.session = session + self.history_collector = history_collector + self.read_page_method = read_page_method + self.reverse_page_order = reverse_page_order + self.max_page_size = max_page_size + self.page_sort_key_func = page_sort_key_func + + def __iter__(self): + self._latest_page_read = False + self._page_items = None + + if self.reverse_page_order: + self.session._call_method(self.session.vim, + "ResetCollector", + self.history_collector) + else: + self.session._call_method(self.session.vim, + "RewindCollector", + self.history_collector) + + return self + + def __next__(self): + if not self._page_items: + self._load_page() + + if not self._page_items: + raise StopIteration + else: + return self._page_items.pop(0) + + def _load_page(self): + if self.reverse_page_order and not self._latest_page_read: + self._load_latest_page() + + if not self._page_items: + self._page_items = self.session._call_method( + self.session.vim, self.read_page_method, + self.history_collector, maxCount=self.max_page_size) + + if self._page_items and self.page_sort_key_func: + self._page_items.sort(reverse=self.reverse_page_order, + key=self.page_sort_key_func) + + def _load_latest_page(self): + self._latest_page_read = True + latest_page = vutil.get_object_property(self.session.vim, + self.history_collector, + "latestPage") + + self._page_items = vim_util.get_array_items(latest_page) + + def destroy_collector(self): + if self.history_collector: + self.session._call_method(self.session.vim, + "DestroyCollector", + self.history_collector) + + +class TaskHistoryCollectorItems(HistoryCollectorItems): + def __init__(self, session, task_filter_spec, reverse_page_order=False, + max_page_size=10): + task_collector = session._call_method( + session.vim, + "CreateCollectorForTasks", + session.vim.service_content.taskManager, + filter=task_filter_spec) + read_page_method = ("ReadPreviousTasks" if reverse_page_order + else "ReadNextTasks") + page_sort_key_func = operator.attrgetter('queueTime') + super(TaskHistoryCollectorItems, self).__init__(session, + task_collector, + read_page_method, + reverse_page_order, + max_page_size, + page_sort_key_func) + + def __del__(self): + self.destroy_collector() + + +class EventHistoryCollectorItems(HistoryCollectorItems): + def __init__(self, session, event_filter_spec, reverse_page_order=False, + max_page_size=10): + event_collector = session._call_method( + session.vim, + "CreateCollectorForEvents", + session.vim.service_content.eventManager, + filter=event_filter_spec) + read_page_method = ("ReadPreviousEvents" if reverse_page_order + else "ReadNextEvents") + page_sort_key_func = operator.attrgetter('createdTime') + super(EventHistoryCollectorItems, self).__init__(session, + event_collector, + read_page_method, + reverse_page_order, + max_page_size, + page_sort_key_func) + + def __del__(self): + self.destroy_collector() + + def vm_refs_cache_reset(): global _VM_REFS_CACHE _VM_REFS_CACHE = {} @@ -179,13 +295,37 @@ def _get_allocation_info(client_factory, limits, allocation_type): return allocation +def append_vif_infos_to_config_spec(client_factory, config_spec, + vif_infos, vif_limits, index=0): + + if not hasattr(config_spec, 'deviceChange') or \ + not config_spec.deviceChange: + config_spec.deviceChange = [] + for offset, vif_info in enumerate(vif_infos): + vif_spec = _create_vif_spec(client_factory, vif_info, vif_limits, + offset) + config_spec.deviceChange.append(vif_spec) + + if not hasattr(config_spec, 'extraConfig') or \ + not config_spec.extraConfig: + config_spec.extraConfig = [] + port_index = index + for vif_info in vif_infos: + if vif_info['iface_id']: + config_spec.extraConfig.append( + _iface_id_option_value(client_factory, + vif_info['iface_id'], + port_index)) + port_index += 1 + + def get_vm_create_spec(client_factory, instance, data_store_name, vif_infos, extra_specs, os_type=constants.DEFAULT_OS_TYPE, - profile_spec=None, metadata=None): + profile_spec=None, metadata=None, vm_name=None): """Builds the VM Create spec.""" config_spec = client_factory.create('ns0:VirtualMachineConfigSpec') - config_spec.name = instance.uuid + config_spec.name = vm_name or instance.uuid config_spec.guestId = os_type # The name is the unique identifier for the VM. config_spec.instanceUuid = instance.uuid @@ -237,11 +377,6 @@ def get_vm_create_spec(client_factory, instance, data_store_name, config_spec.firmware = extra_specs.firmware devices = [] - for i, vif_info in enumerate(vif_infos): - vif_spec = _create_vif_spec(client_factory, vif_info, - extra_specs.vif_limits, i) - devices.append(vif_spec) - serial_port_spec = create_serial_port_spec(client_factory) if serial_port_spec: devices.append(serial_port_spec) @@ -266,14 +401,6 @@ def get_vm_create_spec(client_factory, instance, data_store_name, opt.value = 'true' extra_config.append(opt) - port_index = 0 - for vif_info in vif_infos: - if vif_info['iface_id']: - extra_config.append(_iface_id_option_value(client_factory, - vif_info['iface_id'], - port_index)) - port_index += 1 - if (CONF.vmware.console_delay_seconds and CONF.vmware.console_delay_seconds > 0): opt = client_factory.create('ns0:OptionValue') @@ -289,6 +416,9 @@ def get_vm_create_spec(client_factory, instance, data_store_name, config_spec.extraConfig = extra_config + append_vif_infos_to_config_spec(client_factory, config_spec, + vif_infos, extra_specs.vif_limits) + # Set the VM to be 'managed' by 'OpenStack' managed_by = client_factory.create('ns0:ManagedByInfo') managed_by.extensionKey = constants.EXTENSION_KEY @@ -321,9 +451,9 @@ def create_serial_port_spec(client_factory): backing.proxyURI = CONF.vmware.serial_port_proxy_uri connectable_spec = client_factory.create('ns0:VirtualDeviceConnectInfo') - connectable_spec.startConnected = True + connectable_spec.startConnected = CONF.vmware.serial_port_connected_default connectable_spec.allowGuestControl = True - connectable_spec.connected = True + connectable_spec.connected = CONF.vmware.serial_port_connected_default serial_port = client_factory.create('ns0:VirtualSerialPort') serial_port.connectable = connectable_spec @@ -362,6 +492,14 @@ def get_vm_resize_spec(client_factory, vcpus, memory_mb, extra_specs, resize_spec.cpuAllocation = _get_allocation_info( client_factory, extra_specs.cpu_limits, 'ns0:ResourceAllocationInfo') + resize_spec.memoryAllocation = _get_allocation_info( + client_factory, extra_specs.memory_limits, + 'ns0:ResourceAllocationInfo') + # NOTE(jkulik) We used to set this setting instead of `memoryAllocation`, + # so we need to deconfigure it on VMs created before the patch adding + # `memoryAllocation` support on resize. + resize_spec.memoryReservationLockedToMax = False + if metadata: resize_spec.annotation = metadata return resize_spec @@ -433,9 +571,40 @@ def _create_vif_spec(client_factory, vif_info, vif_limits=None, offset=0): # NOTE(asomya): Only works on ESXi if the portgroup binding is set to # ephemeral. Invalid configuration if set to static and the NIC does # not come up on boot if set to dynamic. + mac_address = vif_info['mac_address'] + set_net_device_backing(client_factory, net_device, vif_info) + connectable_spec = client_factory.create('ns0:VirtualDeviceConnectInfo') + connectable_spec.startConnected = True + connectable_spec.allowGuestControl = True + connectable_spec.connected = True + + net_device.connectable = connectable_spec + + # The Server assigns a Key to the device. Here we pass a -ve temporary key. + # -ve because actual keys are +ve numbers and we don't + # want a clash with the key that server might associate with the device + net_device.key = -47 - offset + net_device.addressType = "manual" + net_device.macAddress = mac_address + net_device.wakeOnLanEnabled = True + + # vnic limits are only supported from version 6.0 + if vif_limits and vif_limits.has_limits(): + if hasattr(net_device, 'resourceAllocation'): + net_device.resourceAllocation = _get_allocation_info( + client_factory, vif_limits, + 'ns0:VirtualEthernetCardResourceAllocation') + else: + msg = _('Limits only supported from vCenter 6.0 and above') + raise exception.Invalid(msg) + + network_spec.device = net_device + return network_spec + + +def set_net_device_backing(client_factory, net_device, vif_info): network_ref = vif_info['network_ref'] network_name = vif_info['network_name'] - mac_address = vif_info['mac_address'] backing = None if network_ref and network_ref['type'] == 'OpaqueNetwork': backing = client_factory.create( @@ -467,47 +636,17 @@ def _create_vif_spec(client_factory, vif_info, vif_limits=None, offset=0): backing = client_factory.create( 'ns0:VirtualEthernetCardNetworkBackingInfo') backing.deviceName = network_name - - connectable_spec = client_factory.create('ns0:VirtualDeviceConnectInfo') - connectable_spec.startConnected = True - connectable_spec.allowGuestControl = True - connectable_spec.connected = True - - net_device.connectable = connectable_spec net_device.backing = backing - # The Server assigns a Key to the device. Here we pass a -ve temporary key. - # -ve because actual keys are +ve numbers and we don't - # want a clash with the key that server might associate with the device - net_device.key = -47 - offset - net_device.addressType = "manual" - net_device.macAddress = mac_address - net_device.wakeOnLanEnabled = True - - # vnic limits are only supported from version 6.0 - if vif_limits and vif_limits.has_limits(): - if hasattr(net_device, 'resourceAllocation'): - net_device.resourceAllocation = _get_allocation_info( - client_factory, vif_limits, - 'ns0:VirtualEthernetCardResourceAllocation') - else: - msg = _('Limits only supported from vCenter 6.0 and above') - raise exception.Invalid(msg) - - network_spec.device = net_device - return network_spec - def get_network_attach_config_spec(client_factory, vif_info, index, vif_limits=None): """Builds the vif attach config spec.""" config_spec = client_factory.create('ns0:VirtualMachineConfigSpec') - vif_spec = _create_vif_spec(client_factory, vif_info, vif_limits) - config_spec.deviceChange = [vif_spec] - if vif_info['iface_id'] is not None: - config_spec.extraConfig = [_iface_id_option_value(client_factory, - vif_info['iface_id'], - index)] + + append_vif_infos_to_config_spec(client_factory, config_spec, + [vif_info], vif_limits, index) + return config_spec @@ -990,10 +1129,11 @@ def clone_vm_spec(client_factory, location, def relocate_vm_spec(client_factory, res_pool=None, datastore=None, host=None, disk_move_type="moveAllDiskBackingsAndAllowSharing", - devices=None): + devices=None, folder=None): rel_spec = client_factory.create('ns0:VirtualMachineRelocateSpec') rel_spec.datastore = datastore rel_spec.host = host + rel_spec.folder = folder rel_spec.pool = res_pool rel_spec.diskMoveType = disk_move_type if devices is not None: @@ -1003,10 +1143,10 @@ def relocate_vm_spec(client_factory, res_pool=None, datastore=None, host=None, def relocate_vm(session, vm_ref, res_pool=None, datastore=None, host=None, disk_move_type="moveAllDiskBackingsAndAllowSharing", - devices=None): + devices=None, spec=None): client_factory = session.vim.client.factory - rel_spec = relocate_vm_spec(client_factory, res_pool, datastore, host, - disk_move_type, devices) + rel_spec = spec or relocate_vm_spec(client_factory, res_pool, datastore, + host, disk_move_type, devices) relocate_task = session._call_method(session.vim, "RelocateVM_Task", vm_ref, spec=rel_spec) session._wait_for_task(relocate_task) @@ -1126,6 +1266,14 @@ def _get_vm_ref_from_vm_uuid(session, instance_uuid): return vm_refs[0] +def find_by_inventory_path(session, inv_path): + return session._call_method( + session.vim, + "FindByInventoryPath", + session.vim.service_content.searchIndex, + inventoryPath=inv_path) + + def _get_vm_ref_from_extraconfig(session, instance_uuid): """Get reference to the VM with the uuid specified.""" vms = session._call_method(vim_util, "get_objects", @@ -1191,60 +1339,283 @@ def get_vm_state(session, instance): return constants.POWER_STATES[vm_state] +def _set_host_reservations(stats, host_reservations_map, host_moref): + """Compute the number of vcpus and memory in MB reserved for the host from + the configured reservations or the default. + + For every host, both vcpus and memory can be given as static number or in + percent, with static number taking priority. + """ + host_vcpus = stats["vcpus"] + host_memory_mb = stats["memory_mb"] + stats["vcpus_reserved"] = 0 + stats["memory_mb_reserved"] = 0 + + default_key = _HOST_RESERVATIONS_DEFAULT_KEY + host_reservations = host_reservations_map.get(default_key, {}) + group_reservations = host_reservations_map.get(host_moref.value, {}) + for key in ['vcpus', 'vcpus_percent', 'memory_mb', 'memory_percent']: + if key in group_reservations: + host_reservations[key] = group_reservations[key] + + if not host_reservations: + return + + # compute the number of vcpus + if host_reservations.get('vcpus') is not None: + vcpus = max(0, host_reservations['vcpus']) + stats['vcpus_reserved'] = min(host_vcpus, vcpus) + elif host_reservations.get('vcpus_percent') is not None: + percent = max(0, min(100, host_reservations['vcpus_percent'])) + # This will round down. + stats['vcpus_reserved'] = host_vcpus * percent // 100 + + # compute the memory in MB + if host_reservations.get('memory_mb') is not None: + memory_mb = max(0, host_reservations['memory_mb']) + stats['memory_mb_reserved'] = min(host_memory_mb, memory_mb) + elif host_reservations.get('memory_percent') is not None: + percent = max(0, min(100, host_reservations['memory_percent'])) + # This will round down. + stats['memory_mb_reserved'] = host_memory_mb * percent // 100 + + +def _get_host_reservations_map(groups=None): + """return a mapping from hosts to reservations + + Reservations are read from CONF.vmware.hostgroup_reservations_json_file, + which is mapping from hostgroup to reservation. We thus look up the + hostgroups we find in the mapping and create a mapping from each host in + that hostgroup to the reservation. + """ + if groups is None: + groups = [] + + if not CONF.vmware.hostgroup_reservations_json_file: + return {} + + with open(CONF.vmware.hostgroup_reservations_json_file, 'rb') as f: + reservations = jsonutils.load(f) + + hrm = {} + + default = reservations.get(_HOST_RESERVATIONS_DEFAULT_KEY) + if default is not None: + hrm[_HOST_RESERVATIONS_DEFAULT_KEY] = default + + for group in groups: + if not hasattr(group, 'host'): + continue + + if group.name not in reservations: + continue + + for host_moref in group.host: + reservation = reservations[group.name] + hrm[host_moref.value] = reservation + return hrm + + +# Only for satisfying the tests, and to ensure it is producing the same results def get_stats_from_cluster(session, cluster): + return aggregate_stats_from_cluster( + get_stats_from_cluster_per_host(session, cluster)) + + +def aggregate_stats_from_cluster(host_stats): """Get the aggregate resource stats of a cluster.""" - vcpus = 0 + total_vcpus = 0 + total_vcpus_used = 0 + total_vcpus_reserved = 0 max_vcpus_per_host = 0 - used_mem_mb = 0 - total_mem_mb = 0 + + total_memory_mb = 0 + total_memory_mb_used = 0 + total_memory_mb_reserved = 0 max_mem_mb_per_host = 0 + + # NOTE (jakobk): For the total amount of hosts it doesn't matter + # whether the host is in MM or unreachable, because the count is + # used to calculate safety margins for resource allocations, and MM + # or otherwise unreachable hosts is precisely what that is supposed + # to guard against. + total_hypervisor_count = len(host_stats) + aggregated_cpu_info = {} + + for stats in host_stats.values(): + if not stats["available"]: + continue + # Total vcpus is the sum of all pCPUs of individual hosts + # The overcommitment ratio is factored in by the scheduler + vcpus = stats["vcpus"] + total_vcpus += vcpus + vcpus_used = stats["vcpus_used"] + total_vcpus_used += vcpus_used + vcpus_reserved = stats["vcpus_reserved"] + total_vcpus_reserved += vcpus_reserved + max_vcpus_per_host = max(max_vcpus_per_host, + vcpus - vcpus_reserved) + + memory_mb = stats["memory_mb"] + total_memory_mb += memory_mb + memory_mb_used = stats["memory_mb_used"] + total_memory_mb_used += memory_mb_used + memory_mb_reserved = stats["memory_mb_reserved"] + total_memory_mb_reserved += memory_mb_reserved + max_mem_mb_per_host = max(max_mem_mb_per_host, + memory_mb - memory_mb_reserved) + if not aggregated_cpu_info: + aggregated_cpu_info = stats["cpu_info"].copy() + else: + cpu_info = stats["cpu_info"] + for key, value in cpu_info.items(): + if aggregated_cpu_info.get(key) != value: + aggregated_cpu_info[key] = "Mismatching values" + + # Calculate VM-reservable memory as a ratio of total available + # memory, depending on either the configured tolerance for failed + # hypervisors or a single configurable ratio. + max_fail_hvs = \ + CONF.vmware.memory_reservation_cluster_hosts_max_fail + if max_fail_hvs and total_hypervisor_count: + vm_reservable_memory_ratio = \ + (1 - max_fail_hvs / total_hypervisor_count) + else: + vm_reservable_memory_ratio = \ + CONF.vmware.memory_reservation_max_ratio_fallback + + return { + "vcpus": total_vcpus, + "vcpus_used": total_vcpus_used, + "vcpus_reserved": total_vcpus_reserved, + "max_vcpus_per_host": max_vcpus_per_host, + "memory_mb": total_memory_mb, + "memory_mb_used": total_memory_mb_used, + "memory_mb_reserved": total_memory_mb_reserved, + "max_mem_mb_per_host": max_mem_mb_per_host, + "vm_reservable_memory_ratio": vm_reservable_memory_ratio, + "cpu_info": aggregated_cpu_info + } + + +def _host_props_to_cpu_info(host_props): + processor_type = None + cpu_vendor = None + hardware_cpu_pkg = host_props.get("hardware.cpuPkg") + if hardware_cpu_pkg and hardware_cpu_pkg.HostCpuPackage: + t = hardware_cpu_pkg.HostCpuPackage[0] + processor_type = t.description + cpu_vendor = t.vendor.title() + + features = [] + if "config.featureCapability" in host_props: + feature_capability = host_props["config.featureCapability"] + for feature in feature_capability.HostFeatureCapability: + if not feature.featureName.startswith("cpuid."): + continue + if feature.value != "1": + continue + + name = feature.featureName + features.append(name.split(".", 1)[1].lower()) + cpu_info = { + "model": processor_type, + "vendor": cpu_vendor, + "features": sorted(features) + } + hardware_cpu_info = host_props.get("hardware.cpuInfo") + if hardware_cpu_info: + cpu_info["topology"] = { + "cores": hardware_cpu_info.numCpuCores, + "sockets": hardware_cpu_info.numCpuPackages, + "threads": hardware_cpu_info.numCpuThreads + } + return cpu_info + + +def _set_hypervisor_type_and_version(stats, host_props): + product = host_props.get("summary.config.product") + if not product: + return + + stats["hypervisor_type"] = product.name + stats["hypervisor_version"] = convert_version_to_int(product.version) + + +def _process_host_stats(obj, host_reservations_map): + host_props = propset_dict(obj.propSet) + runtime_summary = host_props["summary.runtime"] + hardware_summary = host_props.get("summary.hardware") + stats_summary = host_props.get("summary.quickStats") + # Total vcpus is the sum of all pCPUs of individual hosts + # The overcommitment ratio is factored in by the scheduler + threads = getattr(hardware_summary, "numCpuThreads", 0) + mem_mb = getattr(hardware_summary, "memorySize", 0) // units.Mi + + stats = { + "available": (not runtime_summary.inMaintenanceMode and + runtime_summary.connectionState == "connected"), + "vcpus": threads, + "vcpus_used": 0, + "memory_mb": mem_mb, + "memory_mb_used": getattr(stats_summary, "overallMemoryUsage", 0), + "cpu_info": _host_props_to_cpu_info(host_props), + } + + _set_hypervisor_type_and_version(stats, host_props) + _set_host_reservations(stats, host_reservations_map, obj.obj) + return host_props["name"], stats + + +def get_stats_from_cluster_per_host(session, cluster): + """Get the resource stats per host of a cluster.""" + host_mors, host_reservations_map = \ + get_hosts_and_reservations_for_cluster(session, cluster) + + if not host_mors: + return {} + + result = session._call_method(vim_util, + "get_properties_for_a_collection_of_objects", + "HostSystem", host_mors, + ["name", "summary.hardware", "summary.runtime", + "summary.quickStats", "summary.config.product", + "hardware.cpuPkg", "hardware.cpuInfo", + "config.featureCapability", + ]) + with vutil.WithRetrieval(session.vim, result) as objects: + return dict(_process_host_stats(obj, host_reservations_map) + for obj in objects if hasattr(obj, "propSet")) + + +def get_hosts_and_reservations_for_cluster(session, cluster): # Get the Host and Resource Pool Managed Object Refs - props = ["host", "resourcePool", - "configuration.dasConfig.admissionControlPolicy"] + admission_policy_key = "configuration.dasConfig.admissionControlPolicy" + props = ["host", "resourcePool", admission_policy_key] + if CONF.vmware.hostgroup_reservations_json_file: + props.append("configurationEx") prop_dict = session._call_method(vutil, "get_object_properties_dict", cluster, props) - if prop_dict: - failover_hosts = [] - key = 'configuration.dasConfig.admissionControlPolicy' - policy = prop_dict.get(key) - if policy and hasattr(policy, 'failoverHosts'): - failover_hosts = set(h.value for h in policy.failoverHosts) - - host_ret = prop_dict.get('host') - if host_ret: - host_mors = [m for m in host_ret.ManagedObjectReference + if not prop_dict: + return None, None + + failover_hosts = [] + policy = prop_dict.get(admission_policy_key) + if policy and hasattr(policy, 'failoverHosts'): + failover_hosts = set(h.value for h in policy.failoverHosts) + + group_ret = getattr(prop_dict.get('configurationEx'), 'group', None) + + host_ret = prop_dict.get('host') + + if not host_ret: + return None, None + + host_mors = [m for m in host_ret.ManagedObjectReference if m.value not in failover_hosts] - result = session._call_method(vim_util, - "get_properties_for_a_collection_of_objects", - "HostSystem", host_mors, - ["summary.hardware", "summary.runtime", - "summary.quickStats"]) - with vutil.WithRetrieval(session.vim, result) as objects: - for obj in objects: - host_props = propset_dict(obj.propSet) - runtime_summary = host_props['summary.runtime'] - if (runtime_summary.inMaintenanceMode is not False or - runtime_summary.connectionState != "connected"): - continue - hardware_summary = host_props['summary.hardware'] - stats_summary = host_props['summary.quickStats'] - # Total vcpus is the sum of all pCPUs of individual hosts - # The overcommitment ratio is factored in by the scheduler - threads = hardware_summary.numCpuThreads - vcpus += threads - max_vcpus_per_host = max(max_vcpus_per_host, threads) - used_mem_mb += stats_summary.overallMemoryUsage - mem_mb = hardware_summary.memorySize // units.Mi - total_mem_mb += mem_mb - max_mem_mb_per_host = max(max_mem_mb_per_host, mem_mb) - stats = {'cpu': {'vcpus': vcpus, - 'max_vcpus_per_host': max_vcpus_per_host}, - 'mem': {'total': total_mem_mb, - 'free': total_mem_mb - used_mem_mb, - 'max_mem_mb_per_host': max_mem_mb_per_host}} - return stats + return host_mors, _get_host_reservations_map(group_ret) def get_host_ref(session, cluster=None): @@ -1291,7 +1662,7 @@ def get_vmdk_backed_disk_device(hardware_devices, uuid): if (device.__class__.__name__ == "VirtualDisk" and device.backing.__class__.__name__ == "VirtualDiskFlatVer2BackingInfo" and - device.backing.uuid == uuid): + getattr(device.backing, 'uuid', None) == uuid): return device @@ -1332,6 +1703,27 @@ def get_cluster_ref_by_name(session, cluster_name): return cluster.obj +def get_datastore_ref_by_name(session, datastore_name): + """Return the ManagedObjectReference to the datastore with the given name + + Returns None if no datastore with that name can be found. + """ + try: + results = session._call_method(vim_util, "get_objects", + "Datastore", ["name"]) + with vutil.WithRetrieval(session.vim, results) as objects: + for datastore in objects: + if not getattr(datastore, 'propSet', None): + continue + + if datastore.propSet[0].val != datastore_name: + continue + + return datastore.obj + except Exception as exc: + LOG.warning("Failed to get datastore references %s", exc) + + def get_vmdk_adapter_type(adapter_type): """Return the adapter type to be used in vmdk descriptor. @@ -1388,10 +1780,29 @@ def destroy_vm(session, instance, vm_ref=None): vm_ref) session._wait_for_task(destroy_task) LOG.info("Destroyed the VM", instance=instance) + except vexc.VimFaultException as e: + with excutils.save_and_reraise_exception() as ctx: + LOG.exception('Destroy VM failed', instance=instance) + # we need the `InvalidArgument` fault to bubble out of this + # function so it can be acted upon on higher levels + if 'InvalidArgument' not in e.fault_list: + ctx.reraise = False except Exception: LOG.exception('Destroy VM failed', instance=instance) +def mark_vm_as_template(session, instance, vm_ref=None): + """Mark a VM instance as template. Assumes VM is powered off.""" + try: + if not vm_ref: + vm_ref = get_vm_ref(session, instance) + LOG.debug("Marking the VM as template", instance=instance) + session._call_method(session.vim, "MarkAsTemplate", vm_ref) + LOG.info("Marked the VM as template", instance=instance) + except Exception: + LOG.exception('Mark VM as template failed', instance=instance) + + def create_virtual_disk(session, dc_ref, adapter_type, disk_type, virtual_disk_path, size_in_kb): # Create a Virtual Disk of the size of the flat vmdk file. This is @@ -1685,3 +2096,60 @@ def detach_fcd(session, vm_ref, fcd_id): task = session._call_method( session.vim, "DetachDisk_Task", vm_ref, diskId=disk_id) session._wait_for_task(task) + + +def create_service_locator_name_password(client_factory, username, password): + """Creates a ServiceLocatorNamePassword object, which in turn is + derived of ServiceLocatorCredential + """ + o = client_factory.create('ns0:ServiceLocatorNamePassword') + o.username = username + o.password = password + return o + + +def get_sha1_ssl_thumbprint(url, timeout=1): + """Returns the sha-1 thumbprint of the ssl cert of the vcenter or None + """ + try: + parsed_url = urlparse(url) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + wrappedSocket = ssl.wrap_socket(sock) + thumb = None + wrappedSocket.connect((parsed_url.hostname, parsed_url.port or 443)) + + der_cert_bin = wrappedSocket.getpeercert(True) + + thumb = hashlib.sha1(der_cert_bin).hexdigest().upper() + t = iter(thumb) + return ':'.join(a + b for a, b in zip(t, t)) + except (OSError, ValueError, socket.timeout): + return + + +def create_service_locator(client_factory, url, vcenter_instance_uuid, + credential, ssl_thumbprint=None): + """Creates a ServiceLocator WSDL-object + + :param client_factory: - factory for creating the object + :param url: - url to the vcenter api + :param vcenter_instance_uuid: - uuid of the vcenter instance + :param credential: - An instance of ServiceLocatorCredential + (See: create_service_locator_name_password) + :param ssl_thumbprint: - The sha1 thumbprint of the cert of the instance + (See: get_sha1_ssl_thumbprint) + :returns: A ServiceLocator WSDL-object + """ + + # While it is nominally optional, operations seems to fail without it + # We will do a best effort + if not ssl_thumbprint: + ssl_thumbprint = get_sha1_ssl_thumbprint(url) + + sl = client_factory.create('ns0:ServiceLocator') + sl.url = url + sl.instanceUuid = vcenter_instance_uuid + sl.credential = credential + sl.sslThumbprint = ssl_thumbprint + return sl diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 09b920fde73..57d9854335a 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -20,11 +20,12 @@ """ import collections +import copy +import decorator import os import re import time -import decorator from oslo_concurrency import lockutils from oslo_log import log as logging from oslo_serialization import jsonutils @@ -40,6 +41,7 @@ from nova.compute import api as compute from nova.compute import power_state from nova.compute import task_states +from nova.compute import utils as compute_utils import nova.conf from nova.console import type as ctype from nova import context as nova_context @@ -53,11 +55,14 @@ from nova.virt import configdrive from nova.virt import driver from nova.virt import hardware +from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import imagecache from nova.virt.vmwareapi import images +from nova.virt.vmwareapi.rpc import VmwareRpcApi +from nova.virt.vmwareapi import special_spawning from nova.virt.vmwareapi import vif as vmwarevif from nova.virt.vmwareapi import vim_util from nova.virt.vmwareapi import vm_util @@ -67,7 +72,7 @@ LOG = logging.getLogger(__name__) -RESIZE_TOTAL_STEPS = 6 +RESIZE_TOTAL_STEPS = 7 class VirtualMachineInstanceConfigInfo(object): @@ -130,13 +135,14 @@ def retry_if_task_in_progress(f, *args, **kwargs): class VMwareVMOps(object): """Management class for VM-related tasks.""" - def __init__(self, session, virtapi, volumeops, cluster=None, + def __init__(self, session, virtapi, volumeops, vc_state, cluster=None, datastore_regex=None): """Initializer.""" self.compute_api = compute.API() self._session = session self._virtapi = virtapi self._volumeops = volumeops + self._vc_state = vc_state self._cluster = cluster self._root_resource_pool = vm_util.get_res_pool_ref(self._session, self._cluster) @@ -148,6 +154,9 @@ def __init__(self, session, virtapi, volumeops, cluster=None, self._base_folder) self._network_api = neutron.API() + # set when our driver's init_host() is called + self._compute_host = None + def _get_base_folder(self): # Enable more than one compute node to run on the same host if CONF.vmware.cache_prefix: @@ -202,8 +211,17 @@ def _delete_datastore_file(self, datastore_path, dc_ref): def _extend_if_required(self, dc_info, image_info, instance, root_vmdk_path): - """Increase the size of the root vmdk if necessary.""" - if instance.flavor.root_gb * units.Gi > image_info.file_size: + root_vmdk = ds_obj.DatastorePath.parse( + root_vmdk_path.replace(".vmdk", "-flat.vmdk")) + + datastore = ds_util.get_datastore(self._session, dc_info.ref, + re.compile(r"^{}$".format( + root_vmdk.datastore))) + ds_browser = self._get_ds_browser(datastore.ref) + actual_file_size = ds_util.file_size(self._session, ds_browser, + root_vmdk.parent, + root_vmdk.basename) + if instance.flavor.root_gb * units.Gi > actual_file_size: size_in_kb = instance.flavor.root_gb * units.Mi self._extend_virtual_disk(instance, size_in_kb, root_vmdk_path, dc_info.ref) @@ -269,9 +287,19 @@ def _get_folder_name(self, name, id_): # We cannot truncate the 'id' as this is unique across OpenStack. return '%s (%s)' % (name[:40], id_[:36]) - def build_virtual_machine(self, instance, image_info, - dc_info, datastore, network_info, extra_specs, - metadata): + def _get_project_folder_path(self, project_id, type_): + folder_name = self._get_folder_name('Project', project_id) + folder_path = 'OpenStack/%s/%s' % (folder_name, type_) + + return folder_path + + def _get_project_folder(self, dc_info, project_id=None, type_=None): + folder_path = self._get_project_folder_path(project_id, type_) + return self._create_folders(dc_info.vmFolder, folder_path) + + def _get_vm_config_spec(self, instance, image_info, + datastore, network_info, extra_specs, + metadata, vm_name=None): vif_infos = vmwarevif.get_vif_info(self._session, self._cluster, image_info.vif_model, @@ -292,16 +320,26 @@ def build_virtual_machine(self, instance, image_info, extra_specs, image_info.os_type, profile_spec=profile_spec, - metadata=metadata) + metadata=metadata, + vm_name=vm_name) - folder_name = self._get_folder_name('Project', - instance.project_id) - folder_path = 'OpenStack/%s/Instances' % folder_name - folder = self._create_folders(dc_info.vmFolder, folder_path) + return config_spec + + def build_virtual_machine(self, instance, context, image_info, + dc_info, datastore, network_info, extra_specs, + metadata, folder_type='Instances', vm_name=None): + config_spec = self._get_vm_config_spec(instance, image_info, datastore, + network_info, extra_specs, + metadata, vm_name=vm_name) + + folder = self._get_project_folder(dc_info, + project_id=instance.project_id, + type_=folder_type) # Create the VM vm_ref = vm_util.create_vm(self._session, instance, folder, config_spec, self._root_resource_pool) + return vm_ref def _get_extra_specs(self, flavor, image_meta=None): @@ -316,6 +354,18 @@ def _get_extra_specs(self, flavor, image_meta=None): if value: setattr(getattr(extra_specs, resource + '_limits'), key, type(value)) + if CONF.vmware.reserve_all_memory \ + or utils.is_big_vm(int(flavor.memory_mb), flavor): + extra_specs.memory_limits.reservation = int(flavor.memory_mb) + else: + try: + memory_reserved_mb = int(flavor.extra_specs[ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY]) + if memory_reserved_mb > 0: + extra_specs.memory_limits.reservation = min( + int(flavor.memory_mb), memory_reserved_mb) + except (ValueError, KeyError): + pass extra_specs.cpu_limits.validate() extra_specs.memory_limits.validate() extra_specs.disk_io_limits.validate() @@ -325,7 +375,8 @@ def _get_extra_specs(self, flavor, image_meta=None): extra_specs.firmware = 'efi' elif hw_firmware_type == fields.FirmwareType.BIOS: extra_specs.firmware = 'bios' - hw_version = flavor.extra_specs.get('vmware:hw_version') + hw_version = flavor.extra_specs.get('vmware:hw_version', + CONF.vmware.default_hw_version) hv_enabled = flavor.extra_specs.get('vmware:hv_enabled') extra_specs.hv_enabled = hv_enabled extra_specs.hw_version = hw_version @@ -339,15 +390,20 @@ def _get_extra_specs(self, flavor, image_meta=None): if video_ram and max_vram: extra_specs.hw_video_ram = video_ram * units.Mi / units.Ki - if CONF.vmware.pbm_enabled: - storage_policy = flavor.extra_specs.get('vmware:storage_policy', - CONF.vmware.pbm_default_policy) + storage_policy = self._get_storage_policy(flavor) + if storage_policy: extra_specs.storage_policy = storage_policy topology = hardware.get_best_cpu_topology(flavor, image_meta, allow_threads=False) extra_specs.cores_per_socket = topology.cores return extra_specs + def _get_storage_policy(self, flavor): + if CONF.vmware.pbm_enabled: + return flavor.extra_specs.get('vmware:storage_policy', + CONF.vmware.pbm_default_policy) + return None + def _get_esx_host_and_cookies(self, datastore, dc_path, file_path): hosts = datastore.get_connected_hosts(self._session) host = ds_obj.Datastore.choose_host(hosts) @@ -429,52 +485,60 @@ def _fetch_image_as_file(self, context, vi, image_ds_loc): image_ds_loc.rel_path, cookies=cookies) + def _get_image_template_vm_name(self, image_id, datastore_name): + templ_vm_name = '%s (%s)' % (image_id, datastore_name) + return templ_vm_name + def _fetch_image_as_vapp(self, context, vi, image_ds_loc): """Download stream optimized image to host as a vApp.""" - # The directory of the imported disk is the unique name - # of the VM use to import it with. - vm_name = image_ds_loc.parent.basename + vm_name = self._get_image_template_vm_name(vi.ii.image_id, + vi.datastore.name) LOG.debug("Downloading stream optimized image %(image_id)s to " - "%(file_path)s on the data store " + "%(vm_name)s on the data store " "%(datastore_name)s as vApp", {'image_id': vi.ii.image_id, - 'file_path': image_ds_loc, + 'vm_name': vm_name, 'datastore_name': vi.datastore.name}, instance=vi.instance) - image_size = images.fetch_image_stream_optimized( + image_size, src_folder_ds_path = images.fetch_image_stream_optimized( context, vi.instance, self._session, vm_name, vi.datastore.name, - vi.dc_info.vmFolder, + self._get_project_folder(vi.dc_info, + project_id=vi.instance.project_id, type_='Images'), self._root_resource_pool) # The size of the image is different from the size of the virtual disk. # We want to use the latter. On vSAN this is the only way to get this # size because there is no VMDK descriptor. vi.ii.file_size = image_size + self._cache_vm_image(vi, src_folder_ds_path) def _fetch_image_as_ova(self, context, vi, image_ds_loc): """Download root disk of an OVA image as streamOptimized.""" - # The directory of the imported disk is the unique name - # of the VM use to import it with. - vm_name = image_ds_loc.parent.basename - - image_size = images.fetch_image_ova(context, - vi.instance, - self._session, - vm_name, - vi.datastore.name, - vi.dc_info.vmFolder, - self._root_resource_pool) + vm_name = self._get_image_template_vm_name( + vi.ii.image_id, vi.datastore.name) + + image_size, src_folder_ds_path = images.fetch_image_ova( + context, + vi.instance, + self._session, + vm_name, + vi.datastore.name, + self._get_project_folder(vi.dc_info, + project_id=vi.instance.project_id, type_='Images'), + self._root_resource_pool) + # The size of the image is different from the size of the virtual disk. # We want to use the latter. On vSAN this is the only way to get this # size because there is no VMDK descriptor. vi.ii.file_size = image_size + self._cache_vm_image(vi, src_folder_ds_path) def _prepare_sparse_image(self, vi): tmp_dir_loc = vi.datastore.build_path( @@ -553,6 +617,19 @@ def _cache_flat_image(self, vi, tmp_image_ds_loc): tmp_image_ds_loc.parent, vi.cache_image_folder) + def _cache_vm_image(self, vi, tmp_image_ds_loc): + dst_path = vi.cache_image_folder.join("%s.vmdk" % vi.ii.image_id) + try: + ds_util.mkdir(self._session, + vi.cache_image_folder, vi.dc_info.ref) + except vexc.FileAlreadyExistsException: + pass + try: + ds_util.disk_copy(self._session, vi.dc_info.ref, + tmp_image_ds_loc, dst_path) + except vexc.FileAlreadyExistsException: + pass + def _cache_stream_optimized_image(self, vi, tmp_image_ds_loc): dst_path = vi.cache_image_folder.join("%s.vmdk" % vi.ii.image_id) ds_util.mkdir(self._session, vi.cache_image_folder, vi.dc_info.ref) @@ -567,11 +644,13 @@ def _cache_iso_image(self, vi, tmp_image_ds_loc): tmp_image_ds_loc.parent, vi.cache_image_folder) - def _get_vm_config_info(self, instance, image_info, - extra_specs): + def _get_vm_config_info(self, instance, image_info, extra_specs): """Captures all relevant information from the spawn parameters.""" + boot_from_volume = compute_utils.is_volume_backed_instance( + instance._context, instance) if (instance.flavor.root_gb != 0 and + not boot_from_volume and image_info.file_size > instance.flavor.root_gb * units.Gi): reason = _("Image disk size greater than requested disk size") raise exception.InstanceUnacceptable(instance_id=instance.uuid, @@ -604,7 +683,10 @@ def _get_image_callbacks(self, vi): else: image_fetch = self._fetch_image_as_file - if vi.ii.is_iso: + if vi.ii.is_ova or disk_type == constants.DISK_TYPE_STREAM_OPTIMIZED: + image_prepare = lambda vi: (None, None) + image_cache = lambda vi, image_loc: None + elif vi.ii.is_iso: image_prepare = self._prepare_iso_image image_cache = self._cache_iso_image elif disk_type == constants.DISK_TYPE_SPARSE: @@ -621,6 +703,49 @@ def _get_image_callbacks(self, vi): raise exception.InvalidDiskInfo(reason=reason) return image_prepare, image_fetch, image_cache + def _fetch_image_from_other_datastores(self, vi): + dc_all_datastores = ds_util.get_available_datastores( + self._session, dc_ref=vi.dc_info.ref) + dc_other_datastores = [ds for ds in dc_all_datastores if + dict(ds.ref) != dict(vi.datastore.ref)] + + client_factory = self._session.vim.client.factory + tmp_vi = copy.copy(vi) + for ds in dc_other_datastores: + tmp_vi.datastore = ds + other_templ_vm_ref = self._find_image_template_vm(tmp_vi) + if other_templ_vm_ref: + rel_spec = vm_util.relocate_vm_spec( + client_factory, + res_pool=self._root_resource_pool, + disk_move_type="moveAllDiskBackingsAndDisallowSharing", + datastore=vi.datastore.ref) + clone_spec = vm_util.clone_vm_spec(client_factory, + rel_spec, template=True) + templ_vm_clone_task = self._session._call_method( + self._session.vim, + "CloneVM_Task", + other_templ_vm_ref, + folder=self._get_project_folder(vi.dc_info, + project_id=vi.instance.project_id, type_='Images'), + name=self._get_image_template_vm_name( + vi.ii.image_id, vi.datastore.name), + spec=clone_spec) + try: + task_info = \ + self._session._wait_for_task(templ_vm_clone_task) + except vexc.VimFaultException as e: + if 'VirtualHardwareVersionNotSupported' in e.fault_list: + LOG.debug('Could not clone image-template from ' + 'incompatible hardware platform') + else: + LOG.warning('Could not clone image-template from ' + 'other datastore.') + continue + + templ_vm_ref = task_info.result + return templ_vm_ref + def _fetch_image_if_missing(self, context, vi): image_prepare, image_fetch, image_cache = self._get_image_callbacks(vi) LOG.debug("Processing image %s", vi.ii.image_id, instance=vi.instance) @@ -629,9 +754,21 @@ def _fetch_image_if_missing(self, context, vi): lock_file_prefix='nova-vmware-fetch_image'): self.check_cache_folder(vi.datastore.name, vi.datastore.ref) ds_browser = self._get_ds_browser(vi.datastore.ref) - if not ds_util.file_exists(self._session, ds_browser, - vi.cache_image_folder, - vi.cache_image_path.basename): + image_available = ds_util.file_exists( + self._session, ds_browser, + vi.cache_image_folder, + vi.cache_image_path.basename) + + if not image_available: + templ_vm_ref = self._find_image_template_vm(vi) + image_available = (templ_vm_ref is not None) + + if (not image_available and + CONF.vmware.fetch_image_from_other_datastores): + templ_vm_ref = self._fetch_image_from_other_datastores(vi) + image_available = (templ_vm_ref is not None) + + if not image_available: LOG.debug("Preparing fetch location", instance=vi.instance) tmp_dir_loc, tmp_image_ds_loc = image_prepare(vi) LOG.debug("Fetch image to %s", tmp_image_ds_loc, @@ -641,7 +778,9 @@ def _fetch_image_if_missing(self, context, vi): image_cache(vi, tmp_image_ds_loc) LOG.debug("Cleaning up location %s", str(tmp_dir_loc), instance=vi.instance) - self._delete_datastore_file(str(tmp_dir_loc), vi.dc_info.ref) + if tmp_dir_loc: + self._delete_datastore_file(str(tmp_dir_loc), + vi.dc_info.ref) # The size of the sparse image is different from the size of the # virtual disk. We want to use the latter. @@ -737,6 +876,62 @@ def prepare_for_spawn(self, instance): raise exception.InstanceUnacceptable(instance_id=instance.uuid, reason=reason) + def update_cluster_placement(self, context, instance, remove=False): + self.sync_instance_server_group(context, instance) + self.update_admin_vm_group_membership(instance, remove=remove) + + def sync_instance_server_group(self, context, instance): + try: + instance_group_object = objects.instance_group.InstanceGroup + server_group = instance_group_object.get_by_instance_uuid( + context, instance.uuid) + self.sync_server_group(context, server_group.uuid) + except nova.exception.InstanceGroupNotFound: + pass + + @staticmethod + def _get_admin_group_name_for_instance(instance): + vm_group_name = CONF.vmware.special_spawning_vm_group + if not vm_group_name: + return None + + needs_empty_host = utils.vm_needs_special_spawning( + int(instance.memory_mb), instance.flavor) + if needs_empty_host: + return None + + return vm_group_name + + def update_admin_vm_group_membership(self, instance, remove=False): + vm_group_name = self._get_admin_group_name_for_instance(instance) + if not vm_group_name: + return + vm_ref = vm_util.get_vm_ref(self._session, instance) + cluster_util.update_vm_group_membership(self._session, self._cluster, + vm_group_name, vm_ref, + remove=remove) + + def _build_template_vm_inventory_path(self, vi): + vm_folder_name = self._session._call_method(vutil, + "get_object_property", + vi.dc_info.vmFolder, + "name") + images_folder_path = self._get_project_folder_path( + vi.instance.project_id, 'Images') + templ_vm_name = self._get_image_template_vm_name(vi.ii.image_id, + vi.datastore.name) + templ_vm_inventory_path = '%s/%s/%s/%s' % ( + vi.dc_info.name, vm_folder_name, + images_folder_path, templ_vm_name) + return templ_vm_inventory_path + + def _find_image_template_vm(self, vi): + templ_vm_inventory_path = self._build_template_vm_inventory_path(vi) + templ_vm_ref = vm_util.find_by_inventory_path(self._session, + templ_vm_inventory_path) + + return templ_vm_ref + def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info, block_device_info=None): @@ -749,10 +944,13 @@ def spawn(self, context, instance, image_meta, injected_files, vi = self._get_vm_config_info(instance, image_info, extra_specs) + boot_from_volume = compute_utils.is_volume_backed_instance( + instance._context, instance) metadata = self._get_instance_metadata(context, instance) # Creates the virtual machine. The virtual machine reference returned # is unique within Virtual Center. vm_ref = self.build_virtual_machine(instance, + context, image_info, vi.dc_info, vi.datastore, @@ -764,6 +962,9 @@ def spawn(self, context, instance, image_meta, injected_files, # instance uuid. vm_util.vm_ref_cache_update(instance.uuid, vm_ref) + # Update all DRS related rules + self.update_cluster_placement(context, instance) + # Update the Neutron VNIC index self._update_vnic_index(context, instance, network_info) @@ -782,7 +983,7 @@ def spawn(self, context, instance, image_meta, injected_files, block_device_mapping = driver.block_device_info_get_mapping( block_device_info) - if instance.image_ref: + if instance.image_ref and not boot_from_volume: self._imagecache.enlist_image( image_info.image_id, vi.datastore, vi.dc_info.ref) self._fetch_image_if_missing(context, vi) @@ -804,7 +1005,8 @@ def spawn(self, context, instance, image_meta, injected_files, # block_device_mapping (i.e. disk_bus) is valid self._is_bdm_valid(block_device_mapping) - for disk in block_device_mapping: + for disk in sorted(block_device_mapping, + key=lambda x: x.get('boot_index') != 0): connection_info = disk['connection_info'] adapter_type = disk.get('disk_bus') or vi.ii.adapter_type @@ -835,8 +1037,54 @@ def spawn(self, context, instance, image_meta, injected_files, # 'uuid' of the instance vm_util.rename_vm(self._session, vm_ref, instance) + # Make sure we don't automatically move around "big" VMs + if utils.is_big_vm(int(instance.memory_mb), instance.flavor): + behavior = constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED + LOG.debug("Adding DRS override '%s' for big VM.", behavior, + instance=instance) + cluster_util.update_cluster_drs_vm_override(self._session, + self._cluster, + vm_ref, + operation='add', + behavior=behavior) + vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) + self._clean_up_after_special_spawning(context, instance.memory_mb, + instance.flavor) + + def _clean_up_after_special_spawning(self, context, instance_memory_mb, + instance_flavor): + if utils.vm_needs_special_spawning(int(instance_memory_mb), + instance_flavor): + # we're using a child resource provider, so we don't have to change + # the drivers' report-code to keep the CUSTOM_BIGVM resource, but + # instead can independently add it or remove it on a cluster + placement_client = self._virtapi._compute.reportclient + cn = self._virtapi._compute._get_compute_info(context, CONF.host) + parent_rp_uuid = cn.uuid + parent_tree = placement_client.get_provider_tree_and_ensure_root( + context, parent_rp_uuid) + rp_name = '{}-{}'.format(CONF.bigvm_deployment_rp_name_prefix, + cn.host) + try: + rp = parent_tree.data(rp_name) + except ValueError: + LOG.warning('Could not find resource-provider %(rp)s for ' + 'reserving resources after (re)starting a big VM.', + {'rp': rp_name}) + else: + # we need to update the inventory of the bigvm provider in our + # cache, because the generation might be too old after the + # allocations of our currently spawning big vm + placement_client._refresh_and_get_inventory(context, rp.uuid) + # reserve the bigvm resource. this prohibits any further + # deployment needing a free host on that compute-node. + inv_data = rp.inventory + inv_data[special_spawning.BIGVM_RESOURCE]['reserved'] = 1 + placement_client.set_inventory_for_provider(context, rp.uuid, + inv_data) + def _is_bdm_valid(self, block_device_mapping): """Checks if the block device mapping is valid.""" valid_bus = (constants.DEFAULT_ADAPTER_TYPE, @@ -911,12 +1159,12 @@ def _attach_cdrom_to_vm(self, vm_ref, instance, LOG.debug("Reconfigured VM instance to attach cdrom %s", file_path, instance=instance) - def _create_vm_snapshot(self, instance, vm_ref): + def _create_vm_snapshot(self, instance, vm_ref, image_id=None): LOG.debug("Creating Snapshot of the VM instance", instance=instance) snapshot_task = self._session._call_method( self._session.vim, "CreateSnapshot_Task", vm_ref, - name="%s-snapshot" % instance.uuid, + name="%s-snapshot" % (image_id or instance.uuid), description="Taking Snapshot of the VM", memory=False, quiesce=True) @@ -970,6 +1218,83 @@ def _create_linked_clone_from_snapshot(self, instance, "info") return task_info.result + def _create_vm_clone(self, instance, vm_ref, snapshot_ref, dc_info, + disk_move_type=None, image_id=None, disks=None): + """Clone VM to be deployed to same ds as source VM + """ + image_id = image_id or uuidutils.generate_uuid() + + if disks: + datastore = disks[0].device.backing.datastore + else: + if disk_move_type == "createNewChildDiskBacking": + datastore = None + else: + datastore = ds_util.get_datastore(self._session, self._cluster, + self._datastore_regex) + + vm_name = "%s_%s" % (constants.SNAPSHOT_VM_PREFIX, + image_id) + client_factory = self._session.vim.client.factory + rel_spec = vm_util.relocate_vm_spec( + client_factory, + datastore=datastore, + host=None, + disk_move_type=disk_move_type) + config_spec = client_factory.create('ns0:VirtualMachineConfigSpec') + config_spec.name = vm_name + config_spec.annotation = "Created from %s" % (instance.uuid) + config_spec.numCPUs = 1 + config_spec.numCoresPerSocket = 1 + config_spec.memoryMB = 16 + config_spec.uuid = image_id # Not instanceUuid, + # as we need to import the same image in different datastores + + if disks: + disk_devices = [vmdk_info.device.key for vmdk_info in disks] + hardware_devices = self._session._call_method( + vutil, + "get_object_property", + vm_ref, + "config.hardware.device") + if hardware_devices.__class__.__name__ == "ArrayOfVirtualDevice": + hardware_devices = hardware_devices.VirtualDevice + + device_change = [] + for device in hardware_devices: + if getattr(device, 'macAddress', None) or \ + device.__class__.__name__ == "VirtualDisk" and \ + device.key not in disk_devices: + removal = client_factory.create( + 'ns0:VirtualDeviceConfigSpec') + removal.device = device + removal.operation = 'remove' + device_change.append(removal) + + config_spec.deviceChange = device_change + + clone_spec = vm_util.clone_vm_spec(client_factory, rel_spec, + power_on=False, + snapshot=snapshot_ref, + template=True, + config=config_spec) + + LOG.debug("Cloning VM %s", vm_name, instance=instance) + vm_clone_task = self._session._call_method( + self._session.vim, + "CloneVM_Task", + vm_ref, + folder=self._get_project_folder(dc_info, + project_id=instance.project_id, type_='Images'), + name=vm_name, + spec=clone_spec) + self._session._wait_for_task(vm_clone_task) + task_info = self._session._call_method(vutil, + "get_object_property", + vm_clone_task, + "info") + return task_info.result + def snapshot(self, context, instance, image_id, update_task_state): """Create snapshot from a running VM instance. @@ -1012,27 +1337,46 @@ def _get_vm_and_vmdk_attribs(): # TODO(vui): convert to creating plain vm clone and uploading from it # instead of using live vm snapshot. - snapshot_ref = self._create_vm_snapshot(instance, vm_ref) - update_task_state(task_state=task_states.IMAGE_UPLOADING, - expected_state=task_states.IMAGE_PENDING_UPLOAD) + snapshot_ref = None + snapshot_vm_ref = None try: - # Create a temporary VM (linked clone from snapshot), then export - # the VM's root disk to glance via HttpNfc API - snapshot_vm_ref = self._create_linked_clone_from_snapshot( - instance, vm_ref, snapshot_ref, dc_info) + # If we do linked clones, we need to have a snapshot + if (CONF.vmware.clone_from_snapshot or + not CONF.vmware.full_clone_snapshots): + snapshot_ref = self._create_vm_snapshot( + instance, vm_ref, image_id=image_id) + + if not CONF.vmware.full_clone_snapshots: + disk_move_type = "createNewChildDiskBacking" + else: + disk_move_type = None + + snapshot_vm_ref = self._create_vm_clone( + instance, vm_ref, snapshot_ref, dc_info, + disk_move_type=disk_move_type, image_id=image_id, + disks=[vmdk]) + + update_task_state(task_state=task_states.IMAGE_UPLOADING, + expected_state=task_states.IMAGE_PENDING_UPLOAD) images.upload_image_stream_optimized( context, image_id, instance, self._session, vm=snapshot_vm_ref, vmdk_size=vmdk.capacity_in_bytes) finally: if snapshot_vm_ref: - vm_util.destroy_vm(self._session, instance, snapshot_vm_ref) + try: + vm_util.destroy_vm(self._session, instance, + snapshot_vm_ref) + except Exception: + # exception is logged inside the function. we can continue. + pass # Deleting the snapshot after destroying the temporary VM created # based on it allows the instance vm's disks to be consolidated. # TODO(vui) Add handling for when vmdk volume is attached. - self._delete_vm_snapshot(instance, vm_ref, snapshot_ref) + if snapshot_ref: + self._delete_vm_snapshot(instance, vm_ref, snapshot_ref) def reboot(self, instance, network_info, reboot_type="SOFT"): """Reboot a VM instance.""" @@ -1100,6 +1444,7 @@ def _destroy_instance(self, instance, destroy_disks=True): LOG.warning("In vmwareapi:vmops:_destroy_instance, got " "this exception while un-registering the VM: %s", excep, instance=instance) + # Delete the folder holding the VM related content on # the datastore. if destroy_disks and vm_ds_path: @@ -1328,6 +1673,19 @@ def _clean_shutdown(self, instance, timeout, retry_interval): return False + def is_instance_in_resource_pool(self, instance): + try: + vm_ref = vm_util.get_vm_ref(self._session, instance) + res_pool = self._session._call_method(vutil, "get_object_property", + vm_ref, "resourcePool") + + return vutil.get_moref_value(res_pool) == \ + vutil.get_moref_value(self._root_resource_pool) + except (exception.InstanceNotFound, + vexc.ManagedObjectNotFoundException): + LOG.debug("Failed to find instance", instance=instance) + return False + def _get_instance_props(self, vm_ref): lst_properties = ["summary.guest.toolsStatus", "runtime.powerState", @@ -1371,10 +1729,41 @@ def _resize_vm(self, context, instance, vm_ref, flavor, image_meta): metadata=metadata) vm_util.reconfigure_vm(self._session, vm_ref, vm_resize_spec) + old_flavor = instance.old_flavor + old_is_big = utils.is_big_vm(int(old_flavor.memory_mb), old_flavor) + new_is_big = utils.is_big_vm(int(flavor.memory_mb), flavor) + + if not old_is_big and new_is_big: + # Make sure we don't automatically move around "big" VMs + behavior = constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED + LOG.debug("Adding DRS override '%s' for big VM.", behavior, + instance=instance) + cluster_util.update_cluster_drs_vm_override(self._session, + self._cluster, + vm_ref, + operation='add', + behavior=behavior) + elif old_is_big and not new_is_big: + # remove the old override, if we had one before. make sure we don't + # error out if it was already deleted another way + LOG.debug("Removing DRS override for former big VM.", + instance=instance) + try: + cluster_util.update_cluster_drs_vm_override(self._session, + self._cluster, + vm_ref, + operation='remove') + except Exception: + LOG.exception('Could not remove DRS override.', + instance=instance) + + self._clean_up_after_special_spawning(context, flavor.memory_mb, + flavor) + def _resize_disk(self, instance, vm_ref, vmdk, flavor): extra_specs = self._get_extra_specs(instance.flavor, instance.image_meta) - if (flavor.root_gb > instance.flavor.root_gb and + if ((flavor.root_gb > instance.old_flavor.root_gb) and flavor.root_gb > vmdk.capacity_in_bytes / units.Gi): root_disk_in_kb = flavor.root_gb * units.Mi ds_ref = vmdk.device.backing.datastore @@ -1425,34 +1814,25 @@ def _resize_create_ephemerals_and_swap(self, vm_ref, instance, self._create_swap(block_device_info, instance, vm_ref, dc_info, datastore, folder, vmdk.adapter_type) - def migrate_disk_and_power_off(self, context, instance, dest, - flavor, block_device_info): + def migrate_disk_and_power_off(self, context, instance, dest, flavor): """Transfers the disk of a running instance in multiple phases, turning off the instance before the end. """ vm_ref = vm_util.get_vm_ref(self._session, instance) vmdk = vm_util.get_vmdk_info(self._session, vm_ref) - def _is_volume_backed(bdi): - # this contains anything with _valid_destination = 'volume', - # ephemerals have their own list - bdm = driver.block_device_info_get_mapping(bdi) - for disk in bdm: - if disk.get('boot_index') == 0: - return True - return False + boot_from_volume = compute_utils.is_volume_backed_instance(context, + instance) # Checks if the migration needs a disk resize down. - if (not _is_volume_backed(block_device_info) and ( - flavor.root_gb < instance.flavor.root_gb or - (flavor.root_gb != 0 and - flavor.root_gb < vmdk.capacity_in_bytes / units.Gi))): + if (not boot_from_volume and ( + flavor.root_gb < instance.flavor.root_gb or + (flavor.root_gb != 0 and + flavor.root_gb < vmdk.capacity_in_bytes / units.Gi))): reason = _("Unable to shrink disk.") raise exception.InstanceFaultRollback( exception.ResizeError(reason=reason)) - # TODO(garyk): treat dest parameter. Migration needs to be treated. - # 0. Zero out the progress to begin self._update_instance_progress(context, instance, step=0, @@ -1464,25 +1844,18 @@ def _is_volume_backed(bdi): step=1, total_steps=RESIZE_TOTAL_STEPS) - # 2. Reconfigure the VM properties - self._resize_vm(context, instance, vm_ref, flavor, instance.image_meta) - - self._update_instance_progress(context, instance, - step=2, - total_steps=RESIZE_TOTAL_STEPS) + def get_vif_info(self, ctxt, vif_model=None, network_info=None): + vif_info = vmwarevif.get_vif_info(self._session, + self._cluster, + vif_model, + network_info) + return vif_info - # 3.Reconfigure the disk properties - if not _is_volume_backed(block_device_info): - self._resize_disk(instance, vm_ref, vmdk, flavor) - self._update_instance_progress(context, instance, - step=3, - total_steps=RESIZE_TOTAL_STEPS) + def api_for_migration(self, migration): + if migration.dest_compute == migration.source_compute: + return self - # 4. Purge ephemeral and swap disks - self._remove_ephemerals_and_swap(vm_ref) - self._update_instance_progress(context, instance, - step=4, - total_steps=RESIZE_TOTAL_STEPS) + return VmwareRpcApi(migration.dest_compute) def confirm_migration(self, migration, instance, network_info): """Confirms a resize, destroying the source VM.""" @@ -1556,6 +1929,26 @@ def finish_revert_migration(self, context, instance, network_info, if vmdk.device: self._revert_migration_update_disks(vm_ref, instance, vmdk, block_device_info) + # Relocate the instance back, if needed + if instance.uuid not in self.list_instances(): + # Get the root disk vmdk object's adapter type + adapter_type = vmdk.adapter_type + + self._detach_volumes(instance, block_device_info) + LOG.debug("Relocating VM for reverting migration", + instance=instance) + try: + self._relocate_vm(vm_ref, context, instance, network_info) + LOG.debug("Relocated VM for reverting migration", + instance=instance) + except Exception as e: + with excutils.save_and_reraise_exception(): + LOG.error("Relocating the VM failed: %s", e, + instance=instance) + else: + self.update_cluster_placement(context, instance) + finally: + self._attach_volumes(instance, block_device_info, adapter_type) if power_on: vm_util.power_on_instance(self._session, instance) @@ -1566,20 +1959,209 @@ def finish_migration(self, context, migration, instance, disk_info, """Completes a resize, turning on the migrated instance.""" vm_ref = vm_util.get_vm_ref(self._session, instance) - # 5. Update ephemerals if necessary + flavor = instance.flavor + boot_from_volume = compute_utils.is_volume_backed_instance(context, + instance) + reattach_volumes = False + # 2. Relocate the VM if necessary + # If the dest_compute is different from the source_compute, it means we + # need to relocate the VM here since we are running on the dest_compute + if migration.source_compute != migration.dest_compute: + # Get the root disk vmdk object's adapter type + vmdk = vm_util.get_vmdk_info(self._session, vm_ref, + uuid=instance.uuid) + adapter_type = vmdk.adapter_type + + self._detach_volumes(instance, block_device_info) + reattach_volumes = True + LOG.debug("Relocating VM for migration to %s", + migration.dest_compute, instance=instance) + try: + self._relocate_vm(vm_ref, context, instance, network_info, + image_meta) + LOG.debug("Relocated VM to %s", migration.dest_compute, + instance=instance) + except Exception as e: + with excutils.save_and_reraise_exception(): + LOG.error("Relocating the VM failed with error: %s", e, + instance=instance) + self._attach_volumes(instance, block_device_info, + adapter_type) + + self.update_cluster_placement(context, instance) + + self._update_instance_progress(context, instance, + step=2, + total_steps=RESIZE_TOTAL_STEPS) + # 3.Reconfigure the VM and disk + self._resize_vm(context, instance, vm_ref, flavor, image_meta) + if not boot_from_volume and resize_instance: + vmdk = vm_util.get_vmdk_info(self._session, vm_ref, + uuid=instance.uuid) + self._resize_disk(instance, vm_ref, vmdk, flavor) + self._update_instance_progress(context, instance, + step=3, + total_steps=RESIZE_TOTAL_STEPS) + + # 4. Purge ephemeral and swap disks + self._remove_ephemerals_and_swap(vm_ref) + self._update_instance_progress(context, instance, + step=4, + total_steps=RESIZE_TOTAL_STEPS) + # 5. Update ephemerals self._resize_create_ephemerals_and_swap(vm_ref, instance, block_device_info) - self._update_instance_progress(context, instance, step=5, total_steps=RESIZE_TOTAL_STEPS) - # 6. Start VM - if power_on: - vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) + # 6. Attach the volumes (if necessary) + if reattach_volumes: + self._attach_volumes(instance, block_device_info, adapter_type) self._update_instance_progress(context, instance, step=6, total_steps=RESIZE_TOTAL_STEPS) + # 7. Start VM + if power_on: + vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) + self._update_instance_progress(context, instance, + step=7, + total_steps=RESIZE_TOTAL_STEPS) + + def _relocate_vm(self, vm_ref, context, instance, network_info, + image_meta=None): + image_meta = image_meta or instance.image_meta + storage_policy = self._get_storage_policy(instance.flavor) + allowed_ds_types = ds_util.get_allowed_datastore_types( + image_meta.properties.hw_disk_type) + datastore = ds_util.get_datastore(self._session, self._cluster, + self._datastore_regex, + storage_policy, + allowed_ds_types) + dc_info = self.get_datacenter_ref_and_name(datastore.ref) + folder = self._get_project_folder(dc_info, instance.project_id, + 'Instances') + + client_factory = self._session.vim.client.factory + spec = vm_util.relocate_vm_spec(client_factory, + res_pool=self._root_resource_pool, + folder=folder, datastore=datastore.ref) + + # Iterate over the network adapters and update the backing + if network_info: + spec.deviceChange = [] + vif_model = image_meta.properties.get('hw_vif_model', + constants.DEFAULT_VIF_MODEL) + hardware_devices = self._session._call_method( + vutil, + "get_object_property", + vm_ref, + "config.hardware.device") + vif_infos = vmwarevif.get_vif_info(self._session, + self._cluster, + vif_model, + network_info) + for vif_info in vif_infos: + device = vmwarevif.get_network_device(hardware_devices, + vif_info['mac_address']) + if not device: + msg = _("No device with MAC address %s exists on the " + "VM") % vif_info['mac_address'] + raise exception.NotFound(msg) + + # Update the network device backing + config_spec = client_factory.create( + 'ns0:VirtualDeviceConfigSpec') + vm_util.set_net_device_backing( + client_factory, device, vif_info) + config_spec.operation = "edit" + config_spec.device = device + spec.deviceChange.append(config_spec) + + vm_util.relocate_vm(self._session, vm_ref, spec=spec) + + def live_migration(self, instance, migrate_data, volume_mapping): + defaults = migrate_data.relocate_defaults + + client_factory = self._session.vim.client.factory + relocate_spec = vim_util.deserialize_object(client_factory, + defaults["relocate_spec"], "VirtualMachineRelocateSpec") + + if not migrate_data.is_same_vcenter: + disk_move_type = "moveAllDiskBackingsAndDisallowSharing" + else: + disk_move_type = "moveAllDiskBackingsAndAllowSharing" + + relocate_spec.diskMoveType = disk_move_type + + datastore = relocate_spec.datastore + + service = defaults.get("service") + if service: + relocate_spec.service = vim_util.deserialize_object( + client_factory, service, "ServiceLocator") + + vm_ref = vm_util.get_vm_ref(self._session, instance) + + device_config_spec = [] + relocate_spec.deviceChange = device_config_spec + disks = [] + relocate_spec.disk = disks + + netdevices = [] + for device in vm_util.get_hardware_devices(self._session, vm_ref): + class_name = device.__class__.__name__ + if class_name in vm_util.ALL_SUPPORTED_NETWORK_DEVICES: + netdevices.append(device) + elif class_name == "VirtualDisk": + locator = client_factory.create( + "ns0:VirtualMachineRelocateSpecDiskLocator") + locator.diskId = device.key + target = volume_mapping.get(device.key) + if not target: # Not a volume + locator.datastore = datastore + else: + locator.datastore = target["datastore_ref"] + profile_id = target.get("profile_id") + if profile_id: + profile_spec = client_factory.create( + "ns0:VirtualMachineDefinedProfileSpec") + profile_spec.profileId = profile_id + locator.profile = [profile_spec] + disks.append(locator) + + for vif_info in migrate_data.vif_infos: + device = vmwarevif.get_network_device(netdevices, + vif_info["mac_address"]) + if not device: + msg = _("No device with MAC address %s exists on the " + "VM") % vif_info["mac_address"] + raise exception.NotFound(msg) + + # Update the network device backing + config_spec = client_factory.create("ns0:VirtualDeviceConfigSpec") + vm_util.set_net_device_backing(client_factory, device, vif_info) + config_spec.operation = "edit" + config_spec.device = device + device_config_spec.append(config_spec) + + vm_util.relocate_vm(self._session, vm_ref, spec=relocate_spec) + + def _detach_volumes(self, instance, block_device_info): + block_devices = driver.block_device_info_get_mapping(block_device_info) + for disk in block_devices: + self._volumeops.detach_volume(disk['connection_info'], instance) + + def _attach_volumes(self, instance, block_device_info, adapter_type): + disks = driver.block_device_info_get_mapping(block_device_info) + # make sure the disks are attached by the boot_index order (if any) + for disk in sorted(disks, + key=lambda d: d['boot_index'] + if 'boot_index' in d and d['boot_index'] > -1 + else len(disks)): + adapter_type = disk.get('disk_bus') or adapter_type + self._volumeops.attach_volume(disk['connection_info'], instance, + adapter_type) def _find_esx_host(self, cluster_ref, ds_ref): """Find ESX host in the specified cluster which is also connected to @@ -1626,59 +2208,6 @@ def _find_datastore_for_migration(self, instance, vm_ref, cluster_ref, return ds_util.get_datastore(self._session, cluster_ref, datastore_regex) - def live_migration(self, context, instance, dest, - post_method, recover_method, block_migration, - migrate_data): - LOG.debug("Live migration data %s", migrate_data, instance=instance) - vm_ref = vm_util.get_vm_ref(self._session, instance) - cluster_name = migrate_data.cluster_name - cluster_ref = vm_util.get_cluster_ref_by_name(self._session, - cluster_name) - datastore_regex = re.compile(migrate_data.datastore_regex) - res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref) - # find a datastore where the instance will be migrated to - ds = self._find_datastore_for_migration(instance, vm_ref, cluster_ref, - datastore_regex) - if ds is None: - LOG.error("Cannot find datastore", instance=instance) - raise exception.HostNotFound(host=dest) - LOG.debug("Migrating instance to datastore %s", ds.name, - instance=instance) - # find ESX host in the destination cluster which is connected to the - # target datastore - esx_host = self._find_esx_host(cluster_ref, ds.ref) - if esx_host is None: - LOG.error("Cannot find ESX host for live migration, cluster: %s, " - "datastore: %s", migrate_data.cluster_name, ds.name, - instance=instance) - raise exception.HostNotFound(host=dest) - # Update networking backings - network_info = instance.get_network_info() - client_factory = self._session.vim.client.factory - devices = [] - hardware_devices = vm_util.get_hardware_devices(self._session, vm_ref) - vif_model = instance.image_meta.properties.get('hw_vif_model', - constants.DEFAULT_VIF_MODEL) - for vif in network_info: - vif_info = vmwarevif.get_vif_dict( - self._session, cluster_ref, vif_model, vif) - device = vmwarevif.get_network_device(hardware_devices, - vif['address']) - devices.append(vm_util.update_vif_spec(client_factory, vif_info, - device)) - - LOG.debug("Migrating instance to cluster '%s', datastore '%s' and " - "ESX host '%s'", cluster_name, ds.name, esx_host, - instance=instance) - try: - vm_util.relocate_vm(self._session, vm_ref, res_pool_ref, - ds.ref, esx_host, devices=devices) - LOG.info("Migrated instance to host %s", dest, instance=instance) - except Exception: - with excutils.save_and_reraise_exception(): - recover_method(context, instance, dest, migrate_data) - post_method(context, instance, dest, block_migration, migrate_data) - def poll_rebooting_instances(self, timeout, instances): """Poll for rebooting instances.""" ctxt = nova_context.get_admin_context() @@ -1890,26 +2419,40 @@ def manage_image_cache(self, context, instances): datastores_info.append((ds, dc_info)) self._imagecache.update(context, instances, datastores_info) - def _get_valid_vms_from_retrieve_result(self, retrieve_result): - """Returns list of valid vms from RetrieveResult object.""" + def _get_valid_vms_from_retrieve_result(self, retrieve_result, + return_properties=False): + """Returns list of valid vms from RetrieveResult object. + + If `return_properties` is True, it will also return the properties of + these VMs, thus returning a tuple (vm_uuid, properties). + """ lst_vm_names = [] with vutil.WithRetrieval(self._session.vim, retrieve_result) as \ objects: for vm in objects: + prop_set = getattr(vm, 'propSet', None) + if not prop_set: + continue vm_uuid = None conn_state = None + props = {} for prop in vm.propSet: if prop.name == "runtime.connectionState": conn_state = prop.val elif prop.name == 'config.extraConfig["nvp.vm-uuid"]': vm_uuid = prop.val.value + props[prop.name] = prop.val # Ignore VM's that do not have nvp.vm-uuid defined if not vm_uuid: continue # Ignoring the orphaned or inaccessible VMs - if conn_state not in ["orphaned", "inaccessible"]: - lst_vm_names.append(vm_uuid) + if conn_state in ["orphaned", "inaccessible"]: + continue + if return_properties: + lst_vm_names.append((vm_uuid, props)) + else: + lst_vm_names.append(vm_uuid) return lst_vm_names def instance_exists(self, instance): @@ -2127,19 +2670,29 @@ def get_datacenter_ref_and_name(self, ds_ref): return ds_util.get_dc_info(self._session, ds_ref) def list_instances(self): + lst_vm_names = self._list_instances_in_cluster() + + LOG.debug("Got total of %s instances", str(len(lst_vm_names))) + + return lst_vm_names + + def _list_instances_in_cluster(self, additional_properties=None): """Lists the VM instances that are registered with vCenter cluster.""" properties = ['runtime.connectionState', 'config.extraConfig["nvp.vm-uuid"]'] + if additional_properties is not None: + properties.extend(additional_properties) LOG.debug("Getting list of instances from cluster %s", - self._cluster) + vutil.get_moref_value(self._cluster)) vms = [] if self._root_resource_pool: vms = self._session._call_method( vim_util, 'get_inner_objects', self._root_resource_pool, 'vm', 'VirtualMachine', properties) - lst_vm_names = self._get_valid_vms_from_retrieve_result(vms) + return_properties = additional_properties is not None + lst_vm_names = self._get_valid_vms_from_retrieve_result(vms, + return_properties=return_properties) - LOG.debug("Got total of %s instances", str(len(lst_vm_names))) return lst_vm_names def get_vnc_console(self, instance): @@ -2174,3 +2727,311 @@ def get_mks_console(self, instance): 'thumbprint': thumbprint} internal_access_path = jsonutils.dumps(mks_auth) return ctype.ConsoleMKS(ticket.host, ticket.port, internal_access_path) + + def set_compute_host(self, compute_host): + """Called by the driver on init_host() so we know the compute host""" + self._compute_host = compute_host + + def sync_server_group(self, context, sg_uuid): + """Sync a server group by its uuid for the current host/cluster + """ + # we have to ignore instances currently in a volatitle state, where + # either VMware cannot support them being in a DRS rule or we expect + # them to go away during the syncing process, which could lead to + # errors. Therefore, we explicitly remove those members from the list + # of expected members of a rule, which also removes them in the + # cluster. + STATES_EXCLUDING_MEMBERS_FROM_DRS_RULES = [ + task_states.DELETING, + task_states.SHELVING, + task_states.REBUILDING, + task_states.REBUILD_BLOCK_DEVICE_MAPPING, + ] + LOG.debug('Starting sync for server-group %s', sg_uuid) + + @utils.synchronized('vmware-server-group-{}'.format(sg_uuid)) + def _sync_sync_server_group(context, sg_uuid): + rule_prefix = '{}{}'.format(constants.DRS_PREFIX, sg_uuid) + + # retrieve the server-group with its members + try: + sg = objects.instance_group.InstanceGroup.get_by_uuid(context, + sg_uuid) + except exception.InstanceGroupNotFound: + LOG.info('Server-group %s cannot be found in DB', sg_uuid) + # check if we have it in the vCenter. if yes, it's an orphan + # and needs deletion + rules = cluster_util.get_rules_by_prefix( + self._session, self._cluster, rule_prefix) + for rule in rules: + LOG.debug('Deleting DRS rule %s as orphan', rule.name) + cluster_util.delete_rule( + self._session, self._cluster, rule) + LOG.info('Deleted rule %s as orphan', rule.name) + LOG.debug('Sync for server-group %s done', sg_uuid) + return + + # First we check for all instances, which have ongoing migrations + # on the given host, either as source or destination + + # Decision matrix for migrations: + # Mig-Status Action/Host + # Source Dest + # Preparing Remove N/A + # Running Remove Add + # (Other states are consistent with default behaviour) + # + # So the Instance.host will always be the source of the migration, + # and we want to remove the rules. + # We only need to handle specially the case a running migration + # on the destination host, and add it + + MigrationList = objects.migration.MigrationList + filters = { + "host": self._compute_host, + "instance_uuid": sg.members, + "status": ["preparing", "running"], + } + + expected_members = {} + + migrations_by_instance_uuid = {} + for migration in MigrationList.get_by_filters(context, filters): + instance_uuid = migration.instance_uuid + if migration.source_compute == self._compute_host: + # The host is the source of a migration + # That means the instance will be part of the instance list + # So we have to remember that instance to be removed from + # the DRS rule-set + migrations_by_instance_uuid[instance_uuid] = \ + migration + else: + # We now handle the destination side + if migration.status == "preparing": + # Not even started, we can ignore that one + continue + + # Polling the cache, as vm_util.get_vm_ref is very slow + # for the negative search. + # We just have to ensure, that the cache holds a value + # before syncing the server group on the destination host + # Race conditions are averted by this functions lock + moref = vm_util.vm_ref_cache_get(instance_uuid) + if moref: + expected_members[instance_uuid] = moref + + # retrieve the instances, because sg.members contains all members + # and we need to filter them for our host + InstanceList = objects.instance.InstanceList + filters = {'host': self._compute_host, 'uuid': sg.members, + 'deleted': False} + instances = InstanceList.get_by_filters(context, filters, + expected_attrs=[]) + + for instance in instances: + task_state = instance.task_state + if task_state in STATES_EXCLUDING_MEMBERS_FROM_DRS_RULES: + LOG.debug("Excluding member %s of server-group %s, " + "because it's in task_state %s.", + instance.uuid, sg.uuid, task_state) + continue + + migration = migrations_by_instance_uuid.get(instance.uuid) + if migration: + LOG.debug("Excluding member %s of server-group %s, " + "due to being on the source side of " + "ongoing migration %s.", + instance.uuid, sg.uuid, migration.uuid) + continue + + try: + moref = vm_util.get_vm_ref(self._session, instance) + except exception.InstanceNotFound: + LOG.warning('Could not find moref for instance %s. ' + 'Ignoring member of server-group %s', + instance.uuid, sg.uuid) + continue + expected_members[instance.uuid] = moref + + rule_members_by_name = {} + if sg.policy == 'soft-anti-affinity': + # we chunk by available hosts - 1, because we can spawn only as + # many VMs as there are hosts as VMWare doesn't provide any + # "soft" anti-affinity except for VM-Host relations, while we + # still allow 1 host to go into maintenance mode. + # Only hosts have an 'available' field - the cluster doesn't. + # Therefore, we don't have to filter out the aggregated cluster + # stats explicitly. + member_chunk_size = len( + [stat for stat in self._vc_state.get_host_stats().values() + if stat.get('available', False)]) + member_chunk_size = max(member_chunk_size - 1, 1) + + # to generate stable chunks we have to sort the expected + # members. we also need a list to be able to access parts of + # them. + expected_members = sorted(expected_members.items()) + + # generate chunks of the expected members with rules having a + # postfix counting up for the soft-anti-affinity policy + for i, j in enumerate(range(0, len(expected_members), + member_chunk_size)): + rule_name = '{}-{}-{}'.format(rule_prefix, sg.policy, i) + rule_members = expected_members[j:j + member_chunk_size] + rule_members_by_name[rule_name] = rule_members + + # we need to add in the existing rules with the same prefix, + # because there might be 1) old rules from before the chunking + # and 2) rules we don't reach anymore because the number of + # members of the sg is much lower now + existing_rule_names = [ + rule['name'] for rule in cluster_util.get_rules_by_prefix( + self._session, self._cluster, rule_prefix)] + + rule_names = \ + set(existing_rule_names) | set(rule_members_by_name) + for rule_name in rule_names: + rule_members = rule_members_by_name.get(rule_name, []) + _update_rule(rule_name, dict(rule_members), sg) + else: + # no chunking necessary - just update the rule + rule_name = '{}-{}'.format(rule_prefix, sg.policy) + _update_rule(rule_name, expected_members, sg) + + LOG.debug('Sync for server-group %s done', sg_uuid) + + def _update_rule(rule_name, expected_members, sg): + # we need to get by "prefix", to get all rules matching our name, + # as there can be duplication happening with automatically + # vSphere-created rules during vMotion + rules = [r for r in cluster_util.get_rules_by_prefix( + self._session, self._cluster, rule_name) + if r.name == rule_name] + + rule = rules[0] if rules else None + + # if we have duplicates (with the same name), delete them + for dupl_rule in rules[1:]: + LOG.debug('Deleting DRS rule %s with key %s as duplicate', + dupl_rule.name, dupl_rule.key) + cluster_util.delete_rule( + self._session, self._cluster, dupl_rule) + LOG.info('Deleted rule %s with key %s as duplicate', + dupl_rule.name, dupl_rule.key) + + if not rule: + if len(expected_members) < 2 or sg.policy == 'soft-affinity': + return + # we have to create a new rule + LOG.debug('Creating missing DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + client_factory = self._session.vim.client.factory + rule = cluster_util.create_vm_rule( + client_factory, rule_name, list(expected_members.values()), + policy=sg.policy) + cluster_util.add_rule( + self._session, self._cluster, rule) + LOG.info('Created missing DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + return + + if sg.policy == 'soft-affinity': + LOG.debug('Deleting DRS rule %s with policy soft-affinity', + rule_name) + cluster_util.delete_rule( + self._session, self._cluster, rule) + LOG.info('Deleted DRS rule %s with policy soft-affinity.', + rule_name) + return + + if len(expected_members) < 2: + # we have to delete the rule + LOG.debug('Deleting DRS rule %s with < 2 members.', rule_name) + cluster_util.delete_rule( + self._session, self._cluster, rule) + LOG.info('Deleted DRS rule %s with < 2 members.', rule_name) + return + + if not rule.enabled: + LOG.debug('Enabling DRS rule %s.', rule_name) + rule.enabled = True + cluster_util.update_rule( + self._session, self._cluster, rule) + LOG.info('Enabled DRS rule %s.', rule_name) + + expected_moref_values = set(vutil.get_moref_value(m) + for m in expected_members.values()) + existing_moref_values = set(vutil.get_moref_value(m) + for m in rule.vm) + if expected_moref_values == existing_moref_values: + return + + # we have to update the DRS rule to contain the right members + rule.vm = list(expected_members.values()) + LOG.debug('Updating DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + cluster_util.update_rule(self._session, self._cluster, rule) + LOG.info('Updated DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + + _sync_sync_server_group(context, sg_uuid) + + def place_vm(self, context, instance): + # We currently only fill the bare-minimum to get a placement. + # The datastore for the VM is selected as on instance creation, + # instead of allowing placevm to decide it (which may be better) + # We also do not pass the information about the NICs and other + # attached disks, which may give the placement a better information + client_factory = self._session.vim.client.factory + flavor = instance.flavor + image_meta = instance.image_meta + image_info = images.VMwareImage.from_image(context, + instance.image_ref, + image_meta) + + extra_specs = self._get_extra_specs(flavor, image_meta) + + vi = self._get_vm_config_info(instance, image_info, extra_specs) + + vm_folder = self._get_project_folder(vi.dc_info, + project_id=instance.project_id, type_='Instances') + + relocate_spec = vm_util.relocate_vm_spec( + client_factory, + res_pool=self._root_resource_pool, + datastore=vi.datastore.ref, + disk_move_type="moveAllDiskBackingsAndDisallowSharing", + folder=vm_folder) + + placement_spec = client_factory.create("ns0:PlacementSpec") + placement_spec.placementType = "relocate" + # Maybe we want to allow that? Default is True + # placement_spec.disallowPrerequisiteMoves = False + placement_spec.relocateSpec = relocate_spec + # So we do not place the vm on a failover host + placement_spec.hosts, _ = \ + vm_util.get_hosts_and_reservations_for_cluster( + self._session, self._cluster) + + # Sets cpuAllocation, memoryAllocation, numCPUs, memoryMB + config_spec = vm_util.get_vm_resize_spec(client_factory, + int(flavor.vcpus), + int(flavor.memory_mb), + extra_specs) + # The last mandatory field for config_spec per doc + config_spec.version = extra_specs.hw_version + placement_spec.configSpec = config_spec + + vm_group_name = self._get_admin_group_name_for_instance(instance) + if vm_group_name: + placement_rules = [] + placement_spec.rules = placement_rules + cluster_rules = cluster_util.fetch_cluster_rules(self._session, + self._cluster) + for rule in cluster_rules.values(): + if getattr(rule, "vmGroupName", None) == vm_group_name: + placement_rules.append(rule) + + result = self._session._call_method(self._session.vim, "PlaceVm", + self._cluster, placementSpec=placement_spec) + return result diff --git a/nova/virt/vmwareapi/volumeops.py b/nova/virt/vmwareapi/volumeops.py index 9bc195f192d..1f668dc48ac 100644 --- a/nova/virt/vmwareapi/volumeops.py +++ b/nova/virt/vmwareapi/volumeops.py @@ -25,6 +25,7 @@ import nova.conf from nova import exception from nova.i18n import _ +from nova.virt import driver from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import session from nova.virt.vmwareapi import vm_util @@ -671,3 +672,120 @@ def attach_root_volume(self, connection_info, instance, # we don't do that. This comment should help us detect upstream changes # to the function as merging should fail. self.attach_volume(connection_info, instance, adapter_type) + + def fixup_shadow_vms(self, instance, shadow_vms): + """The function ensures that all volumes attached to a VM are also + attached at the corresponding shadow vms. + + As the vcenter controls both VMs and volumes, and you can only move + them together in a single migration operation between vcenters, + Cinder cannot move the volumes independently. + It only can create the shadow-vms in the target vcenter. + The backing then gets moved through the live-migration and needs to + be reassociated with (attached to) the corresponding shadow-vms + + It handles the case that the shadow-vm is without backing (normal-case) + But also that there is a pre-existing backing (e.g. we retry a failed + migration) + In the latter case, we discard the existing attachment of the + shadow-vm, as the attached volume is the one with the more current data + + :param instance: The instance moved via a live-migration and to which + the volumes are attached to + :param shadow_vms: A mapping of devices with volumes to the + connection_info.data information, most importantly holding the "volume" + field, which is a managed-object reference to a shadow-vm + """ + # This should sensibly moved out to cinder: + # Cinder can delete the old shadow-vm, as soon as the attachment + # for the vm prior the vmotion gets deleted + vm_ref = vm_util.get_vm_ref(self._session, instance) + for device in vm_util.get_hardware_devices(self._session, vm_ref): + class_name = device.__class__.__name__ + + if class_name != "VirtualDisk": + continue + + data = shadow_vms.get(device.key) + if not data: + continue + + try: + current_device_path = device.backing.fileName + volume = data["volume"] + volume_ref = vutil.get_moref(volume, "VirtualMachine") + original_device = self._get_vmdk_base_volume_device(volume_ref) + if original_device: + original_device_path = original_device.backing.fileName + if original_device_path == current_device_path: + # Already attached + continue + + # That should not happen + LOG.warning("Shadow-vm %s already has a disk" + " attached at %s replacing it with %s", + volume, original_device_path, current_device_path, + instance=instance + ) + self.detach_disk_from_vm(volume_ref, instance, + original_device, destroy_disk=True) + + disk_type = vm_util._get_device_disk_type(device) + self.attach_disk_to_vm(volume_ref, + instance, + constants.DEFAULT_ADAPTER_TYPE, + disk_type, + current_device_path + ) + except Exception: + LOG.exception("Failed to attach volume %s. Device %s", + data["volume_id"], device.key, + instance=instance) + + def delete_shadow_vms(self, block_device_info, instance=None): + # We need to delete the migrated shadow vms + # (until we implement it in cinder) + block_device_mapping = driver.block_device_info_get_mapping( + block_device_info) + + if not block_device_mapping: + return + + session = self._session + deleted = [] + for disk in block_device_mapping: + connection_info = disk["connection_info"] + try: + data = connection_info["data"] + volume_ref = self._get_volume_ref(data) + destroy_task = session._call_method(session.vim, + "Destroy_Task", + volume_ref) + session._wait_for_task(destroy_task) + deleted.append("{volume_id} ({volume})".format(**data)) + except oslo_vmw_exceptions.ManagedObjectNotFoundException: + LOG.debug("Volume %s already deleted", + data.get("volume_id"), instance=instance) + except Exception: + LOG.exception("Failed to delete volume %s", + data.get("volume_id"), instance=instance) + + LOG.info("Deleted %s", deleted, instance=instance) + + def map_volumes_to_devices(self, instance, disk_infos): + """Maps a connection_info.data to a device of the instance by its key + :param instance: The instance the volumes are attached to + :param disk_infos: An iterable over a dict containing at least + 'volume_id' + :returns: A dict {device_key: disk_info} mapping the passed disk_info + dicts to a a device by the stored volume_id + """ + remapped = {} + vm_ref = vm_util.get_vm_ref(self._session, instance) + # TODO(fwiesel) Create a function + # _get_vmdk_backed_disk_devices (plural) + # so we do not have two calls for each device + for disk_info in disk_infos: + device = self._get_vmdk_backed_disk_device(vm_ref, disk_info) + remapped[device.key] = disk_info + return remapped diff --git a/nova/volume/cinder.py b/nova/volume/cinder.py index b43d4b551b5..d6f4b2c1164 100644 --- a/nova/volume/cinder.py +++ b/nova/volume/cinder.py @@ -583,28 +583,33 @@ def detach(self, context, volume_id, instance_uuid=None, client = cinderclient(context) if attachment_id is None: volume = self.get(context, volume_id) + attachments = volume.get('attachments', {}) + if instance_uuid: + attachment_id = attachments.get(instance_uuid, {}).\ + get('attachment_id') + else: + LOG.warning("attachment_id couldn't be retrieved for " + "volume %(volume_id)s.", + {'volume_id': volume_id}) if volume['multiattach']: - attachments = volume.get('attachments', {}) - if instance_uuid: - attachment_id = attachments.get(instance_uuid, {}).\ - get('attachment_id') - if not attachment_id: - LOG.warning("attachment_id couldn't be retrieved " - "for volume %(volume_id)s with " - "instance_uuid %(instance_id)s. The " - "volume has the 'multiattach' flag " - "enabled, without the attachment_id " - "Cinder most probably cannot perform " - "the detach.", - {'volume_id': volume_id, - 'instance_id': instance_uuid}) - else: - LOG.warning("attachment_id couldn't be retrieved for " - "volume %(volume_id)s. The volume has the " - "'multiattach' flag enabled, without the " - "attachment_id Cinder most probably " - "cannot perform the detach.", - {'volume_id': volume_id}) + if not attachment_id: + LOG.warning("attachment_id couldn't be retrieved " + "for volume %(volume_id)s with " + "instance_uuid %(instance_id)s. The " + "volume has the 'multiattach' flag " + "enabled, without the attachment_id " + "Cinder most probably cannot perform " + "the detach.", + {'volume_id': volume_id, + 'instance_id': instance_uuid}) + elif not attachment_id and attachments: + # NOTE(jkulik): If we detach unconditionally although we have + # other attachments, we will delete the other attachment's + # attachment entry in Cinder. + LOG.info("Not calling detach, because there are attachments " + "that don't belong to instance %(instance_uuid)s.", + {'instance_uuid': instance_uuid}) + return client.volumes.detach(volume_id, attachment_id) diff --git a/setup.cfg b/setup.cfg index 1b878af0912..8ca42f584c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,6 +77,7 @@ console_scripts = nova-api = nova.cmd.api:main nova-api-metadata = nova.cmd.api_metadata:main nova-api-os-compute = nova.cmd.api_os_compute:main + nova-bigvm = nova.cmd.bigvm:main nova-compute = nova.cmd.compute:main nova-conductor = nova.cmd.conductor:main nova-manage = nova.cmd.manage:main