From 5d68b07746bad3206d541a8dbb6ed24f5a63cb2a Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 24 Feb 2021 10:41:02 +0100 Subject: [PATCH 01/92] manage: Support Ironic in sync_aggregates "nova-manage placement sync_aggregates" doesn't support Ironic hosts, because they have multiple compute nodes. In general, Nova doesn't support aggregates for Ironic hosts. But we still use them for assigning an availability zone to a rack (a building block) of nodes grouped into a Nova host. Therefore, we patch above-mentioned command to handle a list of (in most cases one) compute node UUIDs, only erroring out if the host we find multiple compute nodes for doesn't look like an Ironic host (based on having "ironic" in the name). There can be Ironic hosts, that only have nodes assigned when those nodes are getting repairs or getting build up. Those Ironic hosts would come up empty when searching for ComputeNodes in the sync_aggregates command and would be reported as a problem, which makes the command fail with exit-code 5. Since it's no problem if an Ironic Host doesn't have a ComputeNode, because each node is its own resource provider in placement anyways, we ignore Ironic hosts without nodes now in the error-reporting. Change-Id: I4f7e5fd82c51ce5d6f42089beb5a70e469ec54df Incorporated-Change-Id: I163f3e46f2e375531b870a363b84bba67816954d (cherry picked from commit f899a7c23551fc91e38d4400e8fa25b54fb67104) --- nova/cmd/manage.py | 87 +++++++++++++++++------------- nova/tests/unit/cmd/test_manage.py | 12 ++--- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index 5a00a3c6a40..a9b73dfe46f 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -2377,8 +2377,11 @@ def heal_allocations(self, max_count=None, verbose=False, dry_run=False, return 0 @staticmethod - def _get_rp_uuid_for_host(ctxt, host): - """Finds the resource provider (compute node) UUID for the given host. + def _get_rp_uuids_for_host(ctxt, host): + """Finds the resource provider UUID(s) for the given host. + + This is usually a single value, but we return a list as we must support + Ironic nodes, that have the same host but different node uuids. :param ctxt: cell-targeted nova RequestContext :param host: name of the compute host @@ -2408,12 +2411,13 @@ def _get_rp_uuid_for_host(ctxt, host): nodes = objects.ComputeNodeList.get_all_by_host( cctxt, host) - if len(nodes) > 1: - # This shouldn't happen, so we need to bail since we - # won't know which node to use. + if len(nodes) > 1 and 'ironic' not in host: + # Multiple nodes for non-ironic hosts shouldn't happen, so + # we need to bail since we won't know which node to use. raise exception.TooManyComputesForHost( num_computes=len(nodes), host=host) - return nodes[0].uuid + + return [node.uuid for node in nodes] @action_description( _("Mirrors compute host aggregates to resource provider aggregates " @@ -2466,7 +2470,7 @@ def sync_aggregates(self, verbose=False): # Since hosts can be in more than one aggregate, keep track of the host # to its corresponding resource provider uuid to avoid redundant # lookups. - host_to_rp_uuid = {} + host_to_rp_uuids = {} unmapped_hosts = set() # keep track of any missing host mappings computes_not_found = set() # keep track of missing nodes providers_not_found = {} # map of hostname to missing provider uuid @@ -2474,18 +2478,24 @@ def sync_aggregates(self, verbose=False): output(_('Processing aggregate: %s') % aggregate.name) for host in aggregate.hosts: output(_('Processing host: %s') % host) - rp_uuid = host_to_rp_uuid.get(host) - if not rp_uuid: + rp_uuids = host_to_rp_uuids.get(host) + if not rp_uuids: try: - rp_uuid = self._get_rp_uuid_for_host(ctxt, host) - host_to_rp_uuid[host] = rp_uuid + rp_uuids = self._get_rp_uuids_for_host(ctxt, host) + host_to_rp_uuids[host] = rp_uuids except exception.HostMappingNotFound: # Don't fail on this now, we can dump it at the end. unmapped_hosts.add(host) continue except exception.ComputeHostNotFound: - # Don't fail on this now, we can dump it at the end. - computes_not_found.add(host) + # Don't fail on this now, we can dump it at the end. We + # only need to dump it, if it's a non-ironic host + # though, as for ironic there can be hosts that + # currently don't have any node assigned e.g. + # conductor-groups that are only used for + # testing/during buildup. + if 'ironic' not in host: + computes_not_found.add(host) continue except exception.TooManyComputesForHost as e: # TODO(mriedem): Should we treat this like the other @@ -2497,30 +2507,33 @@ def sync_aggregates(self, verbose=False): # the matching resource provider, found via compute node uuid, # is in the same aggregate in placement, found via aggregate # uuid. - try: - placement.aggregate_add_host(ctxt, aggregate.uuid, - rp_uuid=rp_uuid) - output(_('Successfully added host (%(host)s) and ' - 'provider (%(provider)s) to aggregate ' - '(%(aggregate)s).') % - {'host': host, 'provider': rp_uuid, - 'aggregate': aggregate.uuid}) - except exception.ResourceProviderNotFound: - # The resource provider wasn't found. Store this for later. - providers_not_found[host] = rp_uuid - except exception.ResourceProviderAggregateRetrievalFailed as e: - print(e.message) - return 2 - except exception.NovaException as e: - # The exception message is too generic in this case - print(_('Failed updating provider aggregates for ' - 'host (%(host)s), provider (%(provider)s) ' - 'and aggregate (%(aggregate)s). Error: ' - '%(error)s') % - {'host': host, 'provider': rp_uuid, - 'aggregate': aggregate.uuid, - 'error': e.message}) - return 3 + for rp_uuid in rp_uuids: + try: + placement.aggregate_add_host(ctxt, aggregate.uuid, + rp_uuid=rp_uuid) + output(_('Successfully added host (%(host)s) and ' + 'provider (%(provider)s) to aggregate ' + '(%(aggregate)s).') % + {'host': host, 'provider': rp_uuid, + 'aggregate': aggregate.uuid}) + except exception.ResourceProviderNotFound: + # The resource provider wasn't found. Store this for + # later. + providers_not_found[host] = rp_uuid + except (exception. + ResourceProviderAggregateRetrievalFailed) as e: + print(e.message) + return 2 + except exception.NovaException as e: + # The exception message is too generic in this case + print(_('Failed updating provider aggregates for ' + 'host (%(host)s), provider (%(provider)s) ' + 'and aggregate (%(aggregate)s). Error: ' + '%(error)s') % + {'host': host, 'provider': rp_uuid, + 'aggregate': aggregate.uuid, + 'error': e.message}) + return 3 # Now do our error handling. Note that there is no real priority on # the error code we return. We want to dump all of the issues we hit diff --git a/nova/tests/unit/cmd/test_manage.py b/nova/tests/unit/cmd/test_manage.py index be2541105d0..e7d18a22047 100644 --- a/nova/tests/unit/cmd/test_manage.py +++ b/nova/tests/unit/cmd/test_manage.py @@ -2741,8 +2741,8 @@ def test_sync_aggregates_get_provider_aggs_placement_server_error( mock_agg_add.side_effect = ( exception.ResourceProviderAggregateRetrievalFailed( uuid=uuidsentinel.rp_uuid)) - with mock.patch.object(self.cli, '_get_rp_uuid_for_host', - return_value=uuidsentinel.rp_uuid): + with mock.patch.object(self.cli, '_get_rp_uuids_for_host', + return_value=[uuidsentinel.rp_uuid]): result = self.cli.sync_aggregates(verbose=True) self.assertEqual(2, result) self.assertIn('Failed to get aggregates for resource provider with ' @@ -2764,8 +2764,8 @@ def test_sync_aggregates_put_aggregates_fails_provider_not_found( """ mock_agg_add.side_effect = exception.ResourceProviderNotFound( name_or_uuid=uuidsentinel.rp_uuid) - with mock.patch.object(self.cli, '_get_rp_uuid_for_host', - return_value=uuidsentinel.rp_uuid): + with mock.patch.object(self.cli, '_get_rp_uuids_for_host', + return_value=[uuidsentinel.rp_uuid]): result = self.cli.sync_aggregates(verbose=True) self.assertEqual(6, result) self.assertIn('Unable to find matching resource provider record in ' @@ -2790,8 +2790,8 @@ def test_sync_aggregates_put_aggregates_fails_generation_conflict( """ mock_agg_add.side_effect = exception.ResourceProviderUpdateConflict( uuid=uuidsentinel.rp_uuid, generation=1, error="Conflict!") - with mock.patch.object(self.cli, '_get_rp_uuid_for_host', - return_value=uuidsentinel.rp_uuid): + with mock.patch.object(self.cli, '_get_rp_uuids_for_host', + return_value=[uuidsentinel.rp_uuid]): result = self.cli.sync_aggregates(verbose=True) self.assertEqual(3, result) self.assertIn("Failed updating provider aggregates for " From 3d284806fe20ae2f10ce7a0949bd2951384db25c Mon Sep 17 00:00:00 2001 From: Jakob Karge Date: Thu, 31 Aug 2023 16:44:51 +0200 Subject: [PATCH 02/92] metrics: Measure orphaned resource allocations Change-Id: I117a4d97ff7e901ce9b590f04a40e58053f0e845 (cherry picked from commit a39dd173ef3382c9c556fc19b696680016e951c9) --- nova/compute/resource_tracker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/compute/resource_tracker.py b/nova/compute/resource_tracker.py index 57a3a4ec5e5..3570d04f3ec 100644 --- a/nova/compute/resource_tracker.py +++ b/nova/compute/resource_tracker.py @@ -39,6 +39,7 @@ import nova.conf from nova import exception from nova.i18n import _ +from nova import metrics from nova import objects from nova.objects import base as obj_base from nova.objects import fields @@ -1705,6 +1706,7 @@ def _remove_deleted_instances_allocations(self, context, cn, # the (potentially expensive) context.elevated construction below. return read_deleted_context = context.elevated(read_deleted='yes') + orphan_allocations = 0 for consumer_uuid, alloc in allocations.items(): if consumer_uuid in self.tracked_instances: LOG.debug("Instance %s actively managed on this compute host " @@ -1737,6 +1739,7 @@ def _remove_deleted_instances_allocations(self, context, cn, "compute host but is not found in the database.", {'uuid': instance_uuid}, exc_info=False) + orphan_allocations += 1 continue # NOTE(mriedem): A cross-cell migration will work with instance @@ -1814,6 +1817,7 @@ def _remove_deleted_instances_allocations(self, context, cn, "the source host that might need to be removed: " "%s.", instance_uuid, instance.host, instance.node, alloc) + metrics.gauge("allocation.orphans", orphan_allocations) def delete_allocation_for_evacuated_instance(self, context, instance, node, node_type='source'): From 872747804e7a4586ab1b66886996a95130446442 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 2 Dec 2024 13:32:15 +0100 Subject: [PATCH 03/92] Support in-cluster (anti-)affinity for VMware We expose a capability "resource_scheduling" which enables the `ComputeManager` to decide whether to allow multiple VMs on the same host in the post-scheduling check. We also update the DRS rules when spawning/resizing/migrating a VM. "soft-affinity" is excluded from DRS rule creation and thus only works on cluster-level not on HV-level. The reason is, that DRS doesn't know about "soft" and will require the VMs to run on the same ESXi. Having the VMs run inside the same rack should be good-enough though. For managing DRS settings, we introduce a new module `cluster_util` to the `vmwareapi` driver. Change-Id: Iba9706b7353575fd3f5df5e903e3f254c5e49d73 Co-Authored-By: Johannes Kulik Co-Authored-By: Fabian Wiesel Incorporated-Change-Id: I6d44910c420de4c76b6112904ccfebe3ec923098 (cherry picked from commit d1b19daa461a724c6dd702effc2e1b64610d279b) --- nova/compute/manager.py | 5 +- .../unit/virt/vmwareapi/test_configdrive.py | 2 + .../unit/virt/vmwareapi/test_driver_api.py | 30 ++-- nova/tests/unit/virt/vmwareapi/test_vmops.py | 32 +++- nova/virt/driver.py | 1 + nova/virt/libvirt/driver.py | 1 + nova/virt/vmwareapi/cluster_util.py | 155 ++++++++++++++++++ nova/virt/vmwareapi/driver.py | 5 +- nova/virt/vmwareapi/vm_util.py | 8 + nova/virt/vmwareapi/vmops.py | 37 ++++- 10 files changed, 259 insertions(+), 17 deletions(-) create mode 100644 nova/virt/vmwareapi/cluster_util.py diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 60cbdee3931..eccbc2fbfaf 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1949,7 +1949,10 @@ def _do_validation(context, instance, group): max_server = rules['max_server_per_host'] else: max_server = 1 - if len(members_on_host) >= max_server: + resource_scheduling = self.driver.capabilities.get( + "resource_scheduling", False) + if (len(members_on_host) >= max_server and + not resource_scheduling): raise exception.GroupAffinityViolation( instance_uuid=instance.uuid, policy='Anti-affinity') diff --git a/nova/tests/unit/virt/vmwareapi/test_configdrive.py b/nova/tests/unit/virt/vmwareapi/test_configdrive.py index 7e8b1c1b63c..7b1b4142185 100644 --- a/nova/tests/unit/virt/vmwareapi/test_configdrive.py +++ b/nova/tests/unit/virt/vmwareapi/test_configdrive.py @@ -123,9 +123,11 @@ def tearDown(self): super(ConfigDriveTestCase, self).tearDown() vmwareapi_fake.cleanup() + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata', return_value='fake_metadata') def _spawn_vm(self, fake_get_instance_meta, + mock_update_cluster_placement, injected_files=None, admin_password=None, block_device_info=None): diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 7f8dec8a619..797fd1d7fe5 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -362,10 +362,14 @@ def _create_vm(self, node=None, num_instances=1, uuid=None, ephemeral=ephemeral, flavor_updates=flavor_updates) self.assertIsNone(vm_util.vm_ref_cache_get(self.uuid)) - self.conn.spawn(self.context, self.instance, self.image, - injected_files=[], admin_password=None, allocations={}, - network_info=self.network_info, - block_device_info=bdi) + with test.nested( + mock.patch.object(self.conn._vmops, 'update_cluster_placement', + return_value=None), + ): + self.conn.spawn(self.context, self.instance, self.image, + injected_files=[], admin_password=None, + allocations={}, network_info=self.network_info, + block_device_info=bdi) self._check_vm_record(num_instances=num_instances, powered_on=powered_on, uuid=uuid) @@ -543,8 +547,7 @@ def fake_attach_cdrom(vm_ref, instance, data_store_ref, @mock.patch.object(nova.virt.vmwareapi.images.VMwareImage, 'from_image') - def test_iso_disk_cdrom_attach_with_config_drive(self, - mock_from_image): + def test_iso_disk_cdrom_attach_with_config_drive(self, mock_from_image): img_props = images.VMwareImage( image_id=self.fake_image_uuid, file_size=80 * units.Gi, @@ -655,8 +658,7 @@ def test_destroy_with_attached_volumes(self, side_effect=vexc.ManagedObjectNotFoundException()) @mock.patch.object(vmops.VMwareVMOps, 'destroy') def test_destroy_with_attached_volumes_missing(self, - mock_destroy, - mock_power_off): + mock_destroy, mock_power_off): self._create_vm() connection_info = {'data': 'fake-data', 'serial': 'volume-fake-id'} bdm = [{'connection_info': connection_info, @@ -674,7 +676,7 @@ def test_destroy_with_attached_volumes_missing(self, side_effect=exception.NovaException()) @mock.patch.object(vmops.VMwareVMOps, 'destroy') def test_destroy_with_attached_volumes_with_exception( - self, mock_destroy, mock_detach_volume): + self, mock_destroy, mock_detach_volume): self._create_vm() connection_info = {'data': 'fake-data', 'serial': 'volume-fake-id'} bdm = [{'connection_info': connection_info, @@ -693,7 +695,7 @@ def test_destroy_with_attached_volumes_with_exception( side_effect=exception.DiskNotFound(message='oh man')) @mock.patch.object(vmops.VMwareVMOps, 'destroy') def test_destroy_with_attached_volumes_with_disk_not_found( - self, mock_destroy, mock_detach_volume): + self, mock_destroy, mock_detach_volume): self._create_vm() connection_info = {'data': 'fake-data', 'serial': 'volume-fake-id'} bdm = [{'connection_info': connection_info, @@ -1096,6 +1098,8 @@ def fake_call_method(module, method, *args, **kwargs): self._check_vm_info(info, power_state.RUNNING) self.assertTrue(self.exception) + @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + 'update_cluster_placement') @mock.patch.object(vm_util, 'relocate_vm') @mock.patch('nova.virt.vmwareapi.volumeops.VMwareVolumeOps.' 'attach_volume') @@ -1108,6 +1112,7 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, mock_get_res_pool_of_vm, mock_attach_volume, mock_relocate_vm, + mock_update_cluster_placement, set_image_ref=True): self._create_instance(set_image_ref=set_image_ref) @@ -1128,6 +1133,8 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, mock_attach_volume.assert_called_once_with(connection_info, self.instance, constants.DEFAULT_ADAPTER_TYPE) + @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + 'update_cluster_placement') @mock.patch('nova.virt.vmwareapi.volumeops.VMwareVolumeOps.' 'attach_volume') @mock.patch('nova.block_device.volume_in_mapping') @@ -1135,7 +1142,8 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, def test_spawn_attach_volume_iscsi(self, mock_info_get_mapping, mock_block_volume_in_mapping, - mock_attach_volume): + mock_attach_volume, + mock_update_cluster_placement): self._create_instance() connection_info = self._test_vmdk_connection_info('iscsi') root_disk = [{'connection_info': connection_info, diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 4842c226230..44bd33dadbe 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -588,16 +588,22 @@ def test_clean_shutdown_no_vmwaretools(self): vmware_tools_status="toolsNotOk", succeeds=False) - def _test_finish_migration(self, power_on=True, resize_instance=False): + def _test_finish_migration(self, power_on=True, resize_instance=False, + migration=None): with test.nested( mock.patch.object(self._vmops, '_resize_create_ephemerals_and_swap'), mock.patch.object(self._vmops, "_update_instance_progress"), mock.patch.object(vm_util, "power_on_instance"), mock.patch.object(vm_util, "get_vm_ref", - return_value='fake-ref') + return_value='fake-ref'), + mock.patch.object(vmops.VMwareVMOps, + "update_cluster_placement") ) as (fake_resize_create_ephemerals_and_swap, - fake_update_instance_progress, fake_power_on, fake_get_vm_ref): + fake_update_instance_progress, fake_power_on, fake_get_vm_ref, + fake_update_cluster_placement): + migration = migration or objects.Migration(dest_compute="nova", + source_compute="nova", uuid=uuids.migration) self._vmops.finish_migration(context=self._context, migration=None, instance=self._instance, @@ -616,6 +622,9 @@ def _test_finish_migration(self, power_on=True, resize_instance=False): else: self.assertFalse(fake_power_on.called) + if migration.dest_compute != migration.source_compute: + fake_update_cluster_placement.assert_called_once_with( + self._context, self._instance) calls = [ mock.call(self._context, self._instance, step=5, total_steps=vmops.RESIZE_TOTAL_STEPS), @@ -1211,8 +1220,10 @@ def test_prepare_for_spawn_invalid_ram(self): @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_if_missing') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.lockutils, 'lock') def test_spawn_mask_block_device_info_password(self, mock_lock, + mock_update_cluster_placement, mock_build_virtual_machine, mock_get_vm_config_info, mock_fetch_image_if_missing, mock_debug, mock_glance): # Very simple test that just ensures block_device_info auth_password @@ -1284,12 +1295,14 @@ def _get_metadata(self, is_image_used=True): @mock.patch( 'nova.virt.vmwareapi.imagecache.ImageCacheManager.enlist_image') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_non_root_block_device(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, enlist_image, fetch_image, use_disk_image, @@ -1328,6 +1341,7 @@ def test_spawn_non_root_block_device(self, from_image, get_vm_config_info.assert_called_once_with(self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, self._get_metadata()) enlist_image.assert_called_once_with(image_info.image_id, @@ -1345,12 +1359,14 @@ def test_spawn_non_root_block_device(self, from_image, return_value='fake_vm_folder') @mock.patch('nova.virt.vmwareapi.vm_util.power_on_instance') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_with_no_image_and_block_devices(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, power_on_instance, create_folders, @@ -1391,6 +1407,7 @@ def test_spawn_with_no_image_and_block_devices(self, from_image, get_vm_config_info.assert_called_once_with(self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, self._get_metadata(is_image_used=False)) volumeops.attach_root_volume.assert_called_once_with( @@ -1407,12 +1424,14 @@ def test_spawn_with_no_image_and_block_devices(self, from_image, return_value='fake_vm_folder') @mock.patch('nova.virt.vmwareapi.vm_util.power_on_instance') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_unsupported_hardware(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, power_on_instance, create_folders): @@ -1444,6 +1463,7 @@ def test_spawn_unsupported_hardware(self, from_image, get_vm_config_info.assert_called_once_with( self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, self._get_metadata(is_image_used=False)) @@ -1638,6 +1658,7 @@ def _verify_spawn_method_calls(self, mock_call_method, extras=None): @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch( 'nova.virt.vmwareapi.vmops.VMwareVMOps._update_vnic_index') @mock.patch( @@ -1674,6 +1695,7 @@ def _test_spawn(self, mock_get_datastore, mock_configure_config_drive, mock_update_vnic_index, + mock_update_cluster_placement, mock_create_folders, block_device_info=None, extra_specs=None, @@ -1918,12 +1940,14 @@ def test_spawn_with_block_device_info_swap(self): @mock.patch( 'nova.virt.vmwareapi.imagecache.ImageCacheManager.enlist_image') @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_vm_config_info') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(images.VMwareImage, 'from_image') def test_spawn_with_ephemerals_and_swap(self, from_image, get_extra_specs, get_vm_config_info, + update_cluster_placement, build_virtual_machine, enlist_image, fetch_image, @@ -1969,6 +1993,7 @@ def test_spawn_with_ephemerals_and_swap(self, from_image, get_vm_config_info.assert_called_once_with(self._instance, image_info, extra_specs) build_virtual_machine.assert_called_once_with(self._instance, + self._context, image_info, vi.dc_info, vi.datastore, [], extra_specs, metadata) enlist_image.assert_called_once_with(image_info.image_id, vi.datastore, vi.dc_info.ref) @@ -2113,6 +2138,7 @@ def test_build_virtual_machine(self, mock_create_folder): extra_specs = vm_util.ExtraSpecs() vm_ref = self._vmops.build_virtual_machine(self._instance, + self._context, image, self._dc_info, self._ds, self.network_info, diff --git a/nova/virt/driver.py b/nova/virt/driver.py index a268d435d11..5d3d54c298d 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -278,6 +278,7 @@ class ComputeDriver(object): "has_imagecache": False, "supports_evacuate": False, "supports_migrate_to_same_host": False, + "resource_scheduling": False, "supports_attach_interface": False, "supports_device_tagging": False, "supports_tagged_attach_interface": False, diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 361522e8e30..692f368ac67 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -441,6 +441,7 @@ def __init__(self, virtapi, read_only=False): self.capabilities = { "has_imagecache": True, + "resource_scheduling": False, "supports_evacuate": True, "supports_migrate_to_same_host": False, "supports_attach_interface": True, diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py new file mode 100644 index 00000000000..bc81e758cc6 --- /dev/null +++ b/nova/virt/vmwareapi/cluster_util.py @@ -0,0 +1,155 @@ +# Copyright (c) 2013 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_log import log as logging +from oslo_vmware import vim_util as vutil + +from nova import exception +from nova.i18n import _ +from nova import utils + +LOG = logging.getLogger(__name__) + + +def reconfigure_cluster(session, cluster, config_spec): + reconfig_task = session._call_method( + session.vim, "ReconfigureComputeResource_Task", + cluster, spec=config_spec, + modify=True) + session.wait_for_task(reconfig_task) + + +def _create_vm_group_spec(client_factory, group_info, vm_refs, + operation="add", group=None): + group = group or client_factory.create('ns0:ClusterVmGroup') + group.name = group_info.uuid + + # On vCenter UI, it is not possible to create VM group without + # VMs attached to it. But, using APIs, it is possible to create + # VM group without VMs attached. Therefore, check for existence + # of vm attribute in the group to avoid exceptions + if hasattr(group, 'vm'): + group.vm += vm_refs + else: + group.vm = vm_refs + + group_spec = client_factory.create('ns0:ClusterGroupSpec') + group_spec.operation = operation + group_spec.info = group + return [group_spec] + + +def _get_vm_group(cluster_config, group_info): + if not hasattr(cluster_config, 'group'): + return + for group in cluster_config.group: + if group.name == group_info.uuid: + return group + + +@utils.synchronized('vmware-vm-group-policy') +def update_placement(session, cluster, vm_ref, group_infos): + """Updates cluster for vm placement using DRS""" + cluster_config = session._call_method( + vutil, "get_object_property", cluster, "configurationEx") + + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.groupSpec = [] + config_spec.rulesSpec = [] + for group_info in group_infos: + group = _get_vm_group(cluster_config, group_info) + + if not group: + # Creating group + operation = "add" + else: + # VM group exists on the cluster which is assumed to be + # created by VC admin. Add instance to this vm group and let + # the placement policy defined by the VC admin take over + operation = "edit" + group_spec = _create_vm_group_spec( + client_factory, group_info, [vm_ref], operation=operation, + group=group) + config_spec.groupSpec.append(group_spec) + + # If server group policies are defined (by tenants), then + # create/edit affinity/anti-affinity rules on cluster. + # Note that this might be add-on to the existing vm group + # (mentioned above) policy defined by VC admin i.e if VC admin has + # restricted placement of VMs to a specific group of hosts, then + # the server group policy from nova might further restrict to + # individual hosts on a cluster + if group_info.policies: + # VM group does not exist on cluster + policy = group_info.policies[0] + if policy != 'soft-affinity': + rule_name = "%s-%s" % (group_info.uuid, policy) + rule = _get_rule(cluster_config, rule_name) + operation = "edit" if rule else "add" + rules_spec = _create_cluster_rules_spec( + client_factory, rule_name, [vm_ref], policy=policy, + operation=operation, rule=rule) + config_spec.rulesSpec.append(rules_spec) + + reconfigure_cluster(session, cluster, config_spec) + + +def _create_cluster_rules_spec(client_factory, name, vm_refs, + policy='affinity', operation="add", + rule=None): + + rules_spec = client_factory.create('ns0:ClusterRuleSpec') + rules_spec.operation = operation + if policy == 'affinity' or policy == 'soft-affinity': + policy_class = 'ns0:ClusterAffinityRuleSpec' + elif policy == 'anti-affinity' or policy == 'soft-anti-affinity': + policy_class = 'ns0:ClusterAntiAffinityRuleSpec' + else: + msg = _('%s policy is not supported.') % policy + raise exception.Invalid(msg) + + rules_info = client_factory.create(policy_class) + rules_info.name = name + rules_info.enabled = True + rules_info.mandatory = True + if operation == "edit": + rules_info.vm = rule.vm + vm_refs + rules_info.key = rule.key + rules_info.ruleUuid = rule.ruleUuid + else: + rules_info.vm = vm_refs + + rules_spec.info = rules_info + return rules_spec + + +def _get_rule(cluster_config, rule_name): + if not hasattr(cluster_config, 'rule'): + return + for rule in cluster_config.rule: + if rule.name == rule_name: + return rule + + +def is_drs_enabled(session, cluster): + """Check if DRS is enabled on a given cluster""" + drs_config = session._call_method(vutil, "get_object_property", cluster, + "configuration.drsConfig") + if drs_config and hasattr(drs_config, 'enabled'): + return drs_config.enabled + + return False diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 69b38f576d8..7cf983d1384 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -44,6 +44,7 @@ from nova import objects import nova.privsep.path from nova.virt import driver +from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util @@ -69,6 +70,7 @@ class VMwareVCDriver(driver.ComputeDriver): "has_imagecache": True, "supports_evacuate": False, "supports_migrate_to_same_host": True, + "resource_scheduling": True, "supports_attach_interface": True, "supports_multiattach": False, "supports_trusted_certs": False, @@ -150,7 +152,8 @@ def __init__(self, virtapi, scheme="https"): self._nodename, self._cluster_ref, self._datastore_regex) - + self.capabilities['resource_scheduling'] = \ + cluster_util.is_drs_enabled(self._session, self._cluster_ref) # Register the OpenStack extension self._register_openstack_extension() diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index a11da9cffb0..f98a931bb42 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -34,6 +34,7 @@ from nova import exception from nova.i18n import _ from nova.network import model as network_model +from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import session from nova.virt.vmwareapi import vim_util @@ -1247,6 +1248,13 @@ def get_stats_from_cluster(session, cluster): return stats +def update_cluster_placement(session, instance, cluster, server_group_infos): + if not server_group_infos: + return + vm_ref = get_vm_ref(session, instance) + cluster_util.update_placement(session, cluster, vm_ref, server_group_infos) + + def get_host_ref(session, cluster=None): """Get reference to a host within the cluster specified.""" if cluster is None: diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 09b920fde73..b62b978ceab 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -70,6 +70,9 @@ RESIZE_TOTAL_STEPS = 6 +GroupInfo = collections.namedtuple('GroupInfo', ['uuid', 'policies']) + + class VirtualMachineInstanceConfigInfo(object): """Parameters needed to create and configure a new instance.""" @@ -269,7 +272,7 @@ def _get_folder_name(self, name, id_): # We cannot truncate the 'id' as this is unique across OpenStack. return '%s (%s)' % (name[:40], id_[:36]) - def build_virtual_machine(self, instance, image_info, + def build_virtual_machine(self, instance, context, image_info, dc_info, datastore, network_info, extra_specs, metadata): vif_infos = vmwarevif.get_vif_info(self._session, @@ -302,6 +305,7 @@ def build_virtual_machine(self, instance, image_info, # Create the VM vm_ref = vm_util.create_vm(self._session, instance, folder, config_spec, self._root_resource_pool) + return vm_ref def _get_extra_specs(self, flavor, image_meta=None): @@ -737,6 +741,12 @@ def prepare_for_spawn(self, instance): raise exception.InstanceUnacceptable(instance_id=instance.uuid, reason=reason) + def update_cluster_placement(self, context, instance): + server_group_infos = self._get_server_groups( + context, instance, include_provider_groups=True) + vm_util.update_cluster_placement(self._session, instance, + self._cluster, server_group_infos) + def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info, block_device_info=None): @@ -753,6 +763,7 @@ def spawn(self, context, instance, image_meta, injected_files, # Creates the virtual machine. The virtual machine reference returned # is unique within Virtual Center. vm_ref = self.build_virtual_machine(instance, + context, image_info, vi.dc_info, vi.datastore, @@ -764,6 +775,8 @@ def spawn(self, context, instance, image_meta, injected_files, # instance uuid. vm_util.vm_ref_cache_update(instance.uuid, vm_ref) + self.update_cluster_placement(context, instance) + # Update the Neutron VNIC index self._update_vnic_index(context, instance, network_info) @@ -1068,6 +1081,28 @@ def reboot(self, instance, network_info, reboot_type="SOFT"): self._session._wait_for_task(reset_task) LOG.debug("Did hard reboot of VM", instance=instance) + def _get_server_groups(self, context, instance, + include_provider_groups=False): + server_group_infos = [] + try: + instance_group_object = objects.instance_group.InstanceGroup + server_group = instance_group_object.get_by_instance_uuid( + context, instance.uuid) + if server_group: + server_group_infos.append(GroupInfo(server_group.uuid, + server_group.policies)) + except nova.exception.InstanceGroupNotFound: + pass + + if include_provider_groups: + needs_empty_host = utils.vm_needs_special_spawning( + int(instance.memory_mb), instance.flavor) + if CONF.vmware.special_spawning_vm_group and not needs_empty_host: + name = CONF.vmware.special_spawning_vm_group + server_group_infos.append(GroupInfo(name, None)) + + return server_group_infos + def _destroy_instance(self, instance, destroy_disks=True): # Destroy a VM instance try: From 8b0a6efddbc35d4040151fdfbf30be7d8343fd55 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 27 Aug 2021 11:11:54 +0200 Subject: [PATCH 04/92] vmware: Restructure vm- and host-group functions in cluster_util We still have to old `_create_vm_group_spec()` to get the same behavior, but nobody outside of `cluster_util` uses it. The new functions `create_vm_group()`, `create_host_group()` and `create_group_spec()` expose a clearer interface and make it possible to overwrite hosts/vms instead of just appending to existing ones. This will be useful if customers can upgrade server-groups via API. Change-Id: I5444318994ac7929a24d357fabb8133410d5bd9d (cherry picked from commit 27a9012658e9ae777a8f9060d29c35222bb18dde) (cherry picked from commit d945c77ecc8757d15db6eba6218465ce50443a29) --- nova/virt/vmwareapi/cluster_util.py | 79 ++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index bc81e758cc6..ce33643a1df 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -32,24 +32,64 @@ def reconfigure_cluster(session, cluster, config_spec): session.wait_for_task(reconfig_task) -def _create_vm_group_spec(client_factory, group_info, vm_refs, - operation="add", group=None): +def create_vm_group(client_factory, name, vm_refs, group=None): + """Create a ClusterVmGroup object + + :param:group: if given, update this ClusterVmGroup object instead of + creating a new one + """ group = group or client_factory.create('ns0:ClusterVmGroup') - group.name = group_info.uuid - - # On vCenter UI, it is not possible to create VM group without - # VMs attached to it. But, using APIs, it is possible to create - # VM group without VMs attached. Therefore, check for existence - # of vm attribute in the group to avoid exceptions - if hasattr(group, 'vm'): - group.vm += vm_refs + group.name = name + group.vm = vm_refs + + return group + + +def create_host_group(client_factory, name, host_refs, group=None): + """Create a ClusterHostGroup object + + :param:group: if given, update the ClusterHostGroup object instead of + creating a new one + """ + group = group or client_factory.create('ns0:ClusterHostGroup') + group.name = name + + if hasattr(group, 'host'): + group.host += host_refs else: - group.vm = vm_refs + group.host = host_refs + + return group + + +def create_group_spec(client_factory, group, operation): + """Create a ClusterGroupSpec object""" + if operation not in ('add', 'edit', 'remove'): + msg = 'Invalid operation for ClusterGroupSpec: {}'.format(operation) + raise exception.ValidationError(msg) group_spec = client_factory.create('ns0:ClusterGroupSpec') group_spec.operation = operation group_spec.info = group - return [group_spec] + if operation == 'remove': + group_spec.removeKey = group.name + + return group_spec + + +def _create_vm_group_spec(client_factory, group_info, vm_refs, + operation="add", group=None): + if group: + # On vCenter UI, it is not possible to create VM group without + # VMs attached to it. But, using APIs, it is possible to create + # VM group without VMs attached. Therefore, check for existence + # of vm attribute in the group to avoid exceptions + if hasattr(group, 'vm'): + vm_refs = vm_refs + group.vm + + group = create_vm_group(client_factory, group_info.uuid, vm_refs, group) + + return create_group_spec(client_factory, group, operation) def _get_vm_group(cluster_config, group_info): @@ -60,6 +100,21 @@ def _get_vm_group(cluster_config, group_info): return group +def delete_vm_group(session, cluster, vm_group): + """Add delete impl for removing group if deleted vm is the + last vm in a vm group + """ + client_factory = session.vim.client.factory + groups = [] + + group_spec = create_group_spec(client_factory, vm_group, "remove") + groups.append(group_spec) + + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.groupSpec = groups + reconfigure_cluster(session, cluster, config_spec) + + @utils.synchronized('vmware-vm-group-policy') def update_placement(session, cluster, vm_ref, group_infos): """Updates cluster for vm placement using DRS""" From 4ad5ad336be509e8f54546d8c0bf67b4aa0d4de5 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 27 Aug 2021 11:54:11 +0200 Subject: [PATCH 05/92] vmware: Refactor VM rule creation in cluster_util This splits up functionality for creating rules between VMs into `create_vm_rule()` and `create_rule_spec()` and thus enables us to use it more controlled also from the outside, e.g. in the upcoming sync loop for server-groups. Since DRS ignores the "mandatory" attribute for VM-VM rules, we remove setting it here, so it doesn't look like changing the value would make a difference. Change-Id: Ib515e02226e674d0f7cbdc3c354ade5cd77a0b8c (cherry picked from commit 9386f11f486dc3e7fc8c304b719a2a26dfdc8547) (cherry picked from commit 03232abeacf020e54b6bfeb68018e52a4c1be648) --- nova/virt/vmwareapi/cluster_util.py | 75 +++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index ce33643a1df..4264e47ef05 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -163,33 +163,76 @@ def update_placement(session, cluster, vm_ref, group_infos): reconfigure_cluster(session, cluster, config_spec) +def create_vm_rule(client_factory, name, vm_refs, policy='affinity', + rule=None): + """Create a ClusterAffinityRuleSpec or ClusterAntiAffinityRuleSpec object + + :param:policy: Defines with of the object types is created. "affinity" and + "soft-affinity" map to ClusterAffinityRuleSpec, + "anti-affinity" and "soft-anti-affinity" map to + ClusterAntiAffinityRuleSpec. Ignored if "rule" is given + :param:rule: if given, don't create any object, but instead update the + given object's attributes. "policy" is ignored here. + """ + if rule is None: + if policy in ('affinity', 'soft-affinity'): + obj_type = 'ns0:ClusterAffinityRuleSpec' + elif policy in ('anti-affinity', 'soft-anti-affinity'): + obj_type = 'ns0:ClusterAntiAffinityRuleSpec' + else: + msg = 'Policy {} is not supported.'.format(policy) + raise exception.ValidationError(msg) + + rule = client_factory.create(obj_type) + + rule.name = name + rule.enabled = True + rule.vm = vm_refs + + return rule + + +def create_rule_spec(client_factory, rule, operation='add'): + """Create a ClusterRuleSpec object""" + rule_spec = client_factory.create('ns0:ClusterRuleSpec') + rule_spec.operation = operation + rule_spec.info = rule + if operation == 'remove': + rule_spec.removeKey = rule.key + return rule_spec + + def _create_cluster_rules_spec(client_factory, name, vm_refs, policy='affinity', operation="add", rule=None): + if operation == 'edit': + vm_refs = vm_refs + rule.vm + rule = create_vm_rule(client_factory, name, vm_refs, policy, rule) + return create_rule_spec(client_factory, rule, operation) - rules_spec = client_factory.create('ns0:ClusterRuleSpec') - rules_spec.operation = operation - if policy == 'affinity' or policy == 'soft-affinity': - policy_class = 'ns0:ClusterAffinityRuleSpec' - elif policy == 'anti-affinity' or policy == 'soft-anti-affinity': - policy_class = 'ns0:ClusterAntiAffinityRuleSpec' - else: - msg = _('%s policy is not supported.') % policy - raise exception.Invalid(msg) - rules_info = client_factory.create(policy_class) +def _create_cluster_group_rules_spec(client_factory, name, vm_group_name, + host_group_name, policy='affinity', + rule=None): + rules_info = client_factory.create('ns0:ClusterVmHostRuleInfo') rules_info.name = name rules_info.enabled = True rules_info.mandatory = True - if operation == "edit": - rules_info.vm = rule.vm + vm_refs + rules_info.vmGroupName = vm_group_name + if policy == 'affinity': + rules_info.affineHostGroupName = host_group_name + elif policy == 'anti-affinity': + rules_info.antiAffineHostGroupName = host_group_name + else: + msg = _('%s policy is not supported.') % policy + raise exception.ValidationError(msg) + + if rule is not None: rules_info.key = rule.key rules_info.ruleUuid = rule.ruleUuid - else: - rules_info.vm = vm_refs - rules_spec.info = rules_info - return rules_spec + operation = 'add' if rule is None else 'edit' + return create_rule_spec(client_factory, rules_info, operation) def _get_rule(cluster_config, rule_name): From 7c93acc3d8736e4e6c39554c2f8912126173ea8d Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 15 Jul 2021 11:31:56 +0200 Subject: [PATCH 06/92] vmware: Add function to retrieve all DRS groups of a cluster We will need this in the sync-loop after customers are enabled to change server-group memebers via API. Change-Id: I5e7f27251b6f2c09002445d6c374f887864ea19f (cherry picked from commit 3f27febf7f46ed9311d61a41bacbc9a1f2eceb12) (cherry picked from commit aa9a0be571d7acb683d810680d2f9cdfba1db60e) --- nova/virt/vmwareapi/cluster_util.py | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 4264e47ef05..31796a1a7cf 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -100,6 +100,90 @@ def _get_vm_group(cluster_config, group_info): return group +def fetch_cluster_properties(session, vm_ref): + max_objects = 1 + vim = session.vim + property_collector = vim.service_content.propertyCollector + client_factory = vim.client.factory + + traversal_spec = vutil.build_traversal_spec( + client_factory, + "v_to_r", + "VirtualMachine", + "resourcePool", + False, + [vutil.build_traversal_spec(client_factory, + "r_to_c", + "ResourcePool", + "parent", + False, + [])]) + + object_spec = vutil.build_object_spec( + client_factory, + vm_ref, + [traversal_spec]) + property_spec = vutil.build_property_spec( + client_factory, + "ClusterComputeResource", + ["configurationEx"]) + + property_filter_spec = vutil.build_property_filter_spec( + client_factory, + [property_spec], + [object_spec]) + options = client_factory.create('ns0:RetrieveOptions') + options.maxObjects = max_objects + + pc_result = vim.RetrievePropertiesEx(property_collector, + specSet=[property_filter_spec], options=options) + result = None + """ Retrieving needed hardware properties from ESX hosts """ + with vutil.WithRetrieval(vim, pc_result) as pc_objects: + for objContent in pc_objects: + LOG.debug("Retrieving cluster: %s", objContent) + result = objContent + break + + return result + + +def fetch_cluster_groups(session, cluster_ref=None, cluster_config=None, + group_type=None): + """Fetch all groups of a cluster + + The cluster can be identified by a cluster_ref or by an explicit + cluster_config. If identified by cluster_ref, we fetch the cluster_config. + + If the caller only needs either HostGroup or VmGroup, group_type can be set + to 'host' or 'vm' respectively. + """ + if group_type not in (None, 'vm', 'host'): + msg = 'Invalid group_type {}'.format(group_type) + raise exception.ValidationError(msg) + + if (cluster_config, cluster_ref) == (None, None): + msg = 'Either cluster_config or cluster_ref must be given.' + raise exception.ValidationError(msg) + + if cluster_config is None: + cluster_config = session._call_method( + vutil, "get_object_property", cluster_ref, "configurationEx") + + groups = {} + for group in getattr(cluster_config, 'group', []): + if group_type == 'vm': + if not vutil.is_vim_instance(group, 'ClusterVmGroup'): + continue + elif group_type == 'host': + if not vutil.is_vim_instance(group, 'ClusterHostGroup'): + continue + + groups[group.name] = group + + return groups + + def delete_vm_group(session, cluster, vm_group): """Add delete impl for removing group if deleted vm is the last vm in a vm group From 5d00466cf91358970aa02e576f993a56235f8d76 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 26 Jul 2021 14:39:28 +0200 Subject: [PATCH 07/92] vmware: Add function to retrieve all DRS rules of a cluster We will need this in the sync-loop after customers are enabled to change server-group members via API. Change-Id: Id2fbaa67b799e370331472ecccc79d99dd07e01f Incorporated-Change-Id: I443338bf97dcf83478e0a9971179480ecb01c009 (cherry picked from commit eafa794f891d0312b30389e2a95cca87564fc880) (cherry picked from commit e219cf2e53ffecedf93e6686a1bedc608e9e8923) --- nova/virt/vmwareapi/cluster_util.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 31796a1a7cf..9dd3651f42b 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -184,6 +184,23 @@ def fetch_cluster_groups(session, cluster_ref=None, cluster_config=None, return groups +def fetch_cluster_rules(session, cluster_ref=None, cluster_config=None): + """Fetch all DRS rules of a cluster + + The cluster can be identified by a cluster_ref or by an explicit + cluster_config. If identified by cluster_ref, we fetch the cluster_config. + """ + if (cluster_config, cluster_ref) == (None, None): + msg = 'Either cluster_config or cluster_ref must be given.' + raise exception.ValidationError(msg) + + if cluster_config is None: + cluster_config = session._call_method( + vutil, "get_object_property", cluster_ref, "configurationEx") + + return {r.name: r for r in getattr(cluster_config, 'rule', [])} + + def delete_vm_group(session, cluster, vm_group): """Add delete impl for removing group if deleted vm is the last vm in a vm group From 831f9735cc66d8b0838374dbaefa28f156593796 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 17 Sep 2021 16:30:21 +0200 Subject: [PATCH 08/92] vmware: Set _compute_host on VMwareVMOps We need to know the compute host to query the instances from the API for sync_server_group(), so we save it onto the VMwareVMOps instance when the VMwareVCDriver instance receives it in the init_host() call. Change-Id: I473539000dde4629e2a251cd9145c8047ce60a41 (cherry picked from commit 22ab4cbec83de85c116c2dc9979c1b71fae5f4af) (cherry picked from commit e925801b03bab392cd7f99e16379ffecd5570196) --- nova/virt/vmwareapi/driver.py | 2 ++ nova/virt/vmwareapi/vmops.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 7cf983d1384..84578d0d97c 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -209,6 +209,8 @@ def init_host(self, host): if vim is None: self._session._create_session() + self._vmops.set_compute_host(host) + def cleanup_host(self, host): self._session.logout() diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index b62b978ceab..0af284660a8 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -151,6 +151,9 @@ def __init__(self, session, virtapi, volumeops, cluster=None, self._base_folder) self._network_api = neutron.API() + # set when our driver's init_host() is called + self._compute_host = None + def _get_base_folder(self): # Enable more than one compute node to run on the same host if CONF.vmware.cache_prefix: @@ -2209,3 +2212,7 @@ def get_mks_console(self, instance): 'thumbprint': thumbprint} internal_access_path = jsonutils.dumps(mks_auth) return ctype.ConsoleMKS(ticket.host, ticket.port, internal_access_path) + + def set_compute_host(self, compute_host): + """Called by the driver on init_host() so we know the compute host""" + self._compute_host = compute_host From 212d54b7dd36ffdd4b75b183cf137c81206925de Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 15 Jul 2021 14:11:54 +0200 Subject: [PATCH 09/92] vmware: Use a prefix in DRS groups and rules We want to create a sync-loop for applying/removing server-groups changed by the user via API. For this we need to be able to distinguish between driver-created DRS groups/rules and admin-created ones. To do this, we introduce a prefix for the DRS group/rule name, which will work as an identifier later on. Since we're now using not only a UUID, but a UUID with a prefix, we change GroupInfo to have a "name" attribute instead of a "uuid" attribute. As we're changing how DRS groups/rules look, we need a migration to run before deploying this to production. Change-Id: I07ecd1953a85d0f53082fa9b0c49b80c2c9bf9d3 (cherry picked from commit 7984caebd9306b71eddab1009533e3cf60f1c25e) (cherry picked from commit 1e5e590dd4203a7e8fc3058e9048c5310eaf84fc) --- nova/virt/vmwareapi/cluster_util.py | 6 +++--- nova/virt/vmwareapi/constants.py | 6 ++++++ nova/virt/vmwareapi/vmops.py | 5 +++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 9dd3651f42b..8bb229c4d2d 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -87,7 +87,7 @@ def _create_vm_group_spec(client_factory, group_info, vm_refs, if hasattr(group, 'vm'): vm_refs = vm_refs + group.vm - group = create_vm_group(client_factory, group_info.uuid, vm_refs, group) + group = create_vm_group(client_factory, group_info.name, vm_refs, group) return create_group_spec(client_factory, group, operation) @@ -96,7 +96,7 @@ def _get_vm_group(cluster_config, group_info): if not hasattr(cluster_config, 'group'): return for group in cluster_config.group: - if group.name == group_info.uuid: + if group.name == group_info.name: return group @@ -253,7 +253,7 @@ def update_placement(session, cluster, vm_ref, group_infos): # VM group does not exist on cluster policy = group_info.policies[0] if policy != 'soft-affinity': - rule_name = "%s-%s" % (group_info.uuid, policy) + rule_name = "%s-%s" % (group_info.name, policy) rule = _get_rule(cluster_config, rule_name) operation = "edit" if rule else "add" rules_spec = _create_cluster_rules_spec( diff --git a/nova/virt/vmwareapi/constants.py b/nova/virt/vmwareapi/constants.py index 5e74d879d48..11a74cd4a3b 100644 --- a/nova/virt/vmwareapi/constants.py +++ b/nova/virt/vmwareapi/constants.py @@ -234,3 +234,9 @@ POWER_STATES = {'poweredOff': power_state.SHUTDOWN, 'poweredOn': power_state.RUNNING, 'suspended': power_state.SUSPENDED} + + +# Prefix used in the name of DRS groups and rules created by the driver to +# distinguish between what has been automatically created and what is +# admin-created. +DRS_PREFIX = 'NOVA_' diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 0af284660a8..4181f2dd412 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -70,7 +70,7 @@ RESIZE_TOTAL_STEPS = 6 -GroupInfo = collections.namedtuple('GroupInfo', ['uuid', 'policies']) +GroupInfo = collections.namedtuple('GroupInfo', ['name', 'policies']) class VirtualMachineInstanceConfigInfo(object): @@ -1092,7 +1092,8 @@ def _get_server_groups(self, context, instance, server_group = instance_group_object.get_by_instance_uuid( context, instance.uuid) if server_group: - server_group_infos.append(GroupInfo(server_group.uuid, + name = '{}{}'.format(constants.DRS_PREFIX, server_group.uuid) + server_group_infos.append(GroupInfo(name, server_group.policies)) except nova.exception.InstanceGroupNotFound: pass From 30348b1a624a57c1a2ac1cb5e1c06e4e2c9f7c5a Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 14:15:06 +0200 Subject: [PATCH 10/92] vmware: Remove no longer used "fetch_cluster_properties()" It was previously used when deleting empty DRS groups. Change-Id: I82612decf3938285c43f5e97abb48da349ad3fba (cherry picked from commit 8e0836d80a07ad89e690744edebaa71916755c0b) (cherry picked from commit 63ec87b459a898665bd2afeb4604254f47c4991b) --- nova/virt/vmwareapi/cluster_util.py | 48 ----------------------------- 1 file changed, 48 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 8bb229c4d2d..3b7f8acd69d 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -100,54 +100,6 @@ def _get_vm_group(cluster_config, group_info): return group -def fetch_cluster_properties(session, vm_ref): - max_objects = 1 - vim = session.vim - property_collector = vim.service_content.propertyCollector - client_factory = vim.client.factory - - traversal_spec = vutil.build_traversal_spec( - client_factory, - "v_to_r", - "VirtualMachine", - "resourcePool", - False, - [vutil.build_traversal_spec(client_factory, - "r_to_c", - "ResourcePool", - "parent", - False, - [])]) - - object_spec = vutil.build_object_spec( - client_factory, - vm_ref, - [traversal_spec]) - property_spec = vutil.build_property_spec( - client_factory, - "ClusterComputeResource", - ["configurationEx"]) - - property_filter_spec = vutil.build_property_filter_spec( - client_factory, - [property_spec], - [object_spec]) - options = client_factory.create('ns0:RetrieveOptions') - options.maxObjects = max_objects - - pc_result = vim.RetrievePropertiesEx(property_collector, - specSet=[property_filter_spec], options=options) - result = None - """ Retrieving needed hardware properties from ESX hosts """ - with vutil.WithRetrieval(vim, pc_result) as pc_objects: - for objContent in pc_objects: - LOG.debug("Retrieving cluster: %s", objContent) - result = objContent - break - - return result - - def fetch_cluster_groups(session, cluster_ref=None, cluster_config=None, group_type=None): """Fetch all groups of a cluster From f92090dd9eb97991f6a335cba95399cd5c956827 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Tue, 28 Jan 2025 12:02:09 +0100 Subject: [PATCH 11/92] vmware: Add helper to set DRS overrides This function is able to add/remove DRS overrides for a VM. DRS overrides can be used to set an explicit behaviour for a VM that can differ from the default of the cluster. Change-Id: I9022869ad546e28ec9de66e17d639c3314b19832 (cherry picked from commit e3e44a4ea676bc95aec452944be4aa17638e6f69) --- nova/virt/vmwareapi/cluster_util.py | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 3b7f8acd69d..116b4907988 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -304,3 +304,40 @@ def is_drs_enabled(session, cluster): return drs_config.enabled return False + + +def update_cluster_drs_vm_override(session, cluster, vm_ref, operation='add', + behavior=None, enabled=True): + """Add/Update `ClusterDrsVmConfigSpec` for a VM. + + `behavior` can be any `DrsBehaviour` as string. + + `behavior` and `enabled` are only used if `operation` is `add`. + """ + if operation not in ('add', 'remove'): + msg = _('%s operation for ClusterDrsVmConfigSpec not supported.') + raise exception.ValidationError(msg % operation) + + client_factory = session.vim.client.factory + + drs_vm_spec = client_factory.create('ns0:ClusterDrsVmConfigSpec') + drs_vm_spec.operation = operation + + if operation == 'add': + if behavior is None: + msg = _('behavior cannot be unset for operation "add"') + raise exception.ValidationError(msg) + drs_vm_info = client_factory.create('ns0:ClusterDrsVmConfigInfo') + drs_vm_info.behavior = behavior + drs_vm_info.enabled = enabled + drs_vm_info.key = vm_ref + + drs_vm_spec.info = drs_vm_info + + elif operation == 'remove': + drs_vm_spec.removeKey = vm_ref + + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.drsVmConfigSpec = [drs_vm_spec] + + reconfigure_cluster(session, cluster, config_spec) From 0b76db8fed0479b5b77a711793f87797aa615997 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 14:19:31 +0200 Subject: [PATCH 12/92] vmware: Don't add VmGroups for server-groups The DRS rules created by Nova do not use those VmGroups and they don't seem to server a purpose otherwise. Therfore, we only update/create a VmGroup that's not based on a server-group, i.e. an admin-defined group. We currently only have this kind of group for the special_spawning case to support a free host for big VMs. Change-Id: Ide011e157ad46037304cfdb52b1db397dde38cc8 (cherry picked from commit 8c3640b283f1f8fe42ab8b97df8b6072a7fb57fe) (cherry picked from commit 0f0df087e3dd5e56f95dab584def3c71d40bac58) --- nova/virt/vmwareapi/cluster_util.py | 32 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 116b4907988..1b3519679f0 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -20,6 +20,7 @@ from nova import exception from nova.i18n import _ from nova import utils +from nova.virt.vmwareapi import constants LOG = logging.getLogger(__name__) @@ -179,20 +180,23 @@ def update_placement(session, cluster, vm_ref, group_infos): config_spec.groupSpec = [] config_spec.rulesSpec = [] for group_info in group_infos: - group = _get_vm_group(cluster_config, group_info) - - if not group: - # Creating group - operation = "add" - else: - # VM group exists on the cluster which is assumed to be - # created by VC admin. Add instance to this vm group and let - # the placement policy defined by the VC admin take over - operation = "edit" - group_spec = _create_vm_group_spec( - client_factory, group_info, [vm_ref], operation=operation, - group=group) - config_spec.groupSpec.append(group_spec) + if not group_info.name.startswith(constants.DRS_PREFIX): + # We only do this, if this is an admin-defined group, because + # VmGroups are not used by the rules created by Nova. + group = _get_vm_group(cluster_config, group_info) + + if not group: + # Creating group + operation = "add" + else: + # VM group exists on the cluster which is assumed to be + # created by VC admin. Add instance to this vm group and let + # the placement policy defined by the VC admin take over + operation = "edit" + group_spec = _create_vm_group_spec( + client_factory, group_info, [vm_ref], operation=operation, + group=group) + config_spec.groupSpec.append(group_spec) # If server group policies are defined (by tenants), then # create/edit affinity/anti-affinity rules on cluster. From 4fbd161b98d949942fba3796625f82754389bb94 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 16:35:50 +0200 Subject: [PATCH 13/92] vmware: Remove cleaning of empty DRS groups Since we don't create new ones, we also don't have to keep track of empty ones. Instead, we will just delete all the VmGroups once via an external script. Change-Id: I88baeadcea3a7bfe946596b169bc3abe6798d9d6 (cherry picked from commit 89dcef00783eedea76e3fe0b6ba19e2d0fe3aa79) (cherry picked from commit e4b85ba35e8333e8dc46f8ef4d270fc2a7e21324) --- nova/virt/vmwareapi/vmops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 4181f2dd412..52efb42427a 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -1139,6 +1139,7 @@ def _destroy_instance(self, instance, destroy_disks=True): LOG.warning("In vmwareapi:vmops:_destroy_instance, got " "this exception while un-registering the VM: %s", excep, instance=instance) + # Delete the folder holding the VM related content on # the datastore. if destroy_disks and vm_ds_path: From 42a71600bfed1d8abc9428c4f23003226be03d8b Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 15:51:36 +0200 Subject: [PATCH 14/92] vmware: Add DRS rule lifecycle helpers These functions are simple helper functions to create, update and delete a DRS rule, so we have to code necessary for that at a single place. Change-Id: I44aeed6f99b9803adca0062b2d7b12cc2e295f03 Incorporated-Change-Id: I86f7ca85d9b0edc1406a54a6f392bfff8f0af00d (cherry picked from commit 94f8c7a78634e338be2bdfed0ed1ccba88ea9348) --- nova/virt/vmwareapi/cluster_util.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 1b3519679f0..559856db432 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -292,6 +292,21 @@ def _create_cluster_group_rules_spec(client_factory, name, vm_group_name, return create_rule_spec(client_factory, rules_info, operation) +def add_rule(session, cluster_ref, rule): + """Add the given DRS rule to the given cluster""" + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.rulesSpec = [create_rule_spec(client_factory, rule, 'add')] + reconfigure_cluster(session, cluster_ref, config_spec) + + +def get_rule(session, cluster_ref, rule_name): + """Get a DRS rule from the cluster by name""" + cluster_config = session._call_method( + vutil, "get_object_property", cluster_ref, "configurationEx") + return _get_rule(cluster_config, rule_name) + + def _get_rule(cluster_config, rule_name): if not hasattr(cluster_config, 'rule'): return @@ -300,6 +315,35 @@ def _get_rule(cluster_config, rule_name): return rule +def get_rules_by_prefix(session, cluster_ref, rule_prefix): + """Get all DRS rules starting with the given prefix + + Useful, if you don't know the policy of a server-group the rule was created + for. + """ + cluster_config = session._call_method( + vutil, "get_object_property", cluster_ref, "configurationEx") + + return [rule for rule in getattr(cluster_config, 'rule', []) + if rule.name.startswith(rule_prefix)] + + +def delete_rule(session, cluster_ref, rule): + """Delete the given DRS rule from the given cluster""" + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.rulesSpec = [create_rule_spec(client_factory, rule, 'remove')] + reconfigure_cluster(session, cluster_ref, config_spec) + + +def update_rule(session, cluster_ref, rule): + """Update the already modified DRS rule in the cluster""" + client_factory = session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.rulesSpec = [create_rule_spec(client_factory, rule, 'edit')] + reconfigure_cluster(session, cluster_ref, config_spec) + + def is_drs_enabled(session, cluster): """Check if DRS is enabled on a given cluster""" drs_config = session._call_method(vutil, "get_object_property", cluster, From 1097c939bc5bb74b542b535b8a947c86c55354ea Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 15:52:50 +0200 Subject: [PATCH 15/92] vmware: Implement sync_server_group() The VMware driver supports syncing server-groups as DRS rules into the cluster managed by the nova-compute node. The method will be called when a user updates a server-group via API and will later get reused when spawning a VM, too. Instances in certain states need to be excluded from the sync for multiple reasons: 1) when instances are going away during the sync, this can lead to an error 2) VMs contained in a DRS rule cannot be vMotioned - excluding them makes it easier for the live-migration code to update the cluster appropriately We do not sync "soft-affinity" server-groups, mainly because we use "soft-affinity" is our way to give customers the possibility to end up on the same cluster, without the need to end up on the same host, too. We also cannot implement the "soft" part currently and thus spawning a VM will fail if the host is too full, as there are no non-mandatory VM-to-VM rules in VMware DRS. We might be able to optimize it a little more, by keeping a local list of DRS rules instead of querying the cluster in real-time. Tests have shown, that it takes < 500ms to query the cluster, though. Change-Id: I534c035a1e2d962cf5d187d56d104e743f7ade15 Incorporated-Change-Id: I8bc94aebebcd878f60c33fe009f048afeb9a42c0 Incorporated-Change-Id: I745ac6616eefd193ce8c7a9a5cba3c68fc59ac75 (cherry picked from commit 52f42fc120daeb3f3f49cf8568bad50b10e348d4) (cherry picked from commit 70aca2b3719daf7ef689c3f9e092d4d125a1ef89) --- nova/virt/vmwareapi/driver.py | 3 + nova/virt/vmwareapi/vmops.py | 126 ++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 84578d0d97c..9b097b5d95e 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -767,3 +767,6 @@ def attach_interface(self, context, instance, image_meta, vif): def detach_interface(self, context, instance, vif): """Detach an interface from the instance.""" self._vmops.detach_interface(context, instance, vif) + + def sync_server_group(self, context, sg_uuid): + self._vmops.sync_server_group(context, sg_uuid) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 52efb42427a..fd9f4b0b9c6 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -53,6 +53,7 @@ from nova.virt import configdrive from nova.virt import driver from nova.virt import hardware +from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util @@ -2218,3 +2219,128 @@ def get_mks_console(self, instance): def set_compute_host(self, compute_host): """Called by the driver on init_host() so we know the compute host""" self._compute_host = compute_host + + def sync_server_group(self, context, sg_uuid): + # we have to ignore instances currently in a volatitle state, where + # either VMware cannot support them being in a DRS rule or we expect + # them to go away during the syncing process, which could lead to + # errors. Therefore, we explicitly remove those members from the list + # of expected members of a rule, which also removes them in the + # cluster. + STATES_EXCLUDING_MEMBERS_FROM_DRS_RULES = [ + task_states.MIGRATING, + task_states.DELETING, + task_states.SHELVING, + task_states.REBUILDING, + task_states.REBUILD_BLOCK_DEVICE_MAPPING, + ] + + LOG.debug('Starting sync for server-group %s', sg_uuid) + + @utils.synchronized('vmware-server-group-{}'.format(sg_uuid)) + def _sync_sync_server_group(context, sg_uuid): + rule_prefix = '{}{}'.format(constants.DRS_PREFIX, sg_uuid) + + # retrieve the server-group with its members + try: + sg = objects.instance_group.InstanceGroup.get_by_uuid(context, + sg_uuid) + except exception.InstanceGroupNotFound: + LOG.info('Server-group %s cannot be found in DB', sg_uuid) + # check if we have it in the vCenter. if yes, it's an orphan + # and needs deletion + rules = cluster_util.get_rules_by_prefix( + self._session, self._cluster, rule_prefix) + for rule in rules: + LOG.debug('Deleting DRS rule %s as orphan', rule.name) + cluster_util.delete_rule( + self._session, self._cluster, rule) + LOG.info('Deleted rule %s as orphan', rule.name) + LOG.debug('Sync for server-group %s done', sg_uuid) + return + + # retrieve the instances, because sg.members contains all members + # and we need to filter them for our host + InstanceList = objects.instance.InstanceList + filters = {'host': self._compute_host, 'uuid': sg.members, + 'deleted': False} + instances = InstanceList.get_by_filters(context, filters, + expected_attrs=[]) + + expected_members = {} + for instance in instances: + task_state = instance.task_state + if task_state in STATES_EXCLUDING_MEMBERS_FROM_DRS_RULES: + LOG.debug("Excluding member %s of server-group %s, " + "because it's in task_state %s.", + instance.uuid, sg.uuid, task_state) + continue + + try: + moref = vm_util.get_vm_ref(self._session, instance) + except exception.InstanceNotFound: + LOG.warning('Could not find moref for instance %s. ' + 'Ignoring member of server-group %s', + instance.uuid, sg.uuid) + continue + expected_members[instance.uuid] = moref + + rule_name = '{}-{}'.format(rule_prefix, sg.policy) + rule = cluster_util.get_rule( + self._session, self._cluster, rule_name) + + if not rule: + if len(expected_members) < 2 or sg.policy == 'soft-affinity': + LOG.debug('Sync for server-group %s done', sg_uuid) + return + # we have to create a new rule + LOG.debug('Creating missing DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + client_factory = self._session.vim.client.factory + rule = cluster_util.create_vm_rule( + client_factory, rule_name, list(expected_members.values()), + policy=sg.policy) + cluster_util.add_rule( + self._session, self._cluster, rule) + LOG.info('Created missing DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + LOG.debug('Sync for server-group %s done', sg_uuid) + return + + if sg.policy == 'soft-affinity': + LOG.debug('Deleting DRS rule %s with policy soft-affinity', + rule_name) + cluster_util.delete_rule( + self._session, self._cluster, rule) + LOG.info('Deleted DRS rule %s with policy soft-affinity.', + rule_name) + LOG.debug('Sync for server-group %s done', sg_uuid) + return + + if len(expected_members) < 2: + # we have to delete the rule + LOG.debug('Deleting DRS rule %s with < 2 members.', rule_name) + cluster_util.delete_rule( + self._session, self._cluster, rule) + LOG.info('Deleted DRS rule %s with < 2 members.', rule_name) + LOG.debug('Sync for server-group %s done', sg_uuid) + return + + expected_moref_values = set(vutil.get_moref_value(m) + for m in expected_members.values()) + existing_moref_values = set(vutil.get_moref_value(m) + for m in rule.vm) + if expected_moref_values == existing_moref_values: + LOG.debug('Sync for server-group %s done', sg_uuid) + return + + # we have to update the DRS rule to contain the right members + rule.vm = list(expected_members.values()) + LOG.debug('Updating DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + cluster_util.update_rule(self._session, self._cluster, rule) + LOG.info('Updated DRS rule %s with members %s', + rule_name, ', '.join(expected_members)) + LOG.debug('Sync for server-group %s done', sg_uuid) + + _sync_sync_server_group(context, sg_uuid) From 4530a518caa198affa387a991c4d336654e4c4f9 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 9 Jul 2021 15:24:19 +0200 Subject: [PATCH 16/92] objects: Add get_by_instance_uuids() to InstanceGroupList Searching for a list of instance groups belonging to a list of instances can be helpful for checking if a server-group update is valid and maybe if a nova-compute wants to sync down server-groups like the VMware HV is able to. Change-Id: Iec93becf0299ec0617e99ce16c06e37c84cb33ee (cherry picked from commit 838142b24f6c3fad090c1bc9c5b21f2e74559232) (cherry picked from commit 243275e9db9cfc0f5d3ecc9a938ae946ff615375) --- nova/objects/instance_group.py | 24 ++++++++++++++++++++++++ nova/tests/unit/objects/test_objects.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/nova/objects/instance_group.py b/nova/objects/instance_group.py index 8a12a876934..ead6a36f17c 100644 --- a/nova/objects/instance_group.py +++ b/nova/objects/instance_group.py @@ -563,6 +563,23 @@ def _get_counts_from_db(context, project_id, user_id=None): counts['user'] = {'server_groups': query.count()} return counts + @staticmethod + @api_db_api.context_manager.reader + def _get_from_db_by_instance_uuids(context, instance_uuids): + if not instance_uuids: + return [] + + members = set(instance_uuids) + groups = context.session.query(api_models.InstanceGroup).\ + join(api_models.InstanceGroupMember, + api_models.InstanceGroupMember.group_id == + api_models.InstanceGroup.id).\ + filter(api_models.InstanceGroupMember.instance_uuid.in_(members)).\ + options(orm.joinedload(api_models.InstanceGroup._policies)).\ + options(orm.contains_eager(api_models.InstanceGroup._members))\ + .all() + return groups + @base.remotable_classmethod def get_by_project_id(cls, context, project_id): api_db_groups = cls._get_from_db(context, project_id=project_id) @@ -589,3 +606,10 @@ def get_counts(cls, context, project_id, user_id=None): 'user': {'server_groups': }} """ return cls._get_counts_from_db(context, project_id, user_id=user_id) + + @base.remotable_classmethod + def get_by_instance_uuids(cls, context, instance_uuids): + api_db_groups = cls._get_from_db_by_instance_uuids(context, + instance_uuids) + return base.obj_make_list(context, cls(context), objects.InstanceGroup, + api_db_groups) diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index fb3a8491457..6edad3219cd 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1116,7 +1116,7 @@ def obj_name(cls): 'InstanceFault': '1.2-7ef01f16f1084ad1304a513d6d410a38', 'InstanceFaultList': '1.2-6bb72de2872fe49ded5eb937a93f2451', 'InstanceGroup': '1.11-852ac511d30913ee88f3c3a869a8f30a', - 'InstanceGroupList': '1.8-90f8f1a445552bb3bbc9fa1ae7da27d4', + 'InstanceGroupList': '1.8-feac6a7c1f4519e069db17e9e6efcff2', 'InstanceInfoCache': '1.5-cd8b96fefe0fc8d4d337243ba0bf0e1e', 'InstanceList': '2.6-238f125650c25d6d12722340d726f723', 'InstanceMapping': '1.2-3bd375e65c8eb9c45498d2f87b882e03', From 154cddf47228761039b277e3ecb523aa7a7e9d8b Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 14:48:24 +0200 Subject: [PATCH 17/92] Add a way to trigger a server-group sync on the driver We're going to change the API to allow updates of server-group members and need to trigger a sync in the backend, whenever such a change occurs. Therefore, we need a bunch of methods going through the whole stack to call the driver. We use a cast and not a call here, because we cannot let the API wait for all of this to happen. If the cast gets lost somehow, a sync-loop implemented in the driver will pick the change up, eventually The API will have to supply a list of hosts to call, so we only sync the group on the necessary hosts. Change-Id: I00e012ed52ba9fd36b094ecf2dc86b023f2f5a21 (cherry picked from commit f3d2ad7e457ab0e47e63c82f09f91572ba04711f) (cherry picked from commit ae57baaf366c607a26b3bc5059a348bb7453f536) --- nova/compute/api.py | 4 ++++ nova/compute/manager.py | 4 ++++ nova/compute/rpcapi.py | 6 ++++++ nova/virt/driver.py | 7 +++++++ 4 files changed, 21 insertions(+) diff --git a/nova/compute/api.py b/nova/compute/api.py index 6f2f8a3c77b..549fe380d32 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -6098,6 +6098,10 @@ def deny_share(self, context, instance, share_mapping): self.compute_rpcapi.deny_share( context, instance, share_mapping) + def sync_server_group(self, context, hosts, sg_uuid): + for host in hosts: + self.compute_rpcapi.sync_server_group(context, host, sg_uuid) + def target_host_cell(fn): """Target a host-based function to a cell. diff --git a/nova/compute/manager.py b/nova/compute/manager.py index eccbc2fbfaf..425b7342c1d 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -12211,6 +12211,10 @@ def _update_migrate_vifs_profile_with_pci(self, "%(profile)s", {'port_id': port_id, 'profile': profile}) + def sync_server_group(self, context, sg_uuid): + """Calls the driver to update the server-group in the backend""" + self.driver.sync_server_group(context, sg_uuid) + # TODO(sbauza): Remove this proxy class in the X release once we drop the 5.x # support. diff --git a/nova/compute/rpcapi.py b/nova/compute/rpcapi.py index 348797b63d2..c09a9a711e0 100644 --- a/nova/compute/rpcapi.py +++ b/nova/compute/rpcapi.py @@ -1583,3 +1583,9 @@ def cache_images(self, ctxt, host, image_ids): call_monitor_timeout=CONF.rpc_response_timeout, timeout=CONF.long_rpc_timeout) return cctxt.call(ctxt, 'cache_images', image_ids=image_ids) + + def sync_server_group(self, ctxt, host, sg_uuid): + version = '5.0' + client = self.router.client(ctxt) + cctxt = client.prepare(server=host, version=version) + return cctxt.cast(ctxt, "sync_server_group", sg_uuid=sg_uuid) diff --git a/nova/virt/driver.py b/nova/virt/driver.py index 5d3d54c298d..2be76fa3806 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -2011,6 +2011,13 @@ def cleanup_lingering_instance_resources(self, instance): """ return True + def sync_server_group(self, context, sg_uuid): + """Sync that specific server-group into the backend + + If the driver manages multiple HVs in a cluster, this method will be + called if a customer changed a server-group for this host via API. + """ + def load_compute_driver(virtapi, compute_driver=None): """Load a compute driver module. From c1f8258f20558358adc7ec2a53af1d8f68ae2711 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 17 Jun 2021 09:20:51 +0200 Subject: [PATCH 18/92] Add public method to remove members from InstanceGroup This function wasn't previously available, because - as the removed comment says - there was no user-facing API that would allow removal of instance group members. Since we plan on changing the API, we need to add a pulic method. Adding this method, we also need to send notifications like we do for "add_members()" and thus added the appropriate functionality. Change-Id: I4270212b57782e5ffeaf69dc3bd57c7c60a7ffe5 (cherry picked from commit df3bd903524bb8e886e09d1ecf3099de107b6ac6) (cherry picked from commit e7c9a01e8073afb01f14de084fe0094d3ec47ea0) --- nova/compute/utils.py | 15 +++++++++++++++ nova/objects/fields.py | 9 +++++---- nova/objects/instance_group.py | 13 ++++++++++--- nova/rpc.py | 1 + .../notifications/objects/test_notification.py | 2 +- nova/tests/unit/objects/test_objects.py | 2 +- 6 files changed, 33 insertions(+), 9 deletions(-) diff --git a/nova/compute/utils.py b/nova/compute/utils.py index fee66d85c5c..ee25341ee2e 100644 --- a/nova/compute/utils.py +++ b/nova/compute/utils.py @@ -899,6 +899,21 @@ def notify_about_server_group_add_member(context, group_id): notification.emit(context) +@rpc.if_notifications_enabled +def notify_about_server_group_remove_member(context, group_id): + group = objects.InstanceGroup.get_by_uuid(context, group_id) + payload = sg_notification.ServerGroupPayload(group) + notification = sg_notification.ServerGroupNotification( + priority=fields.NotificationPriority.INFO, + publisher=notification_base.NotificationPublisher( + host=CONF.host, source=fields.NotificationSource.API), + event_type=notification_base.EventType( + object='server_group', + action=fields.NotificationAction.REMOVE_MEMBER), + payload=payload) + notification.emit(context) + + @rpc.if_notifications_enabled def notify_about_instance_rebuild(context, instance, host, action=fields.NotificationAction.REBUILD, diff --git a/nova/objects/fields.py b/nova/objects/fields.py index d64234f00c6..2a031074687 100644 --- a/nova/objects/fields.py +++ b/nova/objects/fields.py @@ -965,6 +965,7 @@ class NotificationAction(BaseNovaEnum): ADD_HOST = 'add_host' REMOVE_HOST = 'remove_host' ADD_MEMBER = 'add_member' + REMOVE_MEMBER = 'remove_member' UPDATE_METADATA = 'update_metadata' LOCK = 'lock' UNLOCK = 'unlock' @@ -985,10 +986,10 @@ class NotificationAction(BaseNovaEnum): LIVE_MIGRATION_ROLLBACK_DEST, REBUILD, INTERFACE_DETACH, RESIZE_CONFIRM, RESIZE_PREP, RESIZE_REVERT, SHELVE_OFFLOAD, SOFT_DELETE, TRIGGER_CRASH_DUMP, UNRESCUE, UNSHELVE, ADD_HOST, - REMOVE_HOST, ADD_MEMBER, UPDATE_METADATA, LOCK, UNLOCK, - REBUILD_SCHEDULED, UPDATE_PROP, LIVE_MIGRATION_FORCE_COMPLETE, - CONNECT, USAGE, BUILD_INSTANCES, MIGRATE_SERVER, REBUILD_SERVER, - SELECT_DESTINATIONS, IMAGE_CACHE) + REMOVE_HOST, ADD_MEMBER, REMOVE_MEMBER, UPDATE_METADATA, LOCK, + UNLOCK, REBUILD_SCHEDULED, UPDATE_PROP, + LIVE_MIGRATION_FORCE_COMPLETE, CONNECT, USAGE, BUILD_INSTANCES, + MIGRATE_SERVER, REBUILD_SERVER, SELECT_DESTINATIONS, IMAGE_CACHE) # TODO(rlrossit): These should be changed over to be a StateMachine enum from diff --git a/nova/objects/instance_group.py b/nova/objects/instance_group.py index ead6a36f17c..c9b6c7dc672 100644 --- a/nova/objects/instance_group.py +++ b/nova/objects/instance_group.py @@ -333,9 +333,6 @@ def _add_members_in_db(context, group_uuid, members): @staticmethod @api_db_api.context_manager.writer def _remove_members_in_db(context, group_id, instance_uuids): - # There is no public method provided for removing members because the - # user-facing API doesn't allow removal of instance group members. We - # need to be able to remove members to address quota races. context.session.query(api_models.InstanceGroupMember).\ filter_by(group_id=group_id).\ filter(api_models.InstanceGroupMember.instance_uuid. @@ -490,6 +487,16 @@ def add_members(cls, context, group_uuid, instance_uuids): compute_utils.notify_about_server_group_add_member(context, group_uuid) return list(members) + @base.remotable_classmethod + def remove_members(cls, context, group_id, instance_uuids, group_uuid): + payload = {'server_group_id': group_uuid, + 'instance_uuids': instance_uuids} + cls._remove_members_in_db(context, group_id, instance_uuids) + compute_utils.notify_about_server_group_update(context, + "removemember", payload) + compute_utils.notify_about_server_group_remove_member(context, + group_uuid) + @base.remotable def get_hosts(self, exclude=None): """Get a list of hosts for non-deleted instances in the group diff --git a/nova/rpc.py b/nova/rpc.py index 7a92650414b..25e5c71bcf7 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -384,6 +384,7 @@ class LegacyValidatingNotifier(object): 'scheduler.select_destinations.end', 'scheduler.select_destinations.start', 'servergroup.addmember', + 'servergroup.removemember', 'servergroup.create', 'servergroup.delete', 'volume.usage', diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py index bde7fd3f5ea..d45099ed9a7 100644 --- a/nova/tests/unit/notifications/objects/test_notification.py +++ b/nova/tests/unit/notifications/objects/test_notification.py @@ -375,7 +375,7 @@ def test_payload_is_not_generated_if_notification_format_is_unversioned( 'ComputeTaskNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'ComputeTaskPayload': '1.0-59a9b78b6199470a83c8d07bffd13f5e', 'DestinationPayload': '1.0-d8faf610201bf5f460892243f6632a37', - 'EventType': '1.22-fab37f98c6e7513edece32ad5420f549', + 'EventType': '1.22-23923a7e77f1a188085deb00fe5e850b', 'ExceptionNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'ExceptionPayload': '1.1-34f006107693b8c9eaf4b104157d21b4', 'FlavorNotification': '1.0-a73147b93b520ff0061865849d3dfa56', diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 6edad3219cd..c80981a7924 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1115,7 +1115,7 @@ def obj_name(cls): 'InstanceExternalEvent': '1.5-1ec57351a9851c1eb43ccd90662d6dd0', 'InstanceFault': '1.2-7ef01f16f1084ad1304a513d6d410a38', 'InstanceFaultList': '1.2-6bb72de2872fe49ded5eb937a93f2451', - 'InstanceGroup': '1.11-852ac511d30913ee88f3c3a869a8f30a', + 'InstanceGroup': '1.11-a26b476ba20883380476d4cb8c4b8d41', 'InstanceGroupList': '1.8-feac6a7c1f4519e069db17e9e6efcff2', 'InstanceInfoCache': '1.5-cd8b96fefe0fc8d4d337243ba0bf0e1e', 'InstanceList': '2.6-238f125650c25d6d12722340d726f723', From c80477d23912d9bdbb7eb3bdf2bda849db0a3bc1 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 16:08:00 +0200 Subject: [PATCH 19/92] vmware: Add a DRS rule sync-loop In case we missed an update to a server-group, we want to be able to recover at some point in time. Therefore, we implement a sync-loop to call the driver's sync_server_group() for every server-group UUID we find as belonging to our host and also for every DRS rule we find. Since we only start the sync-loop for server-groups once on startup, we have to make it resistant against any exceptions - e.g. if the DB is unreachable for a time or the vCenter is unreachable at the time. Change-Id: I9a633dc87ad1aab7d5f00e5143fac97dd3b87176 Incorporated-Change-Id: I74036b6bfc449b407f687afac1ba4365bcbdc2ee (cherry picked from commit 583b1fd389a61464c81bd9cc6a0f89a94a2f0776) (cherry picked from commit 4a58936ca2a5f68e9f73b11062bbc1278d027730) --- nova/conf/vmware.py | 32 +++++++++++++++++- nova/virt/vmwareapi/driver.py | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index abf02bdd3e3..1ff22f3b4ad 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -307,11 +307,41 @@ """), ] +vmwareapi_driver_opts = [ + cfg.FloatOpt('server_group_sync_loop_max_group_spacing', + default=5, + min=0.5, + help=""" +Amount of time in seconds between server-group sync via sync-loop + +The sync-loop for server-groups in the driver iterates over all applicable +server-groups and calls "sync_server_group()" on them, sleeping a random time +before calling the sync. The amount of time slept at max can be defined by this +config value. + +Possible values: + * floating point value of seconds passed to random.uniform(0.5, X) +"""), + cfg.IntOpt('server_group_sync_loop_spacing', + default=3600, + help=""" +Amount of time in seconds to wait between server-group sync-loop runs + +The sync-loop thread for server-groups runs continuously, sleeping after +syncing all groups. This setting defines how long to sleep between runs. + +Possible values: + * integer >= time in seconds to sleep between runs + * integer < 0: disable the sync-loop +"""), +] + ALL_VMWARE_OPTS = (vmwareapi_vif_opts + vmware_utils_opts + vmwareapi_opts + spbm_opts + - vmops_opts) + vmops_opts + + vmwareapi_driver_opts) def register_opts(conf): diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 9b097b5d95e..07b3670aab3 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -20,7 +20,9 @@ """ import os +import random import re +import time import urllib.request import os_resource_classes as orc @@ -43,6 +45,7 @@ from nova.i18n import _ from nova import objects import nova.privsep.path +from nova import utils from nova.virt import driver from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants @@ -62,6 +65,9 @@ TIME_BETWEEN_API_CALL_RETRIES = 1.0 MAX_CONSOLE_BYTES = 100 * units.Ki +UUID_RE = re.compile(r'[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-' + r'[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') + class VMwareVCDriver(driver.ComputeDriver): """The VC host connection object.""" @@ -211,6 +217,9 @@ def init_host(self, host): self._vmops.set_compute_host(host) + LOG.debug("Starting green server-group sync-loop thread") + utils.spawn(self._server_group_sync_loop, host) + def cleanup_host(self, host): self._session.logout() @@ -770,3 +779,56 @@ def detach_interface(self, context, instance, vif): def sync_server_group(self, context, sg_uuid): self._vmops.sync_server_group(context, sg_uuid) + + def _server_group_sync_loop(self, compute_host): + """Retrieve all groups from the cluster and from the DB and call + self.sync_server_group() for them. + + We always wait a little to prohibit overwhelming the cluster. + """ + context = nova_context.get_admin_context() + + while CONF.vmware.server_group_sync_loop_spacing >= 0: + LOG.debug('Starting server-group sync-loop') + try: + + InstanceList = objects.instance.InstanceList + instance_uuids = set(i.uuid for i in InstanceList.get_by_host( + context, compute_host, expected_attrs=[])) + + InstanceGroupList = objects.instance_group.InstanceGroupList + ig_list = InstanceGroupList.get_by_instance_uuids( + context, instance_uuids) + + sg_uuids = set(ig.uuid for ig in ig_list) + + cluster_rules = cluster_util.fetch_cluster_rules( + self._session, cluster_ref=self._cluster_ref) + for rule_name in cluster_rules: + if not rule_name.startswith(constants.DRS_PREFIX): + # not managed by us + continue + + # should look like + # NOVA_d7134f26-f2e2-42ed-b5f1-4092962e84d5-affinity + # NOVA_d7134f26-f2e2-42ed-b5f1-4092962e84d5-soft-anti-affinity + # NOVA_d7134f26-f2e2-42ed-b5f1-4092962e84d5-soft-affinity + m = UUID_RE.search(rule_name) + if not m: + continue + + sg_uuids.add(m.group(0)) + + for sg_uuid in sg_uuids: + spacing = \ + CONF.vmware.server_group_sync_loop_max_group_spacing + sleep_time = random.uniform(0.5, spacing) + time.sleep(sleep_time) + self._vmops.sync_server_group(context, sg_uuid) + except Exception as e: + LOG.exception("Finished server-group sync-loop with error: %s", + e) + else: + LOG.debug('Finished server-group sync-loop') + + time.sleep(CONF.vmware.server_group_sync_loop_spacing) From a3d9188ebb47c1fa09ecbf63822fafafb35601e2 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 16 Sep 2021 16:29:03 +0200 Subject: [PATCH 20/92] vmware: Use sync_server_group() in VM lifecycle We can use sync_server_group() in update_cluster_placement() to use the same mechanism that's used when a user edits a server-group in the DB. This makes sure, that a DRS rule is only created, if it has more than 1 members and also syncs in the other member in case the newly-spawned instance is the second member. Change-Id: I62c1ae3d897f5ccda8788a4d4b23e553be8bc5cf (cherry picked from commit 2dd55ea5572745dbf9e274585d6d6a3ace8c5e20) (cherry picked from commit f82c01f5fc0097255daf656b85356fdeefc02fd9) --- nova/virt/vmwareapi/vmops.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index fd9f4b0b9c6..eebad0817ee 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -746,10 +746,13 @@ def prepare_for_spawn(self, instance): reason=reason) def update_cluster_placement(self, context, instance): - server_group_infos = self._get_server_groups( - context, instance, include_provider_groups=True) + server_group_infos = self._get_server_groups(context, instance) + for group_info in server_group_infos: + self.sync_server_group(context, group_info.name) + + provider_group_infos = self._get_provider_server_groups(instance) vm_util.update_cluster_placement(self._session, instance, - self._cluster, server_group_infos) + self._cluster, provider_group_infos) def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info, block_device_info=None): @@ -1085,26 +1088,29 @@ def reboot(self, instance, network_info, reboot_type="SOFT"): self._session._wait_for_task(reset_task) LOG.debug("Did hard reboot of VM", instance=instance) - def _get_server_groups(self, context, instance, - include_provider_groups=False): + def _get_server_groups(self, context, instance): server_group_infos = [] try: instance_group_object = objects.instance_group.InstanceGroup server_group = instance_group_object.get_by_instance_uuid( context, instance.uuid) if server_group: - name = '{}{}'.format(constants.DRS_PREFIX, server_group.uuid) + name = server_group.uuid server_group_infos.append(GroupInfo(name, server_group.policies)) except nova.exception.InstanceGroupNotFound: pass - if include_provider_groups: - needs_empty_host = utils.vm_needs_special_spawning( - int(instance.memory_mb), instance.flavor) - if CONF.vmware.special_spawning_vm_group and not needs_empty_host: - name = CONF.vmware.special_spawning_vm_group - server_group_infos.append(GroupInfo(name, None)) + return server_group_infos + + def _get_provider_server_groups(self, instance): + server_group_infos = [] + + needs_empty_host = utils.vm_needs_special_spawning( + int(instance.memory_mb), instance.flavor) + if CONF.vmware.special_spawning_vm_group and not needs_empty_host: + name = CONF.vmware.special_spawning_vm_group + server_group_infos.append(GroupInfo(name, None)) return server_group_infos From b30f1c8950435497af2caaca799e5cf5641e2019 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Tue, 24 Aug 2021 09:19:14 +0200 Subject: [PATCH 21/92] api: Pre-query not deleted members in server groups When retrieving multiple - or all - server groups, the code tries to find not deleted members for each server group in every cell individually. This is highly inefficient, which is especially noticable when the number of server groups rises. We change this to query all members of all server-groups we will reply with (i.e. from the already limited list) in advance and pass this set of existing uuids into the function formatting the server group. This is more efficient, because we only do one large query instead of up to 1000 times the number of cells. Change-Id: I3459ce7a8bec9a9e6f3a3b496a3e441078b86af0 (cherry picked from commit e676dd108a35b4cbf69f4938fc12a4b5819d0c24) (cherry picked from commit f216de67c031fabb84ad065141b4937bfc2d0846) --- nova/api/openstack/compute/server_groups.py | 32 ++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/compute/server_groups.py b/nova/api/openstack/compute/server_groups.py index b8c906552f2..9f391a8b3db 100644 --- a/nova/api/openstack/compute/server_groups.py +++ b/nova/api/openstack/compute/server_groups.py @@ -16,6 +16,7 @@ """The Server Group API Extension.""" import collections +import itertools from oslo_log import log as logging import webob @@ -40,7 +41,15 @@ CONF = nova.conf.CONF -def _get_not_deleted(context, uuids): +GROUP_POLICY_OBJ_MICROVERSION = "2.64" + + +def _get_not_deleted(context, uuids, not_deleted_inst_uuids=None): + if not_deleted_inst_uuids: + # short-cut if we already pre-built a list of not deleted instances to + # be more efficient + return list(set(uuids) & not_deleted_inst_uuids) + mappings = objects.InstanceMappingList.get_by_instance_uuids( context, uuids) inst_by_cell = collections.defaultdict(list) @@ -86,7 +95,16 @@ def _should_enable_custom_max_server_rules(context, rules): class ServerGroupController(wsgi.Controller): """The Server group API controller for the OpenStack API.""" - def _format_server_group(self, context, group, req): + def _format_server_group(self, context, group, req, + not_deleted_inst_uuids=None): + """Format ServerGroup according to API version. + + Displays only not-deleted members. + + :param:not_deleted_inst_uuids: Pre-built set of instance-uuids for + multiple server-groups that are found to + be not deleted. + """ # the id field has its value as the uuid of the server group # There is no 'uuid' key in server_group seen by clients. # In addition, clients see policies as a ["policy-name"] list; @@ -105,7 +123,8 @@ def _format_server_group(self, context, group, req): members = [] if group.members: # Display the instances that are not deleted. - members = _get_not_deleted(context, group.members) + members = _get_not_deleted(context, group.members, + not_deleted_inst_uuids) server_group['members'] = members # Add project id information to the response data for # API version v2.13 @@ -171,7 +190,12 @@ def index(self, req): sgs = objects.InstanceGroupList.get_by_project_id( context, project_id) limited_list = common.limited(sgs.objects, req) - result = [self._format_server_group(context, group, req) + + members = list(itertools.chain.from_iterable(sg.members + for sg in limited_list + if sg.members)) + not_deleted = set(_get_not_deleted(context, members)) + result = [self._format_server_group(context, group, req, not_deleted) for group in limited_list] return {'server_groups': result} From f40525e838c5f57afcb9c99d70c7235bb547af38 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 17 Jun 2021 09:25:53 +0200 Subject: [PATCH 22/92] api: server_group's _get_not_deleted returns hosts We exted "server_groups._get_not_deleted()" function to return a dictionary of instance UUID to host mapping. This is a preliminary step to introducing an update of the server-groups, which will need both the instance UUIDs and the hosts to filter out to-be-removed instances and check the validity of the policy when adding instances. We cannot use "InstanceGroup.get_hosts()" for this, as it's not cell-aware and thus extend this function instead. TODO: * tests Change-Id: I253ef54560c2422baec187b350f05b1b2affc34e (cherry picked from commit a94f6f78fc1904f2ca16700233fd8e4709ce7e06) (cherry picked from commit 0b38c200a79f31e71e7aef2c2f973cc31e80b53a) --- nova/api/openstack/compute/server_groups.py | 33 +++++++++++---------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/nova/api/openstack/compute/server_groups.py b/nova/api/openstack/compute/server_groups.py index 9f391a8b3db..26d3245df9d 100644 --- a/nova/api/openstack/compute/server_groups.py +++ b/nova/api/openstack/compute/server_groups.py @@ -44,17 +44,18 @@ GROUP_POLICY_OBJ_MICROVERSION = "2.64" -def _get_not_deleted(context, uuids, not_deleted_inst_uuids=None): - if not_deleted_inst_uuids: +def _get_not_deleted(context, uuids, not_deleted_inst=None): + if not_deleted_inst: # short-cut if we already pre-built a list of not deleted instances to # be more efficient - return list(set(uuids) & not_deleted_inst_uuids) + return {u: not_deleted_inst[u] for u in uuids + if u in not_deleted_inst} mappings = objects.InstanceMappingList.get_by_instance_uuids( context, uuids) inst_by_cell = collections.defaultdict(list) cell_mappings = {} - found_inst_uuids = [] + found_inst = {} # Get a master list of cell mappings, and a list of instance # uuids organized by cell @@ -62,7 +63,7 @@ def _get_not_deleted(context, uuids, not_deleted_inst_uuids=None): if not im.cell_mapping: # Not scheduled yet, so just throw it in the final list # and move on - found_inst_uuids.append(im.instance_uuid) + found_inst[im.instance_uuid] = None continue if im.cell_mapping.uuid not in cell_mappings: cell_mappings[im.cell_mapping.uuid] = im.cell_mapping @@ -76,11 +77,11 @@ def _get_not_deleted(context, uuids, not_deleted_inst_uuids=None): {'cell': cell_mapping.identity, 'num': len(inst_uuids)}) filters = {'uuid': inst_uuids, 'deleted': False} with nova_context.target_cell(context, cell_mapping) as ctx: - found_inst_uuids.extend([ - inst.uuid for inst in objects.InstanceList.get_by_filters( - ctx, filters=filters)]) + instances = objects.InstanceList.get_by_filters( + ctx, filters=filters) + found_inst.update({inst.uuid: inst.host for inst in instances}) - return found_inst_uuids + return found_inst def _should_enable_custom_max_server_rules(context, rules): @@ -96,14 +97,14 @@ class ServerGroupController(wsgi.Controller): """The Server group API controller for the OpenStack API.""" def _format_server_group(self, context, group, req, - not_deleted_inst_uuids=None): + not_deleted_inst=None): """Format ServerGroup according to API version. Displays only not-deleted members. - :param:not_deleted_inst_uuids: Pre-built set of instance-uuids for - multiple server-groups that are found to - be not deleted. + :param:not_deleted_inst: Pre-built dict of instance-uuid: host for + multiple server-groups that are found to be + not deleted. """ # the id field has its value as the uuid of the server group # There is no 'uuid' key in server_group seen by clients. @@ -123,8 +124,8 @@ def _format_server_group(self, context, group, req, members = [] if group.members: # Display the instances that are not deleted. - members = _get_not_deleted(context, group.members, - not_deleted_inst_uuids) + members = list(_get_not_deleted(context, group.members, + not_deleted_inst)) server_group['members'] = members # Add project id information to the response data for # API version v2.13 @@ -194,7 +195,7 @@ def index(self, req): members = list(itertools.chain.from_iterable(sg.members for sg in limited_list if sg.members)) - not_deleted = set(_get_not_deleted(context, members)) + not_deleted = _get_not_deleted(context, members) result = [self._format_server_group(context, group, req, not_deleted) for group in limited_list] return {'server_groups': result} From db0d2a501ad5ca7500785e5e80761618e2414b08 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 17 Jun 2021 09:29:14 +0200 Subject: [PATCH 23/92] api: Add an update method for server-groups This new API endpoint allows to add and/or remove members from a server-group. We found this to be necessary, because instances might get created without a server-group, but later need an HA-partner and re-installing would mean downtime or too much effort. The endpoint checks the validity of the policy is still given after all changes are done and strives for idempotency in that it allows removal/addition of already removed/added instance uuids to accommodate for requests built in parallel. We do not allow a server to join a second server-group if it already joined one. The whole change to the server-group is errored out if any of the servers are already in another group to make it easier in validation and either apply the whole change requested by the user or nothing. When we add/remove a server to/from a server-group, we have to update the server's RequestSpec's `instance_group` attribute, because this is used during scheduling when resizing a server to record a list of appropriate hosts for the instance. Only members/admins can change the server-groups. TODO: * api-request docs Change-Id: I30d5d1dc3a41553b4336aad3877018989159495c Incorporated-Change-Id: Ibc78cbd80ae49df9be60f6d9eda56a47d9ffe39e Incorporated-Change-Id: I6a06e4b4a8c22737c20ab51e85ceb2bf98082b26 Incorporated-Change-Id: Ic193dd3c59bc717ba5329f63054297f44127d76d Co-Authored-By: Jakob Karge (cherry picked from commit aec6f6e5540eaa504e5425768b2db745ae14eecc) --- nova/api/openstack/compute/routes.py | 1 + .../compute/schemas/server_groups.py | 15 + nova/api/openstack/compute/server_groups.py | 132 ++++++++ nova/policies/server_groups.py | 12 + .../openstack/compute/test_server_groups.py | 313 +++++++++++++++++- nova/tests/unit/test_policy.py | 1 + 6 files changed, 470 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/compute/routes.py b/nova/api/openstack/compute/routes.py index 6083c94b660..c47f40847c7 100644 --- a/nova/api/openstack/compute/routes.py +++ b/nova/api/openstack/compute/routes.py @@ -684,6 +684,7 @@ def _create_controller(main_controller, action_controller_list): }), ('/os-server-groups/{id}', { 'GET': [server_groups_controller, 'show'], + 'PUT': [server_groups_controller, 'update'], 'DELETE': [server_groups_controller, 'delete'] }), ('/os-services', { diff --git a/nova/api/openstack/compute/schemas/server_groups.py b/nova/api/openstack/compute/schemas/server_groups.py index dcaa7616d47..6eb48427627 100644 --- a/nova/api/openstack/compute/schemas/server_groups.py +++ b/nova/api/openstack/compute/schemas/server_groups.py @@ -99,3 +99,18 @@ 'properties': {}, 'additionalProperties': True, } + +update = { + 'type': 'object', + 'properties': { + 'add_members': { + 'type': 'array', + 'items': parameter_types.server_id, + }, + 'remove_members': { + 'type': 'array', + 'items': parameter_types.server_id, + } + }, + 'additionalProperties': False +} diff --git a/nova/api/openstack/compute/server_groups.py b/nova/api/openstack/compute/server_groups.py index 26d3245df9d..5a39337971c 100644 --- a/nova/api/openstack/compute/server_groups.py +++ b/nova/api/openstack/compute/server_groups.py @@ -273,3 +273,135 @@ def create(self, req, body): raise exc.HTTPForbidden(explanation=msg) return {'server_group': self._format_server_group(context, sg, req)} + + @wsgi.Controller.api_version("2.64") + @validation.schema(schema.update) + @wsgi.expected_errors((400, 404)) + def update(self, req, id, body): + """Update a server-group's members + + Striving for idempotency, we accept already removed or already + contained members. + + We always remove first and then check if we can add the requested + members. That way, removing an instance for a host and adding another + one works in one request. + + We do all requested changes or no change. + """ + context = req.environ['nova.context'] + project_id = context.project_id + context.can(sg_policies.POLICY_ROOT % 'update', + target={'project_id': project_id}) + try: + sg = objects.InstanceGroup.get_by_uuid(context, id) + except nova.exception.InstanceGroupNotFound as e: + raise webob.exc.HTTPNotFound(explanation=e.format_message()) + + members_to_remove = set(body.get('remove_members', [])) + members_to_add = set(body.get('add_members', [])) + LOG.info('Called update for server-group %s with add_members: %s and ' + 'remove_members %s', + id, ', '.join(members_to_add), ', '.join(members_to_remove)) + + overlap = members_to_remove & members_to_add + if overlap: + msg = ('Parameters "add_members" and "remove_members" are ' + 'overlapping in {}'.format(', '.join(overlap))) + raise exc.HTTPBadRequest(explanation=msg) + + if not members_to_remove and not members_to_add: + LOG.info("No update requested.") + formatted_sg = self._format_server_group(context, sg, req) + return {'server_group': formatted_sg} + + # don't do work if it's not necessary. we might be able to get a fast + # way out if this request is already fulfilled + members_to_remove = members_to_remove & set(sg.members) + members_to_add = members_to_add - set(sg.members) + + if not members_to_remove and not members_to_add: + LOG.info("State already satisfied.") + formatted_sg = self._format_server_group(context, sg, req) + return {'server_group': formatted_sg} + + # retrieve all the instances to add, failing if one doesn't exist, + # because we need to check the hosts against the policy and adding + # non-existent instances doesn't make sense + members_to_search = members_to_add | members_to_remove + found_instances_hosts = _get_not_deleted(context, members_to_search) + missing_uuids = members_to_add - set(found_instances_hosts) + if missing_uuids: + msg = ("One or more members in add_members cannot be found: {}" + .format(', '.join(missing_uuids))) + raise exc.HTTPBadRequest(explanation=msg) + + # check if (some of) the VMs are already members of another + # instance_group. We cannot support this as they might contradict. + found_server_groups = \ + objects.InstanceGroupList.get_by_instance_uuids(context, + members_to_add) + other_server_groups = [_x.uuid for _x in found_server_groups + if _x.uuid != id] + if other_server_groups: + msg = ("One or more members in add_members is already assigned " + "to another server group. Server groups: {}" + .format(', '.join(other_server_groups))) + raise exc.HTTPBadRequest(explanation=msg) + + # check if the policy is still valid with these changes + if sg.policy in ('affinity', 'anti-affinity'): + current_members_hosts = _get_not_deleted(context, sg.members) + current_hosts = set(h for u, h in current_members_hosts.items() + if u not in members_to_remove) + if sg.policy == 'affinity': + outliers = [u for u, h in found_instances_hosts.items() + if h and h not in current_hosts] + elif sg.policy == 'anti-affinity': + outliers = [u for u, h in found_instances_hosts.items() + if h and h in current_hosts] + else: + outliers = None + LOG.warning('server-group update check not implemented for ' + 'policy %s', sg.policy) + if outliers: + LOG.info('Update of server-group %s with policy %s aborted: ' + 'policy violation by %s', + id, sg.policy, ', '.join(outliers)) + msg = ("Adding instance(s) {} would violate policy '{}'." + .format(', '.join(outliers), sg.policy)) + raise exc.HTTPBadRequest(explanation=msg) + + # update the server group and save it + if members_to_remove: + objects.InstanceGroup.remove_members(context, sg.id, + members_to_remove, sg.uuid) + if members_to_add: + try: + objects.InstanceGroup.add_members(context, id, members_to_add) + except Exception: + LOG.exception('Failed to add members.') + if members_to_remove: + LOG.info('Trying to add removed members again after ' + 'error.') + objects.InstanceGroup.add_members(context, id, + members_to_remove) + raise + + LOG.info("Changed server-group %s in DB.", id) + + # refresh InstanceGroup object, because we changed it directly in the + # DB. + sg.refresh() + + # update the request-specs of the updated members + for member_uuid in found_instances_hosts: + request_spec = \ + objects.RequestSpec.get_by_instance_uuid(context, member_uuid) + if member_uuid in members_to_add: + request_spec.instance_group = sg + else: + request_spec.instance_group = None + request_spec.save() + + return {'server_group': self._format_server_group(context, sg, req)} diff --git a/nova/policies/server_groups.py b/nova/policies/server_groups.py index 8dfbe7c2020..45b130ff4b2 100644 --- a/nova/policies/server_groups.py +++ b/nova/policies/server_groups.py @@ -82,6 +82,18 @@ ], scope_types=['project'] ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'update', + check_str=base.PROJECT_MEMBER_OR_ADMIN, + description="Update members of a server group", + operations=[ + { + 'path': '/os-server-groups/{server_group_id}', + 'method': 'PUT' + } + ], + scope_types=['system', 'project'] + ), ] diff --git a/nova/tests/unit/api/openstack/compute/test_server_groups.py b/nova/tests/unit/api/openstack/compute/test_server_groups.py index 5c4c9095764..714e651865d 100644 --- a/nova/tests/unit/api/openstack/compute/test_server_groups.py +++ b/nova/tests/unit/api/openstack/compute/test_server_groups.py @@ -165,13 +165,13 @@ def test_create_server_group_rbac_default(self): # test as non-admin self.controller.create(self.member_req, body={'server_group': sgroup}) - def _create_instance(self, ctx, cell): + def _create_instance(self, ctx, cell, host='host1'): with context.target_cell(ctx, cell) as cctx: instance = objects.Instance(context=cctx, image_ref=uuidsentinel.fake_image_ref, compute_id=123, node='node1', reservation_id='a', - host='host1', + host=host, project_id=fakes.FAKE_PROJECT_ID, vm_state='fake', system_metadata={'key': 'value'}) @@ -184,10 +184,10 @@ def _create_instance(self, ctx, cell): im.create() return instance - def _create_instance_group(self, context, members): + def _create_instance_group(self, context, members, policy=None): ig = objects.InstanceGroup(context=context, name='fake_name', user_id='fake_user', project_id=fakes.FAKE_PROJECT_ID, - members=members) + members=members, policy=policy) ig.create() return ig.uuid @@ -780,6 +780,311 @@ def test_additional_params(self): self.assertRaises(self.validation_error, self.controller.create, req, body={'server_group': sgroup}) + def test_update_server_group_not_found(self): + """We raise a 404 if the server group does not exist.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.update, req, uuidsentinel.group1, body={}) + + def test_update_server_group_empty(self): + """We do not fail if the user doesn't request any changes""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + res_dict = self.controller.update(req, ig_uuid, body={}) + result_members = res_dict['server_group']['members'] + self.assertEqual(3, len(result_members)) + for member in members: + self.assertIn(member, result_members) + + def test_update_server_group_add_remove_overlap(self): + """We do not accept changes, if there's a server to be both added and + removed, because the result would depend on implementation details if + we remove first or add first. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'add_members': [uuidsentinel.uuid1, uuidsentinel.uuid2], + 'remove_members': [uuidsentinel.uuid2, uuidsentinel.uuid3], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn('Parameters "add_members" and "remove_members" are ' + 'overlapping in {}'.format(uuidsentinel.uuid2), + str(result)) + + def test_update_server_group_remove_nonexisting(self): + """Don't fail if the user tries to remove a server not being member of + the server group. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'remove_members': [uuidsentinel.uuid4], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(3, len(result_members)) + for member in members: + self.assertIn(member, result_members) + + def test_update_server_group_add_already_added(self): + """Don't fail if the user adds a server that's already a member of the + server group. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'add_members': [members[0]], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(3, len(result_members)) + for member in members: + self.assertIn(member, result_members) + + def test_update_server_group_add_against_policy_affinity(self): + """Fail if adding the server would break the policy.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host2') + + body = { + 'add_members': [new_instance.uuid], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn("Adding instance(s) {} would violate policy 'affinity'." + .format(new_instance.uuid), + str(result)) + + def test_update_server_group_add_against_policy_anti_affinity(self): + """Fail if adding the server would break the policy.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='anti-affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host1') + + body = { + 'add_members': [new_instance.uuid], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn("Adding instance(s) {} would violate policy " + "'anti-affinity'.".format(new_instance.uuid), + str(result)) + + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_add_with_remove_fixes_policy(self, + mock_req_spec): + """Don't fail if adding a server would break the policy, but the remove + in the same request fixes that. + """ + """Fail if adding the server would break the policy.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1'), + self._create_instance(ctx, cell1, host='host2')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='anti-affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host2') + + body = { + 'add_members': [new_instance.uuid], + 'remove_members': [instances[1].uuid], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(2, len(result_members)) + for member in [instances[0].uuid, new_instance.uuid]: + self.assertIn(member, result_members) + + def test_update_server_group_add_nonexisting_instance(self): + """Fail if the instances the user tries to add does not exist.""" + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + body = { + 'add_members': [uuidsentinel.uuid1], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid, body=body) + self.assertIn('One or more members in add_members cannot be found: ' + '{}'.format(uuidsentinel.uuid1), + str(result)) + + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_add_instance_multiple_cells(self, + mock_req_spec): + """Don't fail if the instance the user tries to add is in another cell. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + (ig_uuid, instances, members) = self._create_groups_and_instances(ctx) + cell1 = self.cells[uuidsentinel.cell1] + cell2 = self.cells[uuidsentinel.cell2] + new_instances = [self._create_instance(ctx, cell1), + self._create_instance(ctx, cell2)] + body = { + 'add_members': [i.uuid for i in new_instances], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(5, len(result_members)) + for member in members: + self.assertIn(member, result_members) + for instance in new_instances: + self.assertIn(instance.uuid, result_members) + + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_add_against_soft_policy(self, mock_req_spec): + """Don't fail if the policy would fail, but it's a soft-* policy - they + are best-effort by design. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances], + policy='soft-anti-affinity') + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host1') + + body = { + 'add_members': [new_instance.uuid], + } + res_dict = self.controller.update(req, ig_uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(2, len(result_members)) + for member in [instances[0].uuid, new_instance.uuid]: + self.assertIn(member, result_members) + + @mock.patch('nova.objects.InstanceGroupList.get_by_instance_uuids') + def test_update_server_group_add_already_in_other(self, mock_gbiu): + """If any of the servers is already part of another server-group, we + fail. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig_uuid = self._create_instance_group(ctx, [i.uuid for i in instances]) + + cell1 = self.cells[uuidsentinel.cell1] + new_instance = self._create_instance(ctx, cell1, host='host1') + ig_uuid2 = self._create_instance_group(ctx, [new_instance.uuid]) + + mock_gbiu.return_value = objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db({'id': ig_uuid}))]) + + body = { + 'add_members': [instances[0].uuid], + } + result = self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, req, ig_uuid2, body=body) + self.assertIn('One or more members in add_members is already ' + 'assigned to another server group. Server groups: {}' + .format(ig_uuid), str(result)) + + @mock.patch('nova.objects.InstanceGroupList.get_by_instance_uuids') + @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') + def test_update_server_group_updates_request_spec(self, mock_req_spec, + mock_gbiu): + """If we update the server-group membership of a server, we also have + to update the appropriate RequestSpec attribute. + """ + req = fakes.HTTPRequest.blank('', use_admin_context=True, + version=self.wsgi_api_version) + ctx = context.RequestContext('fake_user', 'fake') + + cell1 = self.cells[uuidsentinel.cell1] + instances = [self._create_instance(ctx, cell1, host='host1')] + + ig = objects.InstanceGroup(context=ctx, name='fake_name', + user_id='fake_user', project_id='fake', + members=[i.uuid for i in instances], policy='anti-affinity') + ig.create() + + new_instance = self._create_instance(ctx, cell1, host='host2') + + mock_gbiu.return_value = objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db({'id': ig.uuid}))]) + + fake_spec_remove = mock.Mock(instance_group=ig) + fake_spec_add = mock.Mock(instance_group=None) + + def _get_req_spec(context, instance_uuid): + if instance_uuid == new_instance.uuid: + return fake_spec_add + else: + return fake_spec_remove + + mock_req_spec.side_effect = _get_req_spec + + self.assertEqual(ig.uuid, fake_spec_remove.instance_group.uuid) + self.assertIsNone(fake_spec_add.instance_group) + + body = { + 'add_members': [new_instance.uuid], + 'remove_members': [instances[0].uuid] + } + res_dict = self.controller.update(req, ig.uuid, body=body) + result_members = res_dict['server_group']['members'] + self.assertEqual(1, len(result_members)) + for member in [new_instance.uuid]: + self.assertIn(member, result_members) + + fake_spec_remove.save.assert_called() + self.assertIsNone(fake_spec_remove.instance_group) + ig.members = [new_instance.uuid] + self.assertEqual(ig.uuid, fake_spec_add.instance_group.uuid) + fake_spec_add.save.assert_called() + class ServerGroupTestV275(ServerGroupTestV264): wsgi_api_version = '2.75' diff --git a/nova/tests/unit/test_policy.py b/nova/tests/unit/test_policy.py index 19f3165717a..519cf21ab8e 100644 --- a/nova/tests/unit/test_policy.py +++ b/nova/tests/unit/test_policy.py @@ -471,6 +471,7 @@ def setUp(self): "os_compute_api:os-server-groups:show", "os_compute_api:os-server-groups:create", "os_compute_api:os-server-groups:delete", +"os_compute_api:os-server-groups:update", "os_compute_api:os-shelve:shelve", "os_compute_api:os-shelve:unshelve", "os_compute_api:os-volumes:create", From 732bbc4f85af843059bd22cf8db26bb807b4128c Mon Sep 17 00:00:00 2001 From: i540489 Date: Mon, 23 Jan 2023 11:50:36 +0100 Subject: [PATCH 24/92] Enhance performance of get instance groups from DB Instead of getting all instance groups from DB and after then limit the results with common.limited, we added limit and offset parameters to _instance_group_get_query function to limit the database query itself. SQLAlchemy throws an exception when applying a filter to the query after setting limits and offset that's why project_id parameter added to _instance_group_get_query. Change-Id: I30c9bc43f4b7ff9205f6a9fa6cb947b669a2fbba (cherry picked from commit 6802f8038250539cac1e379827449df7bca83b0e) --- nova/api/openstack/common.py | 10 +++++ nova/api/openstack/compute/server_groups.py | 12 +++--- nova/objects/instance_group.py | 30 ++++++++++----- .../openstack/compute/test_server_groups.py | 22 ++++++++--- .../tests/unit/objects/test_instance_group.py | 37 +++++++++++++++++-- nova/tests/unit/objects/test_objects.py | 2 +- 6 files changed, 89 insertions(+), 24 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index c54cb8bfb89..7ca7688dacf 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -242,6 +242,16 @@ def get_limit_and_marker(request): return limit, marker +def get_limit_and_offset(request): + """Get limit and offset parameters from request.""" + params = get_pagination_params(request) + limit = CONF.api.max_limit + limit = min(limit, params.get('limit', limit)) + offset = params.get('offset', None) + + return limit, offset + + def get_id_from_href(href): """Return the id or uuid portion of a url. diff --git a/nova/api/openstack/compute/server_groups.py b/nova/api/openstack/compute/server_groups.py index 5a39337971c..68f5ec5558b 100644 --- a/nova/api/openstack/compute/server_groups.py +++ b/nova/api/openstack/compute/server_groups.py @@ -169,6 +169,7 @@ def delete(self, req, id): def index(self, req): """Returns a list of server groups.""" context = req.environ['nova.context'] + limit, offset = common.get_limit_and_offset(req) project_id = context.project_id # NOTE(gmann): Using context's project_id as target here so # that when we remove the default target from policy class, @@ -186,18 +187,19 @@ def index(self, req): # Until then, let's keep the old behaviour. context.can(sg_policies.POLICY_ROOT % 'index:all_projects', target={'project_id': project_id}) - sgs = objects.InstanceGroupList.get_all(context) + sgs = objects.InstanceGroupList.get_all(context, + limit=limit, + offset=offset) else: sgs = objects.InstanceGroupList.get_by_project_id( - context, project_id) - limited_list = common.limited(sgs.objects, req) + context, project_id, limit=limit, offset=offset) members = list(itertools.chain.from_iterable(sg.members - for sg in limited_list + for sg in sgs if sg.members)) not_deleted = _get_not_deleted(context, members) result = [self._format_server_group(context, group, req, not_deleted) - for group in limited_list] + for group in sgs] return {'server_groups': result} @wsgi.Controller.api_version("2.1") diff --git a/nova/objects/instance_group.py b/nova/objects/instance_group.py index c9b6c7dc672..d1c3719b052 100644 --- a/nova/objects/instance_group.py +++ b/nova/objects/instance_group.py @@ -34,14 +34,23 @@ LOG = logging.getLogger(__name__) -def _instance_group_get_query(context, id_field=None, id=None): +def _instance_group_get_query(context, id_field=None, id=None, + project_id=None, + limit=None, + offset=None): query = context.session.query(api_models.InstanceGroup).\ options(orm.joinedload(api_models.InstanceGroup._policies)).\ options(orm.joinedload(api_models.InstanceGroup._members)) if not context.is_admin: query = query.filter_by(project_id=context.project_id) + elif project_id is not None: + query = query.filter_by(project_id=project_id) if id and id_field: query = query.filter(id_field == id) + if limit is not None: + query = query.limit(limit) + if offset is not None: + query = query.offset(offset) return query @@ -552,10 +561,11 @@ class InstanceGroupList(base.ObjectListBase, base.NovaObject): @staticmethod @api_db_api.context_manager.reader - def _get_from_db(context, project_id=None): - query = _instance_group_get_query(context) - if project_id is not None: - query = query.filter_by(project_id=project_id) + def _get_from_db(context, project_id=None, limit=None, offset=None): + query = _instance_group_get_query(context, + project_id=project_id, + limit=limit, + offset=offset) return query.all() @staticmethod @@ -588,14 +598,16 @@ def _get_from_db_by_instance_uuids(context, instance_uuids): return groups @base.remotable_classmethod - def get_by_project_id(cls, context, project_id): - api_db_groups = cls._get_from_db(context, project_id=project_id) + def get_by_project_id(cls, context, project_id, limit=None, offset=None): + api_db_groups = cls._get_from_db(context, project_id=project_id, + limit=limit, + offset=offset) return base.obj_make_list(context, cls(context), objects.InstanceGroup, api_db_groups) @base.remotable_classmethod - def get_all(cls, context): - api_db_groups = cls._get_from_db(context) + def get_all(cls, context, limit=None, offset=None): + api_db_groups = cls._get_from_db(context, limit=limit, offset=offset) return base.obj_make_list(context, cls(context), objects.InstanceGroup, api_db_groups) diff --git a/nova/tests/unit/api/openstack/compute/test_server_groups.py b/nova/tests/unit/api/openstack/compute/test_server_groups.py index 714e651865d..a7c1ca02d93 100644 --- a/nova/tests/unit/api/openstack/compute/test_server_groups.py +++ b/nova/tests/unit/api/openstack/compute/test_server_groups.py @@ -275,16 +275,26 @@ def _test_list_server_group(self, mock_get_all, mock_get_by_project, tenant_specific = {'server_groups': tenant_groups} def return_all_server_groups(): - return objects.InstanceGroupList( - objects=[objects.InstanceGroup( - **server_group_db(sg)) for sg in all_groups]) + if limited: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in tenant_groups]) + else: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in all_groups]) mock_get_all.return_value = return_all_server_groups() def return_tenant_server_groups(): - return objects.InstanceGroupList( - objects=[objects.InstanceGroup( - **server_group_db(sg)) for sg in tenant_groups]) + if limited: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in []]) + else: + return objects.InstanceGroupList( + objects=[objects.InstanceGroup( + **server_group_db(sg)) for sg in tenant_groups]) mock_get_by_project.return_value = return_tenant_server_groups() diff --git a/nova/tests/unit/objects/test_instance_group.py b/nova/tests/unit/objects/test_instance_group.py index 5ea566fea7e..b6143b8127e 100644 --- a/nova/tests/unit/objects/test_instance_group.py +++ b/nova/tests/unit/objects/test_instance_group.py @@ -363,7 +363,35 @@ def test_get_by_hint(self, mock_name, mock_uuid): class TestInstanceGroupObject(test_objects._LocalTest, _TestInstanceGroupObject): - pass + def test_get_all_with_limits(self): + entries = {'entry1': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg1, + 'name': 'sg1'}, + 'entry2': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg2, + 'name': 'sg2'}, + 'entry3': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg3, + 'name': 'sg3'}, + 'entry4': {'user_id': self.context.user_id, + 'project_id': self.context.project_id, + 'uuid': uuids.sg4, + 'name': 'sg4'}} + for key, value in entries.items(): + objects.instance_group.InstanceGroup._create_in_db(self.context, + value) + + servg = objects.InstanceGroupList.get_all(self.context, + limit=2, + offset=1) + self.assertEqual(2, len(servg)) + servg = objects.InstanceGroupList.get_all(self.context, + limit=2, + offset=3) + self.assertEqual(1, len(servg)) class TestRemoteInstanceGroupObject(test_objects._RemoteTest, @@ -393,7 +421,9 @@ def test_list_all(self, mock_api_get): mock_api_get.side_effect = _mock_db_list_get inst_list = objects.InstanceGroupList.get_all(self.context) self.assertEqual(4, len(inst_list.objects)) - mock_api_get.assert_called_once_with(self.context) + mock_api_get.assert_called_once_with(self.context, + limit=None, + offset=None) @mock.patch('nova.objects.InstanceGroupList._get_from_db') def test_list_by_project_id(self, mock_api_get): @@ -401,7 +431,8 @@ def test_list_by_project_id(self, mock_api_get): objects.InstanceGroupList.get_by_project_id( self.context, mock.sentinel.project_id) mock_api_get.assert_called_once_with( - self.context, project_id=mock.sentinel.project_id) + self.context, project_id=mock.sentinel.project_id, + limit=None, offset=None) class TestInstanceGroupListObject(test_objects._LocalTest, diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index c80981a7924..2b99df6f960 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1116,7 +1116,7 @@ def obj_name(cls): 'InstanceFault': '1.2-7ef01f16f1084ad1304a513d6d410a38', 'InstanceFaultList': '1.2-6bb72de2872fe49ded5eb937a93f2451', 'InstanceGroup': '1.11-a26b476ba20883380476d4cb8c4b8d41', - 'InstanceGroupList': '1.8-feac6a7c1f4519e069db17e9e6efcff2', + 'InstanceGroupList': '1.8-b713c6273b2c468d11cd058a3b55c373', 'InstanceInfoCache': '1.5-cd8b96fefe0fc8d4d337243ba0bf0e1e', 'InstanceList': '2.6-238f125650c25d6d12722340d726f723', 'InstanceMapping': '1.2-3bd375e65c8eb9c45498d2f87b882e03', From 3a2c8761d28c2dcbbae0151fa4d7335380f1cbf6 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 27 Sep 2021 09:09:50 +0200 Subject: [PATCH 25/92] api: Send server-group update to relevant hosts When a server-group gets updated, we send an update to all hosts that got an update - which are the hosts the instances we removed/added belong to. Change-Id: Ica1e516358bf6301ab7a2c59631203721dbe658d (cherry picked from commit cfa0696af7e3b41318da38d6e0a066bfbde5cd2c) --- nova/api/openstack/compute/server_groups.py | 10 ++++++ .../openstack/compute/test_server_groups.py | 34 +++++++++++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/compute/server_groups.py b/nova/api/openstack/compute/server_groups.py index 68f5ec5558b..f3367008d63 100644 --- a/nova/api/openstack/compute/server_groups.py +++ b/nova/api/openstack/compute/server_groups.py @@ -27,6 +27,7 @@ from nova.api.openstack.compute.schemas import server_groups as schema from nova.api.openstack import wsgi from nova.api import validation +from nova.compute import api as compute import nova.conf from nova import context as nova_context import nova.exception @@ -96,6 +97,10 @@ def _should_enable_custom_max_server_rules(context, rules): class ServerGroupController(wsgi.Controller): """The Server group API controller for the OpenStack API.""" + def __init__(self, **kwargs): + super(ServerGroupController, self).__init__(**kwargs) + self.compute_api = compute.API() + def _format_server_group(self, context, group, req, not_deleted_inst=None): """Format ServerGroup according to API version. @@ -406,4 +411,9 @@ def update(self, req, id, body): request_spec.instance_group = None request_spec.save() + # tell the compute hosts about the update, so they can sync if + # necessary + hosts_to_update = set(h for h in found_instances_hosts.values() if h) + self.compute_api.sync_server_group(context, hosts_to_update, id) + return {'server_group': self._format_server_group(context, sg, req)} diff --git a/nova/tests/unit/api/openstack/compute/test_server_groups.py b/nova/tests/unit/api/openstack/compute/test_server_groups.py index a7c1ca02d93..0e91ebd49a3 100644 --- a/nova/tests/unit/api/openstack/compute/test_server_groups.py +++ b/nova/tests/unit/api/openstack/compute/test_server_groups.py @@ -799,7 +799,8 @@ def test_update_server_group_not_found(self): self.assertRaises(webob.exc.HTTPNotFound, self.controller.update, req, uuidsentinel.group1, body={}) - def test_update_server_group_empty(self): + @mock.patch('nova.compute.api.API.sync_server_group') + def test_update_server_group_empty(self, mock_sync): """We do not fail if the user doesn't request any changes""" req = fakes.HTTPRequest.blank('', use_admin_context=True, version=self.wsgi_api_version) @@ -810,6 +811,7 @@ def test_update_server_group_empty(self): self.assertEqual(3, len(result_members)) for member in members: self.assertIn(member, result_members) + mock_sync.assert_not_called() def test_update_server_group_add_remove_overlap(self): """We do not accept changes, if there's a server to be both added and @@ -830,7 +832,8 @@ def test_update_server_group_add_remove_overlap(self): 'overlapping in {}'.format(uuidsentinel.uuid2), str(result)) - def test_update_server_group_remove_nonexisting(self): + @mock.patch('nova.compute.api.API.sync_server_group') + def test_update_server_group_remove_nonexisting(self, mock_sync): """Don't fail if the user tries to remove a server not being member of the server group. """ @@ -846,8 +849,10 @@ def test_update_server_group_remove_nonexisting(self): self.assertEqual(3, len(result_members)) for member in members: self.assertIn(member, result_members) + mock_sync.assert_not_called() - def test_update_server_group_add_already_added(self): + @mock.patch('nova.compute.api.API.sync_server_group') + def test_update_server_group_add_already_added(self, mock_sync): """Don't fail if the user adds a server that's already a member of the server group. """ @@ -863,6 +868,7 @@ def test_update_server_group_add_already_added(self): self.assertEqual(3, len(result_members)) for member in members: self.assertIn(member, result_members) + mock_sync.assert_not_called() def test_update_server_group_add_against_policy_affinity(self): """Fail if adding the server would break the policy.""" @@ -912,13 +918,14 @@ def test_update_server_group_add_against_policy_anti_affinity(self): "'anti-affinity'.".format(new_instance.uuid), str(result)) + @mock.patch('nova.compute.api.API.sync_server_group') @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') def test_update_server_group_add_with_remove_fixes_policy(self, - mock_req_spec): + mock_req_spec, + mock_sync): """Don't fail if adding a server would break the policy, but the remove in the same request fixes that. """ - """Fail if adding the server would break the policy.""" req = fakes.HTTPRequest.blank('', use_admin_context=True, version=self.wsgi_api_version) ctx = context.RequestContext('fake_user', 'fake') @@ -942,6 +949,9 @@ def test_update_server_group_add_with_remove_fixes_policy(self, self.assertEqual(2, len(result_members)) for member in [instances[0].uuid, new_instance.uuid]: self.assertIn(member, result_members) + req_context = req.environ['nova.context'] + mock_sync.assert_called_with(req_context, set(['host2']), + ig_uuid) def test_update_server_group_add_nonexisting_instance(self): """Fail if the instances the user tries to add does not exist.""" @@ -958,9 +968,11 @@ def test_update_server_group_add_nonexisting_instance(self): '{}'.format(uuidsentinel.uuid1), str(result)) + @mock.patch('nova.compute.api.API.sync_server_group') @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') def test_update_server_group_add_instance_multiple_cells(self, - mock_req_spec): + mock_req_spec, + mock_sync): """Don't fail if the instance the user tries to add is in another cell. """ req = fakes.HTTPRequest.blank('', use_admin_context=True, @@ -981,9 +993,15 @@ def test_update_server_group_add_instance_multiple_cells(self, self.assertIn(member, result_members) for instance in new_instances: self.assertIn(instance.uuid, result_members) + expected_hosts = \ + set([i.host for i in instances] + [i.host for i in new_instances]) + req_context = req.environ['nova.context'] + mock_sync.assert_called_with(req_context, expected_hosts, ig_uuid) + @mock.patch('nova.compute.api.API.sync_server_group') @mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid') - def test_update_server_group_add_against_soft_policy(self, mock_req_spec): + def test_update_server_group_add_against_soft_policy(self, mock_req_spec, + mock_sync): """Don't fail if the policy would fail, but it's a soft-* policy - they are best-effort by design. """ @@ -1008,6 +1026,8 @@ def test_update_server_group_add_against_soft_policy(self, mock_req_spec): self.assertEqual(2, len(result_members)) for member in [instances[0].uuid, new_instance.uuid]: self.assertIn(member, result_members) + req_context = req.environ['nova.context'] + mock_sync.assert_called_with(req_context, set(['host1']), ig_uuid) @mock.patch('nova.objects.InstanceGroupList.get_by_instance_uuids') def test_update_server_group_add_already_in_other(self, mock_gbiu): From f8fa1922d01a3c38a68596fb9298e7bb5fcc60a6 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 28 Mar 2019 08:48:30 +0100 Subject: [PATCH 26/92] Add option to reserve all memory If we don't reserve all memory, there's a swap file pre-created for every VM on the ephemeral storage - even if it's unused. But this option comes at the price of not being able to over-commit memory. Related issue: CCM-9589 Change-Id: I916f3ed8d32efc9de36b943546fee7440bbbf7ca Note: This will later be reverted in favor of manual memory allocations. (cherry picked from commit e028cd61543c632ff0773b1a396fa9c8febc953f) --- nova/conf/vmware.py | 7 +++++++ .../tests/unit/virt/vmwareapi/test_vm_util.py | 20 +++++++++++++++++++ nova/virt/vmwareapi/vm_util.py | 2 ++ 3 files changed, 29 insertions(+) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index 1ff22f3b4ad..103a7bffa74 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -94,6 +94,13 @@ storing console log files and responding to get requests. If defined it will override serial_log_dir. """), + cfg.BoolOpt('reserve_all_memory', + default=False, + help=""" +If true, it is not possible to over-commit memory in the vCenter, but there's +also no swap file pre-created on the ephemeral storage. Swap still works, but +is hot-created. +""") ] vmwareapi_opts = [ diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 38e7cf0edc4..21e02a05fd8 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -691,6 +691,8 @@ def test_get_vm_create_spec(self): extra_specs) expected = self._create_vm_config_spec() + expected.memoryReservationLockedToMax = False + self.assertEqual(expected, result) def test_get_vm_create_spec_with_serial_port(self): @@ -707,6 +709,9 @@ def test_get_vm_create_spec_with_serial_port(self): serial_port_spec = vm_util.create_serial_port_spec(fake_factory) expected = self._create_vm_config_spec() expected.deviceChange = [serial_port_spec] + + expected.memoryReservationLockedToMax = False + self.assertEqual(expected, result) def test_get_vm_create_spec_with_allocations(self): @@ -758,6 +763,8 @@ def test_get_vm_create_spec_with_allocations(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) + expected.memoryReservationLockedToMax = False + self.assertEqual(expected, result) def test_get_vm_create_spec_with_limit(self): @@ -808,6 +815,7 @@ def test_get_vm_create_spec_with_limit(self): cpu_allocation.shares.level = 'normal' cpu_allocation.shares.shares = 0 expected.cpuAllocation = cpu_allocation + expected.memoryReservationLockedToMax = False expected.numCPUs = 2 self.assertEqual(expected, result) @@ -845,6 +853,8 @@ def test_get_vm_create_spec_with_share(self): expected.managedBy.type = 'instance' expected.managedBy.extensionKey = 'org.openstack.compute' + expected.memoryReservationLockedToMax = False + expected.version = None expected.guestId = constants.DEFAULT_OS_TYPE @@ -897,6 +907,8 @@ def test_get_vm_create_spec_with_share_custom(self): expected.managedBy.extensionKey = 'org.openstack.compute' expected.managedBy.type = 'instance' + expected.memoryReservationLockedToMax = False + expected.version = None expected.guestId = constants.DEFAULT_OS_TYPE expected.tools = fake_factory.create('ns0:ToolsConfigInfo') @@ -952,6 +964,8 @@ def test_get_vm_create_spec_with_metadata(self): expected.managedBy.extensionKey = 'org.openstack.compute' expected.managedBy.type = 'instance' + expected.memoryReservationLockedToMax = False + expected.tools = fake_factory.create('ns0:ToolsConfigInfo') expected.tools.afterPowerOn = True expected.tools.afterResume = True @@ -994,6 +1008,7 @@ def test_get_vm_create_spec_with_firmware(self): expected.managedBy = fake_factory.create('ns0:ManagedByInfo') expected.managedBy.extensionKey = 'org.openstack.compute' expected.managedBy.type = 'instance' + expected.memoryReservationLockedToMax = False expected.tools = fake_factory.create('ns0:ToolsConfigInfo') expected.tools.afterPowerOn = True @@ -1804,6 +1819,7 @@ def test_get_vm_create_spec_with_console_delay(self): expected.tools.beforeGuestReboot = True expected.tools.beforeGuestShutdown = True expected.tools.beforeGuestStandby = True + expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -1847,6 +1863,8 @@ def test_get_vm_create_spec_with_cores_per_socket(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) + expected.memoryReservationLockedToMax = False + self.assertEqual(expected, result) def test_get_vm_create_spec_with_memory_allocations(self): @@ -1898,6 +1916,8 @@ def test_get_vm_create_spec_with_memory_allocations(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) + expected.memoryReservationLockedToMax = False + self.assertEqual(expected, result) def test_get_swap(self): diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index f98a931bb42..46811b11d02 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -234,6 +234,8 @@ def get_vm_create_spec(client_factory, instance, data_store_name, client_factory, extra_specs.memory_limits, 'ns0:ResourceAllocationInfo') + config_spec.memoryReservationLockedToMax = CONF.vmware.reserve_all_memory + if extra_specs.firmware: config_spec.firmware = extra_specs.firmware From d20849ffcee2b7c75badbb4aefc05c8d4234f56f Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 24 Jul 2019 11:58:24 +0200 Subject: [PATCH 27/92] Add custom scheduler, memory reservation and DRS overrides for big VMs. Add global setting `bigvm_mb` This is necessary in multiple places e.g. the scheduler and compute-driver to handle big VMs differently. We cannot currently use flavors for that, because there are still custom flavors in the system that we don't control. (cherry picked from commit 881889d41bf25f453769c92930482fcdff8a21d2) [scheduler] Add BigVmFilter The filter filters out hosts having more than `bigvm_memory_utilization_max` memory utilization. This is necessary, because it takes too long to find free space on a host for the big VM otherwise. (cherry picked from commit d284fc330304f5b072929d9228913030532ac486) [vmware] Add DRS override for "big" VMs Since DRS doesn't play well with large differences in VM size and HANA doesn't play well with vMotion, we're disabling automatic re-balancing actions for "big" VMs by the DRS. On spawn, we add a `ClusterDrsVmConfigSpec` for instances having more than by default 1 TB of memory - configurable via `bigvm_mb`` configuration variable. The vCenter deletes the overrides automatically when a VM is deleted. One can view current overrides in the vCenter UI via $cluster (e.g. productionbb098) -> Configure -> VM Overrides. We're always adding the rule in `update_cluster_drs_vm_override`, because we only call it in the driver's `spawn` method which seems to be called only when a new vm object is created. Therefore, there should not be any existing rule for the VM. (cherry picked from commit ae68374a7916f11dfb715e95ae3ba6b3ebcfc514) [vmware] Always reserve all memory for big VMs Since there are still custom flavors in use for big VMs, we cannot rely on the flavors to set the reserved memory for big VMs, which is necessary because otherwise small VMs will compete with the big VMs' HANA, which does not work well. We're using `CONF.bigvm_mb` here to identify a big VM and reserve all memory regardless of `CONF.reserve_all_memory`. (cherry picked from commit 706bcd8aee107b33338303974d2c08ceda2f9499) [scheduler] Make BigVmFilter more dynamic Instead of having a static threshold for the `BigVmFilter`, we compute the threshold depending on the hypervisor size and the requested RAM of the big VM. The reasoning behind this is, that on bigger hypervisors a small big VM doesn't matter as much as a big big VM, e.g. 1.4 TB on a 6 TB hypervisor vs. 5.9 TB on a 6 TB hypervisor. We basically aim at 50 % of the requested RAM free on average over all hosts in the cluster. The filter thus currently only works if a compute-node is in a host aggregate defining the `_AGGREGATE_KEY` (currently "hv_size_mb") having an integer defining the hypervisor's RAM size as a value. (cherry picked from commit 6ad6a2c0300b0a6d2d4f08e94b23cc4e6cf1db1a) [scheduler] Add BigVmHypervisorRamFilter Since we've got multiple BigVM-filters now, this also renames `BigVmFilter` to `BigVmClusterUtilizationFilter` for clarity. With having more diversity in the hypervisor size (e.g. 6 TB hypervisors right around the corner), we have to make sure big VMs actually fit the hypervisor. For that, we introduce a new scheduler filter - `BigVmHypervisorRamFilter` - that prohibits scheduling of big VMs not fitting the hypervisor. In KVM-deployments, this would be the job of the `RamFilter`, but since we run VMware and the scheduler only sees the cluster's and not a single host's resources, we use a special host aggregate attribute - "hv_size_mb" - to retrieve the hypervisor size as already done for `BigVmClusterUtilizationFilter`. (cherry picked from commit 4b6867a4a846726df80830ca9d8d5e494d26243e) [scheduler] pep8 cleanup (cherry picked from commit 39231e4f8b775617de1039922e636dd1014cee13) Add `is_big_vm` utility function This function should be used to check whether one is handling a big VM. There, all necessary conditions are gathered in on place so one cannot forget a condition. This also makes it easy to change the conditions later on. The conditions we use to check for baremetal flavors are the same that `nova-scheduler` uses. (cherry picked from commit cc46cf4ab3f420021ea7e527a369993db5806f68) [scheduler] Remove BigVmHypervisorRamFilter We actually don't need it, because our filter queries the placement API for resource providers capable of fulfilling the request. With the vmwareapi driver setting the `max_unit` value for `MEMORY_MB` to the cluster's maximum host size, we don't have to add another filter into the scheduler, as too small hosts won't even end up as scheduling options. (cherry picked from commit c2fc2810495fd8182fe3aebfa98392c1fa40af4b) [scheduler] Use placement API to get HV size We recently discovered, that the hypervisor size is actually available in the placement API. Every compute host has a resource provider with an inventory. There, the vmwareapi virt driver sets the `max_unit` for the `MEMORY_MB` resource class to the biggest host found in the cluster. This means, we don't have to use aggregates to get the hypervisor size in the BigVm filters, but can retrieve it from the placement API. We're building a cache of the value, that expires after 10 min. This guards the case, where a lot of VMs are scheduled in a short period of time - even though tests showed, that querying the inventory of all hosts in QA did not take longer than a second. Cache expiration also guards against stale values for newly-created hypervisors that did not set their inventory, yet. (cherry picked from commit 965d3b46010aac34f04c2556dc39a0ac0dfd49d5) [vmware] Make `partiallyAutomated` a constant We usually use variables as constants instead of directly embedding strings. (cherry picked from commit 81097b05f87699e6f5a48cbe206c7d7445a2ae6b) (cherry picked from commit fbf0a2d9867c705ae95c006fc1e7897d2e936f9a) Change-Id: Idca77aae46d67623f26397ba5ea0dded2892fc05 (cherry picked from commit a3db710027007babec835d9a1d2054006173ef69) --- nova/conf/base.py | 11 ++ nova/conf/vmware.py | 5 +- nova/scheduler/filters/bigvm_filter.py | 101 ++++++++++ .../scheduler/filters/test_bigvm_filters.py | 179 ++++++++++++++++++ nova/utils.py | 12 ++ nova/virt/vmwareapi/constants.py | 2 + nova/virt/vmwareapi/vm_util.py | 6 +- nova/virt/vmwareapi/vmops.py | 10 + 8 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 nova/scheduler/filters/bigvm_filter.py create mode 100644 nova/tests/unit/scheduler/filters/test_bigvm_filters.py diff --git a/nova/conf/base.py b/nova/conf/base.py index 9e5d9e76e20..dd2730f8fa6 100644 --- a/nova/conf/base.py +++ b/nova/conf/base.py @@ -68,6 +68,17 @@ The total number of coroutines that can be run via nova's default greenthread pool concurrently, defaults to 1000, min value is 100. '''), + cfg.IntOpt( + 'bigvm_mb', + default=1024 ** 2, # 1 TB + min=0, + help=""" +Instance memory usage identifying it as big VM + +For a couple of operations, e.g. scheduling decisions and special settings when +spawning, we have to identify a big VM and handle them differently. Every VM +having more or equal to this setting's amount of RAM is a big VM. +"""), ] metrics_opts = [ diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index 103a7bffa74..cd8225d1ea9 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -100,7 +100,10 @@ If true, it is not possible to over-commit memory in the vCenter, but there's also no swap file pre-created on the ephemeral storage. Swap still works, but is hot-created. -""") + +For big VMs as determined by the `bigvm_mb` setting, this setting is not used. +Big VMs always reserve all their memory. +"""), ] vmwareapi_opts = [ diff --git a/nova/scheduler/filters/bigvm_filter.py b/nova/scheduler/filters/bigvm_filter.py new file mode 100644 index 00000000000..354480318ba --- /dev/null +++ b/nova/scheduler/filters/bigvm_filter.py @@ -0,0 +1,101 @@ +# Copyright (c) 2019 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from oslo_log import log as logging + +import nova.conf +from nova import context +from nova.scheduler.client import report +from nova.scheduler import filters +from nova.utils import is_big_vm + +LOG = logging.getLogger(__name__) + +CONF = nova.conf.CONF + + +class BigVmBaseFilter(filters.BaseHostFilter): + + _HV_SIZE_CACHE = {} + _HV_SIZE_CACHE_RETENTION_TIME = 10 * 60 + + RUN_ON_REBUILD = False + + def _get_hv_size(self, host_state): + # expire the cache 10min after last write + time_diff = time.time() - self._HV_SIZE_CACHE.get('last_modified', 0) + if time_diff > self._HV_SIZE_CACHE_RETENTION_TIME: + self._HV_SIZE_CACHE = {} + + if host_state.uuid not in self._HV_SIZE_CACHE: + placement_client = report.SchedulerReportClient() + elevated = context.get_admin_context() + res = placement_client._get_inventory(elevated, host_state.uuid) + if not res: + return None + inventories = res.get('inventories', {}) + hv_size_mb = inventories.get('MEMORY_MB', {}).get('max_unit') + self._HV_SIZE_CACHE[host_state.uuid] = hv_size_mb + + self._HV_SIZE_CACHE['last_modified'] = time.time() + + return self._HV_SIZE_CACHE[host_state.uuid] + + +class BigVmClusterUtilizationFilter(BigVmBaseFilter): + """Only schedule big VMs to a vSphere cluster (i.e. nova-compute host) if + the memory-utilization of the cluster is below a threshold depending on the + hypervisor size and the requested memory. + """ + + def _get_max_ram_percent(self, requested_ram_mb, hypervisor_ram_mb): + """We want the hosts to have on average half the requested memory free. + """ + requested_ram_mb = float(requested_ram_mb) + hypervisor_ram_mb = float(hypervisor_ram_mb) + hypervisor_max_used_ram_mb = hypervisor_ram_mb - requested_ram_mb / 2 + return hypervisor_max_used_ram_mb / hypervisor_ram_mb * 100 + + def host_passes(self, host_state, spec_obj): + requested_ram_mb = spec_obj.memory_mb + # not scheduling a big VM -> every host is fine + if not is_big_vm(requested_ram_mb, spec_obj.flavor): + return True + + hypervisor_ram_mb = self._get_hv_size(host_state) + if hypervisor_ram_mb is None: + return False + + free_ram_mb = host_state.free_ram_mb + total_usable_ram_mb = host_state.total_usable_ram_mb + used_ram_mb = total_usable_ram_mb - free_ram_mb + used_ram_percent = float(used_ram_mb) / total_usable_ram_mb * 100.0 + + max_ram_percent = self._get_max_ram_percent(requested_ram_mb, + hypervisor_ram_mb) + + if used_ram_percent > max_ram_percent: + LOG.info("%(host_state)s does not have less than " + "%(max_ram_percent)s %% RAM utilization (has " + "%(used_ram_percent)s %%) and is thus not suitable " + "for big VMs.", + {'host_state': host_state, + 'max_ram_percent': max_ram_percent, + 'used_ram_percent': used_ram_percent}) + return False + + return True diff --git a/nova/tests/unit/scheduler/filters/test_bigvm_filters.py b/nova/tests/unit/scheduler/filters/test_bigvm_filters.py new file mode 100644 index 00000000000..711151edb79 --- /dev/null +++ b/nova/tests/unit/scheduler/filters/test_bigvm_filters.py @@ -0,0 +1,179 @@ +# Copyright (c) 2019 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time +from unittest import mock + +import nova.conf +from nova import objects +from nova.scheduler.filters import bigvm_filter +from nova import test +from nova.tests.unit.scheduler import fakes +from oslo_utils.fixture import uuidsentinel + +CONF = nova.conf.CONF + + +@mock.patch('nova.scheduler.client.report.' + 'SchedulerReportClient._get_inventory') +class TestBigVmBaseFilter(test.NoDBTestCase): + + def setUp(self): + super(TestBigVmBaseFilter, self).setUp() + self.filt_cls = bigvm_filter.BigVmBaseFilter() + self.hv_size = CONF.bigvm_mb + 1024 + + def test_big_vm_host_without_inventory(self, mock_inv): + mock_inv.return_value = {} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.assertIsNone(self.filt_cls._get_hv_size(host)) + + def test_big_vm_host_with_placement_error(self, mock_inv): + mock_inv.return_value = None + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.assertIsNone(self.filt_cls._get_hv_size(host)) + + def test_big_vm_host_with_empty_inventory(self, mock_inv): + mock_inv.return_value = {'inventories': {}} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.assertIsNone(self.filt_cls._get_hv_size(host)) + + def test_big_vm_get_hv_size_with_cache(self, mock_inv): + mock_inv.return_value = {} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + self.filt_cls._HV_SIZE_CACHE = { + host.uuid: 1234, + 'last_modified': time.time() + } + self.assertEqual(self.filt_cls._get_hv_size(host), 1234) + + def test_big_vm_get_hv_size_cache_timeout(self, mock_inv): + mock_inv.return_value = {'inventories': {'MEMORY_MB': + {'max_unit': 23}}} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': self.hv_size, + 'total_usable_ram_mb': self.hv_size, + 'uuid': uuidsentinel.host1}) + mod = time.time() - self.filt_cls._HV_SIZE_CACHE_RETENTION_TIME + self.filt_cls._HV_SIZE_CACHE = { + host.uuid: 1234, + 'last_modified': mod + } + self.assertEqual(self.filt_cls._get_hv_size(host), 23) + + +class TestBigVmClusterUtilizationFilter(test.NoDBTestCase): + + def setUp(self): + super(TestBigVmClusterUtilizationFilter, self).setUp() + self.hv_size = CONF.bigvm_mb + 1024 + self.filt_cls = bigvm_filter.BigVmClusterUtilizationFilter() + self.filt_cls._HV_SIZE_CACHE = { + uuidsentinel.host1: self.hv_size, + 'last_modified': time.time() + } + + def test_big_vm_with_small_vm_passes(self): + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=1024, extra_specs={})) + host = fakes.FakeHostState('host1', 'compute', {}) + self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + + def test_baremetal_instance_passes(self): + extra_specs = {'capabilities:cpu_arch': 'x86_64'} + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, + extra_specs=extra_specs)) + host = fakes.FakeHostState('host1', 'compute', {}) + self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_hv_size(self): + """If there's no inventory for this host, it should not even have + passed placement API checks, so we stop it here. + """ + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1}) + self.filt_cls._HV_SIZE_CACHE[host.uuid] = None + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_enough_ram(self): + # there's enough RAM available in the cluster but not enough (~50 % of + # the requested size on average + # 12 hosts (bigvm + 1 GB size) + # 11 big VM + some smaller (12 * 1 GB) already deployed + # -> still bigvm_mb left, but ram utilization ratio of all hosts is too + # high + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': CONF.bigvm_mb, + 'total_usable_ram_mb': total_ram}) + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_enough_ram_ignores_ram_ratio(self): + # same as test_big_vm_without_enough_ram but with more theoretical RAM + # via `ram_allocation_ratio`. big VMs reserve all memory so the ratio + # does not count for them. + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': CONF.bigvm_mb, + 'total_usable_ram_mb': total_ram, + 'ram_allocation_ratio': 1.5}) + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_without_enough_ram_percent(self): + # there's just closely not enough RAM available + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + hv_percent = self.filt_cls._get_max_ram_percent(CONF.bigvm_mb, + self.hv_size) + free_ram_mb = total_ram - (total_ram * hv_percent / 100.0) - 128 + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': free_ram_mb, + 'total_usable_ram_mb': total_ram}) + self.assertFalse(self.filt_cls.host_passes(host, spec_obj)) + + def test_big_vm_with_enough_ram(self): + spec_obj = objects.RequestSpec( + flavor=objects.Flavor(memory_mb=CONF.bigvm_mb, extra_specs={})) + total_ram = self.hv_size * 12 + hv_percent = self.filt_cls._get_max_ram_percent(CONF.bigvm_mb, + self.hv_size) + host = fakes.FakeHostState('host1', 'compute', + {'uuid': uuidsentinel.host1, + 'free_ram_mb': total_ram - (total_ram * hv_percent / 100.0), + 'total_usable_ram_mb': total_ram}) + self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) diff --git a/nova/utils.py b/nova/utils.py index 429cf964fcc..3e3f41eb807 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -1292,3 +1292,15 @@ def is_baremetal_host(host_state): def is_baremetal_flavor(flavor): return 'capabilities:cpu_arch' in flavor.extra_specs + + +def is_big_vm(memory_mb, flavor): + # small VMs are not big + if memory_mb < CONF.bigvm_mb: + return False + + # baremetal instances are not big + if is_baremetal_flavor(flavor): + return False + + return True diff --git a/nova/virt/vmwareapi/constants.py b/nova/virt/vmwareapi/constants.py index 11a74cd4a3b..f9a4d6d3837 100644 --- a/nova/virt/vmwareapi/constants.py +++ b/nova/virt/vmwareapi/constants.py @@ -53,6 +53,8 @@ DEFAULT_DISK_FORMAT = DISK_FORMAT_VMDK DEFAULT_CONTAINER_FORMAT = CONTAINER_FORMAT_BARE +DRS_BEHAVIOR_PARTIALLY_AUTOMATED = 'partiallyAutomated' + IMAGE_VM_PREFIX = "OSTACK_IMG" SNAPSHOT_VM_PREFIX = "OSTACK_SNAP" diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 46811b11d02..84a6f31f514 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -34,6 +34,7 @@ from nova import exception from nova.i18n import _ from nova.network import model as network_model +from nova.utils import is_big_vm from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import session @@ -234,7 +235,10 @@ def get_vm_create_spec(client_factory, instance, data_store_name, client_factory, extra_specs.memory_limits, 'ns0:ResourceAllocationInfo') - config_spec.memoryReservationLockedToMax = CONF.vmware.reserve_all_memory + reservation_lock = CONF.vmware.reserve_all_memory \ + or is_big_vm(int(instance.memory_mb), + instance.flavor) + config_spec.memoryReservationLockedToMax = reservation_lock if extra_specs.firmware: config_spec.firmware = extra_specs.firmware diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index eebad0817ee..663dd4fce5d 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -855,6 +855,16 @@ def spawn(self, context, instance, image_meta, injected_files, # 'uuid' of the instance vm_util.rename_vm(self._session, vm_ref, instance) + # Make sure we don't automatically move around "big" VMs + if utils.is_big_vm(int(instance.memory_mb), instance.flavor): + behavior = constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED + LOG.debug("Adding DRS override '%s' for big VM.", behavior, + instance=instance) + cluster_util.update_cluster_drs_vm_override(self._session, + self._cluster, + vm_ref, + behavior) + vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) def _is_bdm_valid(self, block_device_mapping): From 419ede6256a136a434219f1200a66702eb90d12d Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 26 Mar 2020 08:55:57 +0100 Subject: [PATCH 28/92] [vmware] Remove/Add big VM settings on resize [vmware] Update memoryReservationLockedToMax on resize We need to update this setting in the case where CONF.vmware.reserve_all_memory is False (the default) and we downsize a big VM or upsize a normal VM to a big VM. To make it simple, we just recompute the setting every time and set it in the reconfig spec no matter what the previous state was. Change-Id: Ic9283af63cb3bf90f0bed007474d7b6583866003 (cherry picked from commit 3bca30203aae5b5d727acef4e71f0f35d119d308) [vmware] Support removal of drs overrides This commit changes `update_cluster_drs_vm_override()` to be able to remove overrides again. This is not necessary for deletion of a VM, but we need to remove it if a VM gets resized under the big VM limit. If there is no override, the task will fail with `InvalidArgument`. Change-Id: If1de5136cbc7c59babac24e8293ddb7a12ea8411 (cherry picked from commit 7ab2765927d28c14ac4830289233002df26cac9f) [vmware] Remove/Add DRS overrides on resize A VM becoming a big VM needs to get an DRS override so it doesn't move around automatically, because that might be bad for performance. A VM no longer being a big VM needs to get its DRS override removed, because this interferes with spawning new VMs - especially the special_spawning code implemented for big VMs. We're currently unable to remove a DRS override as our SOAP library is unable to create an appropriate request accepted by the vCenter. Therefore, we still allow the resize to work for now and have to manually remove the override later on. We add code for the removal, so we get a reminder in the logs and sentry, that this needs fixing. We just don't fail the resize for it. Change-Id: I89aedc29e36c86415a205b7181e06b84316bb82e (cherry picked from commit eaad29ef7db048c2ca430c79ecdde0a8868999a8) (cherry picked from commit 21b26dfc5ba69ac9e07b6ab499712db949299177) Incorporated-Change-Id: I4d344347860c7d97d6f4b2e68d9bbac069d71b74 (cherry picked from commit 860c0d50822223e4ba66c2525725f4d14bc98517) --- .../tests/unit/virt/vmwareapi/test_vm_util.py | 8 ++- nova/tests/unit/virt/vmwareapi/test_vmops.py | 72 ++++++++++++++++++- nova/virt/vmwareapi/vm_util.py | 4 +- nova/virt/vmwareapi/vmops.py | 39 +++++++++- 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 21e02a05fd8..376d77d2373 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -185,7 +185,8 @@ def test_get_resize_spec(self): extra_specs = vm_util.ExtraSpecs() fake_factory = fake.FakeFactory() result = vm_util.get_vm_resize_spec(fake_factory, - vcpus, memory_mb, extra_specs) + vcpus, memory_mb, extra_specs, + memory_reservation_locked=False) expected = fake_factory.create('ns0:VirtualMachineConfigSpec') expected.memoryMB = memory_mb expected.numCPUs = vcpus @@ -196,6 +197,7 @@ def test_get_resize_spec(self): cpuAllocation.shares.level = 'normal' cpuAllocation.shares.shares = 0 expected.cpuAllocation = cpuAllocation + expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -207,7 +209,8 @@ def test_get_resize_spec_with_limits(self): extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits) fake_factory = fake.FakeFactory() result = vm_util.get_vm_resize_spec(fake_factory, - vcpus, memory_mb, extra_specs) + vcpus, memory_mb, extra_specs, + memory_reservation_locked=True) expected = fake_factory.create('ns0:VirtualMachineConfigSpec') expected.memoryMB = memory_mb expected.numCPUs = vcpus @@ -218,6 +221,7 @@ def test_get_resize_spec_with_limits(self): cpuAllocation.shares.level = 'normal' cpuAllocation.shares.shares = 0 expected.cpuAllocation = cpuAllocation + expected.memoryReservationLockedToMax = True self.assertEqual(expected, result) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 44bd33dadbe..e1cc9252644 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -25,6 +25,7 @@ from oslo_vmware import vim_util as vutil from nova.compute import power_state +import nova.conf from nova import context from nova import exception from nova.network import model as network_model @@ -34,8 +35,10 @@ from nova.tests.unit import fake_instance from nova.tests.unit.virt.vmwareapi import fake as vmwareapi_fake from nova.tests.unit.virt.vmwareapi import stubs +from nova import utils from nova import version from nova.virt import hardware +from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import images @@ -45,6 +48,8 @@ from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops +CONF = nova.conf.CONF + class DsPathMatcher(object): def __init__(self, expected_ds_path_str): @@ -773,6 +778,7 @@ def _test_finish_revert_migration(self, fake_power_on, int(self._instance.vcpus), int(self._instance.memory_mb), extra_specs, + CONF.vmware.reserve_all_memory, metadata=metadata) fake_reconfigure_vm.assert_called_once_with(self._session, 'fake-ref', @@ -947,17 +953,77 @@ def test_resize_vm(self, fake_resize_spec, fake_reconfigure, memory_mb=1024, vcpus=2, extra_specs={}) - self._vmops._resize_vm(self._context, self._instance, 'vm-ref', flavor, + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() + self._vmops._resize_vm(self._context, instance, 'vm-ref', flavor, None) fake_get_metadata.assert_called_once_with(self._context, - self._instance, + instance, flavor=flavor) fake_resize_spec.assert_called_once_with( self._session.vim.client.factory, 2, 1024, extra_specs, - metadata=self._metadata) + CONF.vmware.reserve_all_memory, metadata=self._metadata) fake_reconfigure.assert_called_once_with(self._session, 'vm-ref', 'fake-spec') + @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') + @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') + @mock.patch.object(vm_util, 'reconfigure_vm') + @mock.patch.object(vm_util, 'get_vm_resize_spec', + return_value='fake-spec') + @mock.patch.object(utils, 'is_big_vm') + @mock.patch.object(cluster_util, 'update_cluster_drs_vm_override') + def test_resize_vm_bigvm_upsize(self, fake_drs_override, fake_is_big_vm, + fake_resize_spec, fake_reconfigure, + fake_get_extra_specs, fake_get_metadata): + # new is big, new is big, old is not + fake_is_big_vm.side_effect = [True, True, False] + extra_specs = vm_util.ExtraSpecs() + fake_get_extra_specs.return_value = extra_specs + fake_get_metadata.return_value = self._metadata + flavor = objects.Flavor(name='m1.small', + memory_mb=1024, + vcpus=2, + extra_specs={}) + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() + self._vmops._resize_vm(self._context, instance, 'vm-ref', flavor, + None) + behavior = constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED + fake_drs_override.assert_called_once_with(self._session, + self._cluster.obj, + 'vm-ref', + operation='add', + behavior=behavior) + + @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') + @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') + @mock.patch.object(vm_util, 'reconfigure_vm') + @mock.patch.object(vm_util, 'get_vm_resize_spec', + return_value='fake-spec') + @mock.patch.object(utils, 'is_big_vm') + @mock.patch.object(cluster_util, 'update_cluster_drs_vm_override') + def test_resize_vm_bigvm_downsize(self, fake_drs_override, fake_is_big_vm, + fake_resize_spec, fake_reconfigure, + fake_get_extra_specs, fake_get_metadata): + # new is not big, new is not big, old is big + fake_is_big_vm.side_effect = [False, False, True] + extra_specs = vm_util.ExtraSpecs() + fake_get_extra_specs.return_value = extra_specs + fake_get_metadata.return_value = self._metadata + flavor = objects.Flavor(name='m1.small', + memory_mb=1024, + vcpus=2, + extra_specs={}) + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() + self._vmops._resize_vm(self._context, instance, 'vm-ref', flavor, + None) + fake_drs_override.assert_called_once_with(self._session, + self._cluster.obj, + 'vm-ref', + operation='remove') + @mock.patch.object(vmops.VMwareVMOps, '_extend_virtual_disk') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(ds_util, 'disk_move') diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 84a6f31f514..6e0f4e7340f 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -361,7 +361,7 @@ def get_vm_boot_spec(client_factory, device): def get_vm_resize_spec(client_factory, vcpus, memory_mb, extra_specs, - metadata=None): + memory_reservation_locked, metadata=None): """Provides updates for a VM spec.""" resize_spec = client_factory.create('ns0:VirtualMachineConfigSpec') resize_spec.numCPUs = vcpus @@ -369,6 +369,8 @@ def get_vm_resize_spec(client_factory, vcpus, memory_mb, extra_specs, resize_spec.cpuAllocation = _get_allocation_info( client_factory, extra_specs.cpu_limits, 'ns0:ResourceAllocationInfo') + resize_spec.memoryReservationLockedToMax = memory_reservation_locked + if metadata: resize_spec.annotation = metadata return resize_spec diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 663dd4fce5d..9575c7e8ea6 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -863,7 +863,8 @@ def spawn(self, context, instance, image_meta, injected_files, cluster_util.update_cluster_drs_vm_override(self._session, self._cluster, vm_ref, - behavior) + operation='add', + behavior=behavior) vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) @@ -1419,15 +1420,47 @@ def _resize_vm(self, context, instance, vm_ref, flavor, image_meta): """Resizes the VM according to the flavor.""" client_factory = self._session.vim.client.factory extra_specs = self._get_extra_specs(flavor, image_meta) + reservation_locked = CONF.vmware.reserve_all_memory \ + or utils.is_big_vm(int(flavor.memory_mb), + flavor) metadata = self._get_instance_metadata(context, instance, flavor=flavor) vm_resize_spec = vm_util.get_vm_resize_spec(client_factory, int(flavor.vcpus), int(flavor.memory_mb), extra_specs, + reservation_locked, metadata=metadata) vm_util.reconfigure_vm(self._session, vm_ref, vm_resize_spec) + old_flavor = instance.old_flavor + new_is_big = utils.is_big_vm(int(old_flavor.memory_mb), old_flavor) + old_is_big = utils.is_big_vm(int(flavor.memory_mb), flavor) + + if not old_is_big and new_is_big: + # Make sure we don't automatically move around "big" VMs + behavior = constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED + LOG.debug("Adding DRS override '%s' for big VM.", behavior, + instance=instance) + cluster_util.update_cluster_drs_vm_override(self._session, + self._cluster, + vm_ref, + operation='add', + behavior=behavior) + elif old_is_big and not new_is_big: + # remove the old override, if we had one before. make sure we don't + # error out if it was already deleted another way + LOG.debug("Removing DRS override for former big VM.", + instance=instance) + try: + cluster_util.update_cluster_drs_vm_override(self._session, + self._cluster, + vm_ref, + operation='remove') + except Exception: + LOG.exception('Could not remove DRS override.', + instance=instance) + def _resize_disk(self, instance, vm_ref, vmdk, flavor): extra_specs = self._get_extra_specs(instance.flavor, instance.image_meta) @@ -1600,12 +1633,16 @@ def finish_revert_migration(self, context, instance, network_info, # Reconfigure the VM properties extra_specs = self._get_extra_specs(instance.flavor, instance.image_meta) + reservation_locked = CONF.vmware.reserve_all_memory \ + or utils.is_big_vm(int(instance.flavor.memory_mb), + instance.flavor) metadata = self._get_instance_metadata(context, instance) vm_resize_spec = vm_util.get_vm_resize_spec( client_factory, int(instance.flavor.vcpus), int(instance.flavor.memory_mb), extra_specs, + reservation_locked, metadata=metadata) vm_util.reconfigure_vm(self._session, vm_ref, vm_resize_spec) From 34700da84b95921bb0a7bb470c3e9b2917139e69 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 28 Oct 2019 14:43:45 +0100 Subject: [PATCH 29/92] Add function to check for special spawning needs Some of our VMs have special spawning needs, which means they need to spawn on their own hypervisor at first. This function helps us define these VMs in a central place. (cherry picked from commit e9829a74ee9c93e6d0cdae71d4bddf07accf3301) (cherry picked from commit 6f4ed6ae76c194ff7860f9e7869617e71924c684) (cherry picked from commit 14919e9d09be752fd35fa3ba36a3a5de16798aa9) --- nova/utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nova/utils.py b/nova/utils.py index 3e3f41eb807..877f8efe02e 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -1304,3 +1304,16 @@ def is_big_vm(memory_mb, flavor): return False return True + + +def vm_needs_special_spawning(memory_mb, flavor): + if is_big_vm(memory_mb, flavor): + return True + + if is_baremetal_flavor(flavor): + return False + + if flavor.extra_specs.get('spawn_on_free_host', 'false').lower() == 'true': + return True + + return False From 96d302c19e327a10a678631baf98b4bc042b701a Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 27 Sep 2021 11:25:44 +0200 Subject: [PATCH 30/92] Vmware: Refactor update_cluster_placement After the recent changes of syncing server-groups, the syncing of rules in cluster_util.update_placement has become unused. The code can be simplified as a preparation for live-migration, as it will add complexity. VMops.update_cluster_placement now calls - sync_instance_server_group, which fetches _the_ server-group for the instance and syncs it with the existing VMOps.sync_server_group - update_admin_vm_group_membership handles the membership to special_spawning_vm_group Since that is the only remaining use-case of cluster_util.update_placement, the function has been renamed to update_vm_group_membership and reduced to that use-case Change-Id: I94c311790610fc4658f9b06ac052ad46660f1cea (cherry picked from commit a05999bc206923dc41bfd65d5a9ac6d8125ee816) (cherry picked from commit 673a3dba5f63066211f1526969c92e1ef9ca03b8) --- nova/virt/vmwareapi/cluster_util.py | 99 +++++++++-------------------- nova/virt/vmwareapi/vm_util.py | 8 --- nova/virt/vmwareapi/vmops.py | 60 ++++++++--------- 3 files changed, 56 insertions(+), 111 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 559856db432..bfa3aebad3d 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -20,7 +20,6 @@ from nova import exception from nova.i18n import _ from nova import utils -from nova.virt.vmwareapi import constants LOG = logging.getLogger(__name__) @@ -33,13 +32,10 @@ def reconfigure_cluster(session, cluster, config_spec): session.wait_for_task(reconfig_task) -def create_vm_group(client_factory, name, vm_refs, group=None): +def create_vm_group(client_factory, name, vm_refs): """Create a ClusterVmGroup object - - :param:group: if given, update this ClusterVmGroup object instead of - creating a new one """ - group = group or client_factory.create('ns0:ClusterVmGroup') + group = client_factory.create('ns0:ClusterVmGroup') group.name = name group.vm = vm_refs @@ -78,27 +74,13 @@ def create_group_spec(client_factory, group, operation): return group_spec -def _create_vm_group_spec(client_factory, group_info, vm_refs, - operation="add", group=None): - if group: - # On vCenter UI, it is not possible to create VM group without - # VMs attached to it. But, using APIs, it is possible to create - # VM group without VMs attached. Therefore, check for existence - # of vm attribute in the group to avoid exceptions - if hasattr(group, 'vm'): - vm_refs = vm_refs + group.vm - - group = create_vm_group(client_factory, group_info.name, vm_refs, group) - - return create_group_spec(client_factory, group, operation) - - -def _get_vm_group(cluster_config, group_info): +def _get_vm_group(cluster_config, group_name): if not hasattr(cluster_config, 'group'): - return + return None for group in cluster_config.group: - if group.name == group_info.name: + if group.name == group_name: return group + return None def fetch_cluster_groups(session, cluster_ref=None, cluster_config=None, @@ -159,63 +141,44 @@ def delete_vm_group(session, cluster, vm_group): last vm in a vm group """ client_factory = session.vim.client.factory - groups = [] group_spec = create_group_spec(client_factory, vm_group, "remove") - groups.append(group_spec) config_spec = client_factory.create('ns0:ClusterConfigSpecEx') - config_spec.groupSpec = groups + config_spec.groupSpec = [group_spec] + reconfigure_cluster(session, cluster, config_spec) @utils.synchronized('vmware-vm-group-policy') -def update_placement(session, cluster, vm_ref, group_infos): - """Updates cluster for vm placement using DRS""" +def update_vm_group_membership(session, cluster, vm_group_name, vm_ref): + """Updates cluster for vm placement using DRS + + Add a VM to a Vm-group, create it if missing + It is up for an administrator to define rules for this group with other + means in the VCenter + """ cluster_config = session._call_method( vutil, "get_object_property", cluster, "configurationEx") client_factory = session.vim.client.factory config_spec = client_factory.create('ns0:ClusterConfigSpecEx') - config_spec.groupSpec = [] - config_spec.rulesSpec = [] - for group_info in group_infos: - if not group_info.name.startswith(constants.DRS_PREFIX): - # We only do this, if this is an admin-defined group, because - # VmGroups are not used by the rules created by Nova. - group = _get_vm_group(cluster_config, group_info) - - if not group: - # Creating group - operation = "add" - else: - # VM group exists on the cluster which is assumed to be - # created by VC admin. Add instance to this vm group and let - # the placement policy defined by the VC admin take over - operation = "edit" - group_spec = _create_vm_group_spec( - client_factory, group_info, [vm_ref], operation=operation, - group=group) - config_spec.groupSpec.append(group_spec) - - # If server group policies are defined (by tenants), then - # create/edit affinity/anti-affinity rules on cluster. - # Note that this might be add-on to the existing vm group - # (mentioned above) policy defined by VC admin i.e if VC admin has - # restricted placement of VMs to a specific group of hosts, then - # the server group policy from nova might further restrict to - # individual hosts on a cluster - if group_info.policies: - # VM group does not exist on cluster - policy = group_info.policies[0] - if policy != 'soft-affinity': - rule_name = "%s-%s" % (group_info.name, policy) - rule = _get_rule(cluster_config, rule_name) - operation = "edit" if rule else "add" - rules_spec = _create_cluster_rules_spec( - client_factory, rule_name, [vm_ref], policy=policy, - operation=operation, rule=rule) - config_spec.rulesSpec.append(rules_spec) + + operation = None + group = _get_vm_group(cluster_config, vm_group_name) + + if not group: + operation = "add" + group = create_vm_group(client_factory, vm_group_name, [vm_ref]) + else: + operation = "edit" + if not hasattr(group, "vm"): + group.vm = [vm_ref] + else: + group.vm.append(vm_ref) + + group_spec = create_group_spec(client_factory, group, operation) + config_spec.groupSpec = [group_spec] reconfigure_cluster(session, cluster, config_spec) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 6e0f4e7340f..7caf3bb22aa 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -35,7 +35,6 @@ from nova.i18n import _ from nova.network import model as network_model from nova.utils import is_big_vm -from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import session from nova.virt.vmwareapi import vim_util @@ -1256,13 +1255,6 @@ def get_stats_from_cluster(session, cluster): return stats -def update_cluster_placement(session, instance, cluster, server_group_infos): - if not server_group_infos: - return - vm_ref = get_vm_ref(session, instance) - cluster_util.update_placement(session, cluster, vm_ref, server_group_infos) - - def get_host_ref(session, cluster=None): """Get reference to a host within the cluster specified.""" if cluster is None: diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 9575c7e8ea6..3399f0ce6a3 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -71,9 +71,6 @@ RESIZE_TOTAL_STEPS = 6 -GroupInfo = collections.namedtuple('GroupInfo', ['name', 'policies']) - - class VirtualMachineInstanceConfigInfo(object): """Parameters needed to create and configure a new instance.""" @@ -746,13 +743,31 @@ def prepare_for_spawn(self, instance): reason=reason) def update_cluster_placement(self, context, instance): - server_group_infos = self._get_server_groups(context, instance) - for group_info in server_group_infos: - self.sync_server_group(context, group_info.name) + self.sync_instance_server_group(context, instance) + self.update_admin_vm_group_membership(instance) - provider_group_infos = self._get_provider_server_groups(instance) - vm_util.update_cluster_placement(self._session, instance, - self._cluster, provider_group_infos) + def sync_instance_server_group(self, context, instance): + try: + instance_group_object = objects.instance_group.InstanceGroup + server_group = instance_group_object.get_by_instance_uuid( + context, instance.uuid) + self.sync_server_group(context, server_group.uuid) + except nova.exception.InstanceGroupNotFound: + pass + + def update_admin_vm_group_membership(self, instance): + vm_group_name = CONF.vmware.special_spawning_vm_group + if not vm_group_name: + return + + needs_empty_host = utils.vm_needs_special_spawning( + int(instance.memory_mb), instance.flavor) + if needs_empty_host: + return + + vm_ref = vm_util.get_vm_ref(self._session, instance) + cluster_util.update_vm_group_membership(self._session, self._cluster, + vm_group_name, vm_ref) def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info, block_device_info=None): @@ -782,6 +797,7 @@ def spawn(self, context, instance, image_meta, injected_files, # instance uuid. vm_util.vm_ref_cache_update(instance.uuid, vm_ref) + # Update all DRS related rules self.update_cluster_placement(context, instance) # Update the Neutron VNIC index @@ -1099,32 +1115,6 @@ def reboot(self, instance, network_info, reboot_type="SOFT"): self._session._wait_for_task(reset_task) LOG.debug("Did hard reboot of VM", instance=instance) - def _get_server_groups(self, context, instance): - server_group_infos = [] - try: - instance_group_object = objects.instance_group.InstanceGroup - server_group = instance_group_object.get_by_instance_uuid( - context, instance.uuid) - if server_group: - name = server_group.uuid - server_group_infos.append(GroupInfo(name, - server_group.policies)) - except nova.exception.InstanceGroupNotFound: - pass - - return server_group_infos - - def _get_provider_server_groups(self, instance): - server_group_infos = [] - - needs_empty_host = utils.vm_needs_special_spawning( - int(instance.memory_mb), instance.flavor) - if CONF.vmware.special_spawning_vm_group and not needs_empty_host: - name = CONF.vmware.special_spawning_vm_group - server_group_infos.append(GroupInfo(name, None)) - - return server_group_infos - def _destroy_instance(self, instance, destroy_disks=True): # Destroy a VM instance try: From b64c26feaececd76178e659f175ba838e2fbd311 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 27 Sep 2021 14:24:10 +0200 Subject: [PATCH 31/92] objects: Refactor Migration filtering We factor out the common functionality of checking the filter parameter for being a list and adding it to the query into a function to reduce the repetitive code. Change-Id: Id4587135d263d3f88609c96c1dcb9c4acc3d7fd9 Incorporated-Change-Id: I2b3c626ecf4a33c3baa20489b66bb7e6b69459b6 (cherry picked from commit 20455561b8754178decb6b88571478653e298659) --- nova/db/main/api.py | 57 ++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/nova/db/main/api.py b/nova/db/main/api.py index 16da893a156..3affb6d1f32 100644 --- a/nova/db/main/api.py +++ b/nova/db/main/api.py @@ -3403,6 +3403,22 @@ def migration_get_in_progress_by_instance(context, instance_uuid, return query.all() +def _migration_filter_item(query, filters, name): + if name not in filters: + return query + + column = getattr(models.Migration, name) + + item = filters[name] + if isinstance(item, str) or item is None: + return query.filter(column == item) + + try: + return query.filter(column.in_(item)) + except TypeError: # in_ expects an iterable, we fall back to comparison + return query.filter(column == item) + + @pick_context_manager_reader def migration_get_all_by_filters(context, filters, sort_keys=None, sort_dirs=None, @@ -3412,44 +3428,20 @@ def migration_get_all_by_filters(context, filters, return [] query = model_query(context, models.Migration) - if "uuid" in filters: - # The uuid filter is here for the MigrationLister and multi-cell - # paging support in the compute API. - uuid = filters["uuid"] - uuid = [uuid] if isinstance(uuid, str) else uuid - query = query.filter(models.Migration.uuid.in_(uuid)) model_object = models.Migration query = _get_query_nova_resource_by_changes_time(query, filters, model_object) - if "status" in filters: - status = filters["status"] - status = [status] if isinstance(status, str) else status - query = query.filter(models.Migration.status.in_(status)) - if "host" in filters: - host = filters["host"] - query = query.filter(sql.or_( - models.Migration.source_compute == host, - models.Migration.dest_compute == host)) - elif "source_compute" in filters: - host = filters['source_compute'] - query = query.filter(models.Migration.source_compute == host) if "node" in filters: node = filters['node'] query = query.filter(sql.or_( models.Migration.source_node == node, models.Migration.dest_node == node)) - if "migration_type" in filters: - migtype = filters["migration_type"] - query = query.filter(models.Migration.migration_type == migtype) if "hidden" in filters: hidden = filters["hidden"] query = query.filter(models.Migration.hidden == hidden) - if "instance_uuid" in filters: - instance_uuid = filters["instance_uuid"] - query = query.filter(models.Migration.instance_uuid == instance_uuid) if 'user_id' in filters: user_id = filters['user_id'] query = query.filter(models.Migration.user_id == user_id) @@ -3457,6 +3449,23 @@ def migration_get_all_by_filters(context, filters, project_id = filters['project_id'] query = query.filter(models.Migration.project_id == project_id) + # The uuid filter is here for the MigrationLister and multi-cell + # paging support in the compute API. + for item in ("uuid", "status", "migration_type", "instance_uuid"): + query = _migration_filter_item(query, filters, item) + + if "host" in filters: + host = filters["host"] + query = query.filter(sql.or_( + models.Migration.source_compute == host, + models.Migration.dest_compute == host)) + else: + query = _migration_filter_item(query, filters, "source_compute") + + if "hidden" in filters: + hidden = filters["hidden"] + query = query.filter(models.Migration.hidden == hidden) + if marker: try: marker = migration_get_by_uuid(context, marker) From 8e86696ff06629ebf02d7c882b0f6a2abb1fc2ad Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 27 Sep 2021 14:24:10 +0200 Subject: [PATCH 32/92] vmware: Sync server-group during migration During a migration, we have to consider the following situations: - On the source-host, we have to remove the group constraints as soon as the migration has started, in order to avoid that the constraints disallow the movement - On the destination-host, we have to add the rules as soon as the VM is in the vCenter, before the instance.host has been updated. Otherwise, we might remove rules added by the migration itself. The parameters to specify the cluster and the host are in preparation for the migration-task, which needs to call the sync for the source-host on the destination-host Change-Id: I2b3c626ecf4a33c3baa20489b66bb7e6b69459b6 (cherry picked from commit 2eef5e9dc545507bddc4ab6a343792e24ee21cfd) --- nova/virt/vmwareapi/vmops.py | 62 ++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 3399f0ce6a3..8e7ea44fd90 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -2264,6 +2264,8 @@ def set_compute_host(self, compute_host): self._compute_host = compute_host def sync_server_group(self, context, sg_uuid): + """Sync a server group by its uuid for the current host/cluster + """ # we have to ignore instances currently in a volatitle state, where # either VMware cannot support them being in a DRS rule or we expect # them to go away during the syncing process, which could lead to @@ -2271,13 +2273,11 @@ def sync_server_group(self, context, sg_uuid): # of expected members of a rule, which also removes them in the # cluster. STATES_EXCLUDING_MEMBERS_FROM_DRS_RULES = [ - task_states.MIGRATING, task_states.DELETING, task_states.SHELVING, task_states.REBUILDING, task_states.REBUILD_BLOCK_DEVICE_MAPPING, ] - LOG.debug('Starting sync for server-group %s', sg_uuid) @utils.synchronized('vmware-server-group-{}'.format(sg_uuid)) @@ -2302,6 +2302,55 @@ def _sync_sync_server_group(context, sg_uuid): LOG.debug('Sync for server-group %s done', sg_uuid) return + # First we check for all instances, which have ongoing migrations + # on the given host, either as source or destination + + # Decision matrix for migrations: + # Mig-Status Action/Host + # Source Dest + # Preparing Remove N/A + # Running Remove Add + # (Other states are consistent with default behaviour) + # + # So the Instance.host will always be the source of the migration, + # and we want to remove the rules. + # We only need to handle specially the case a running migration + # on the destination host, and add it + + MigrationList = objects.migration.MigrationList + filters = { + "host": self._compute_host, + "instance_uuid": sg.members, + "status": ["preparing", "running"], + } + + expected_members = {} + + migrations_by_instance_uuid = {} + for migration in MigrationList.get_by_filters(context, filters): + instance_uuid = migration.instance_uuid + if migration.source_compute == self._compute_host: + # The host is the source of a migration + # That means the instance will be part of the instance list + # So we have to remember that instance to be removed from + # the DRS rule-set + migrations_by_instance_uuid[instance_uuid] = \ + migration + else: + # We now handle the destination side + if migration.status == "preparing": + # Not even started, we can ignore that one + continue + + # Polling the cache, as vm_util.get_vm_ref is very slow + # for the negative search. + # We just have to ensure, that the cache holds a value + # before syncing the server group on the destination host + # Race conditions are averted by this functions lock + moref = vm_util.vm_ref_cache_get(instance_uuid) + if moref: + expected_members[instance_uuid] = moref + # retrieve the instances, because sg.members contains all members # and we need to filter them for our host InstanceList = objects.instance.InstanceList @@ -2310,7 +2359,6 @@ def _sync_sync_server_group(context, sg_uuid): instances = InstanceList.get_by_filters(context, filters, expected_attrs=[]) - expected_members = {} for instance in instances: task_state = instance.task_state if task_state in STATES_EXCLUDING_MEMBERS_FROM_DRS_RULES: @@ -2319,6 +2367,14 @@ def _sync_sync_server_group(context, sg_uuid): instance.uuid, sg.uuid, task_state) continue + migration = migrations_by_instance_uuid.get(instance.uuid) + if migration: + LOG.debug("Excluding member %s of server-group %s, " + "due to being on the source side of " + "ongoing migration %s.", + instance.uuid, sg.uuid, migration.uuid) + continue + try: moref = vm_util.get_vm_ref(self._session, instance) except exception.InstanceNotFound: From ab81aa06b98657e4b86f631d34eb6fe2fa4b211f Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 4 Oct 2021 15:48:40 +0200 Subject: [PATCH 33/92] Vmware: Add option to remove vm from vm-group In order to migrate a vm (within a vcenter) we need to be able to remove a vm from a vm-group constraining it to a set of hosts Change-Id: I8ef5ebdc54b3c3de0310e461132828aa251ee657 (cherry picked from commit e2a08bdfa8499e2d0c6e11942706764f9e29aa4f) (cherry picked from commit 327b134e4e88579428f4b893f3fde41be12f249b) --- nova/virt/vmwareapi/cluster_util.py | 36 ++++++++++++++++++++++------- nova/virt/vmwareapi/vmops.py | 9 ++++---- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index bfa3aebad3d..380a451905f 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -151,12 +151,17 @@ def delete_vm_group(session, cluster, vm_group): @utils.synchronized('vmware-vm-group-policy') -def update_vm_group_membership(session, cluster, vm_group_name, vm_ref): +def update_vm_group_membership(session, cluster, vm_group_name, vm_ref, + remove=False): """Updates cluster for vm placement using DRS - Add a VM to a Vm-group, create it if missing - It is up for an administrator to define rules for this group with other - means in the VCenter + If remove is set to false (default), add the VM to a Vm-group with + the given name and cluster. Create it if missing. + Otherwise, remove the vm from the group. The group itself won't get removed + even if there are no instances left. + + The assumption is that an administrator defines rules with other means + on that group. """ cluster_config = session._call_method( vutil, "get_object_property", cluster, "configurationEx") @@ -164,18 +169,33 @@ def update_vm_group_membership(session, cluster, vm_group_name, vm_ref): client_factory = session.vim.client.factory config_spec = client_factory.create('ns0:ClusterConfigSpecEx') - operation = None group = _get_vm_group(cluster_config, vm_group_name) if not group: - operation = "add" + if remove: + return group = create_vm_group(client_factory, vm_group_name, [vm_ref]) - else: - operation = "edit" + operation = "add" + elif not remove: if not hasattr(group, "vm"): group.vm = [vm_ref] else: group.vm.append(vm_ref) + operation = "edit" + else: # group and remove + if not hasattr(group, "vm"): + return + filtered_vms = [] + found = False + for ref in group.vm: + if (vutil.get_moref_value(ref) == vutil.get_moref_value(vm_ref)): + found = True + else: + filtered_vms.append(ref) + if not found: + return + group.vm = filtered_vms + operation = "edit" group_spec = create_group_spec(client_factory, group, operation) config_spec.groupSpec = [group_spec] diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 8e7ea44fd90..a48f97871f1 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -742,9 +742,9 @@ def prepare_for_spawn(self, instance): raise exception.InstanceUnacceptable(instance_id=instance.uuid, reason=reason) - def update_cluster_placement(self, context, instance): + def update_cluster_placement(self, context, instance, remove=False): self.sync_instance_server_group(context, instance) - self.update_admin_vm_group_membership(instance) + self.update_admin_vm_group_membership(instance, remove=remove) def sync_instance_server_group(self, context, instance): try: @@ -755,7 +755,7 @@ def sync_instance_server_group(self, context, instance): except nova.exception.InstanceGroupNotFound: pass - def update_admin_vm_group_membership(self, instance): + def update_admin_vm_group_membership(self, instance, remove=False): vm_group_name = CONF.vmware.special_spawning_vm_group if not vm_group_name: return @@ -767,7 +767,8 @@ def update_admin_vm_group_membership(self, instance): vm_ref = vm_util.get_vm_ref(self._session, instance) cluster_util.update_vm_group_membership(self._session, self._cluster, - vm_group_name, vm_ref) + vm_group_name, vm_ref, + remove=remove) def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info, block_device_info=None): From 566865f6defdc335984cc925ad5eea36d4ea279c Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Tue, 10 May 2022 15:01:33 +0200 Subject: [PATCH 34/92] vmware: Enable disabled DRS rules in sync When syncing a server-group's DRS rule(s) we now also enable a found rule in case it is disabled. We don't know how this happens, but sometimes rules get disabled and we need them to be enabled to guarantee the customer the appropriate (anti-)affinity settings. Change-Id: Ibc8eb6800640855513716412266fcbb9fbc4db42 (cherry picked from commit d712c23984891160c5fec1d4c68d31e0815726e4) (cherry picked from commit d4271839833aa5c915918fab6a094223d5b80316) --- nova/virt/vmwareapi/vmops.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index a48f97871f1..9a134e40f70 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -2426,6 +2426,13 @@ def _sync_sync_server_group(context, sg_uuid): LOG.debug('Sync for server-group %s done', sg_uuid) return + if not rule.enabled: + LOG.debug('Enabling DRS rule %s.', rule_name) + rule.enabled = True + cluster_util.update_rule( + self._session, self._cluster, rule) + LOG.info('Enabled DRS rule %s.', rule_name) + expected_moref_values = set(vutil.get_moref_value(m) for m in expected_members.values()) existing_moref_values = set(vutil.get_moref_value(m) From 166f24abe45cb0306a0258255e93c031a75fbe6b Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 18 Nov 2022 13:47:43 +0100 Subject: [PATCH 35/92] vmware: Pass VCState into VMwareVMOps We need the VCState inside the VMwareVMOps instance to access information about different hosts. We plan to use this for splitting server-groups based on the amount of available hosts in the cluster, but it can also be used for scheduling big VMs. Change-Id: I47edac9a81ef9a02cf07ab05e63edb9ed02d17b7 (cherry picked from commit 3e24bca63d72469ae14713a8b7d1c9adeb7b0cc0) --- nova/tests/unit/virt/vmwareapi/test_vmops.py | 13 ++++++++----- nova/virt/vmwareapi/driver.py | 9 +++++---- nova/virt/vmwareapi/vmops.py | 3 ++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index e1cc9252644..c5902714fc5 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -114,7 +114,7 @@ def setUp(self): self._instance.flavor = self._flavor self._vmops = vmops.VMwareVMOps(self._session, self._virtapi, None, - cluster=cluster.obj) + mock.Mock, cluster=cluster.obj) self._cluster = cluster self._image_meta = objects.ImageMeta.from_dict({'id': self._image_id}) subnet_4 = network_model.Subnet(cidr='192.168.0.1/24', @@ -192,7 +192,8 @@ def test_get_machine_id_str(self): self.assertEqual('DE:AD:BE:EF:00:00;;;;;#', result) def _setup_create_folder_mocks(self): - ops = vmops.VMwareVMOps(mock.Mock(), mock.Mock(), mock.Mock()) + ops = vmops.VMwareVMOps(mock.Mock(), mock.Mock(), mock.Mock(), + mock.Mock()) base_name = 'folder' ds_name = "datastore" ds_ref = vmwareapi_fake.ManagedObjectReference(value=1) @@ -218,7 +219,8 @@ def test_create_folder_if_missing_exception(self, mock_mkdir): mock_mkdir.assert_called_with(ops._session, path, dc) def test_get_valid_vms_from_retrieve_result(self): - ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock()) + ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock(), + mock.Mock()) fake_objects = vmwareapi_fake.FakeRetrieveResult() for x in range(0, 3): vm = vmwareapi_fake.VirtualMachine() @@ -230,7 +232,8 @@ def test_get_valid_vms_from_retrieve_result(self): self.assertEqual(3, len(vms)) def test_get_valid_vms_from_retrieve_result_with_invalid(self): - ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock()) + ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock(), + mock.Mock()) fake_objects = vmwareapi_fake.FakeRetrieveResult() valid_vm = vmwareapi_fake.VirtualMachine() valid_vm.set('config.extraConfig["nvp.vm-uuid"]', @@ -360,7 +363,7 @@ def mock_call_method(module, method, *args, **kwargs): def _test_get_datacenter_ref_and_name(self, ds_ref_exists=False): instance_ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds-1') - _vcvmops = vmops.VMwareVMOps(self._session, None, None) + _vcvmops = vmops.VMwareVMOps(self._session, None, None, None) result = vmwareapi_fake.FakeRetrieveResult() if ds_ref_exists: ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds-1') diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 07b3670aab3..55046e4c4cf 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -149,15 +149,16 @@ def __init__(self, virtapi, scheme="https"): self._create_nodename(vim_util.get_moref_value(self._cluster_ref)) self._volumeops = volumeops.VMwareVolumeOps(self._session, self._cluster_ref) + self._vc_state = host.VCState(self._session, + self._nodename, + self._cluster_ref, + self._datastore_regex) self._vmops = vmops.VMwareVMOps(self._session, virtapi, self._volumeops, + self._vc_state, self._cluster_ref, datastore_regex=self._datastore_regex) - self._vc_state = host.VCState(self._session, - self._nodename, - self._cluster_ref, - self._datastore_regex) self.capabilities['resource_scheduling'] = \ cluster_util.is_drs_enabled(self._session, self._cluster_ref) # Register the OpenStack extension diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 9a134e40f70..5daf70b3632 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -131,13 +131,14 @@ def retry_if_task_in_progress(f, *args, **kwargs): class VMwareVMOps(object): """Management class for VM-related tasks.""" - def __init__(self, session, virtapi, volumeops, cluster=None, + def __init__(self, session, virtapi, volumeops, vc_state, cluster=None, datastore_regex=None): """Initializer.""" self.compute_api = compute.API() self._session = session self._virtapi = virtapi self._volumeops = volumeops + self._vc_state = vc_state self._cluster = cluster self._root_resource_pool = vm_util.get_res_pool_ref(self._session, self._cluster) From 1e6e106564ed4fbb2fd00f0efb7e5baee8ab8388 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 18 Nov 2022 13:51:15 +0100 Subject: [PATCH 36/92] vmware: soft-anti-affinity can use multiple DRS rules Since VMware doesn't support the "mandatory" setting on VM-VM DRS rules, all rules we create are mandatory. This leads to VMs being unable to spawn in a cluster, if there are already as many VMs in the same soft-anti-affinity server-group as there are hosts in the cluster. Customers expect soft-anti-affinity to work also in this case. To accommodate for that, we now split members of soft-anti-affinity server-groups into multiple chunks. Each chunk has the size of available hosts in the cluster. For each chunk, we create an anti-affinity DRS rule. We try to make the members of each chunk stable by sorting the members in the server-group, but this commit probably still leads to more updates of DRS rules in those bigger server-groups. Since there can be rules in a no-longer-used chunk and since there are currently already rules with a different naming scheme (i.e. without the trailing number), we also fetch all rules of the same prefix and "update" them. Updating a rule without members leads to deletion of this rule. We will use the number of hosts - 1 instead of number of hosts as chunk size, because we need to make sure hosts can go into maintenance mode. Otherwise, if all hosts host VMs of the same soft-anti-affinity server-group, we cannot free up a host as no VM can move to another host. We cannot make this more dynamic, as we currently cannot detect hosts trying to go into MM. Change-Id: Id28fcc71193b491a1ac57e5c4f28c3b4862eeee5 Incorporated-Change-Id: I6550566ba4af116fba9c42cb5ca55a990e4ac9ac Co-Authored-By: Marius Leustean (cherry picked from commit e59205fb1320b4a681a5514a550015c8635db5bf) --- nova/virt/vmwareapi/vmops.py | 55 +++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 5daf70b3632..2526aa32b13 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -2386,13 +2386,59 @@ def _sync_sync_server_group(context, sg_uuid): continue expected_members[instance.uuid] = moref - rule_name = '{}-{}'.format(rule_prefix, sg.policy) + rule_members_by_name = {} + if sg.policy == 'soft-anti-affinity': + # we chunk by available hosts - 1, because we can spawn only as + # many VMs as there are hosts as VMWare doesn't provide any + # "soft" anti-affinity except for VM-Host relations, while we + # still allow 1 host to go into maintenance mode. + # Only hosts have an 'available' field - the cluster doesn't. + # Therefore, we don't have to filter out the aggregated cluster + # stats explicitly. + member_chunk_size = len( + [stat for stat in self._vc_state.get_host_stats().values() + if stat.get('available', False)]) + member_chunk_size = max(member_chunk_size - 1, 1) + + # to generate stable chunks we have to sort the expected + # members. we also need a list to be able to access parts of + # them. + expected_members = sorted(expected_members.items()) + + # generate chunks of the expected members with rules having a + # postfix counting up for the soft-anti-affinity policy + for i, j in enumerate(range(0, len(expected_members), + member_chunk_size)): + rule_name = '{}-{}-{}'.format(rule_prefix, sg.policy, i) + rule_members = expected_members[j:j + member_chunk_size] + rule_members_by_name[rule_name] = rule_members + + # we need to add in the existing rules with the same prefix, + # because there might be 1) old rules from before the chunking + # and 2) rules we don't reach anymore because the number of + # members of the sg is much lower now + existing_rule_names = [ + rule['name'] for rule in cluster_util.get_rules_by_prefix( + self._session, self._cluster, rule_prefix)] + + rule_names = \ + set(existing_rule_names) | set(rule_members_by_name) + for rule_name in rule_names: + rule_members = rule_members_by_name.get(rule_name, []) + _update_rule(rule_name, dict(rule_members), sg) + else: + # no chunking necessary - just update the rule + rule_name = '{}-{}'.format(rule_prefix, sg.policy) + _update_rule(rule_name, expected_members, sg) + + LOG.debug('Sync for server-group %s done', sg_uuid) + + def _update_rule(rule_name, expected_members, sg): rule = cluster_util.get_rule( self._session, self._cluster, rule_name) if not rule: if len(expected_members) < 2 or sg.policy == 'soft-affinity': - LOG.debug('Sync for server-group %s done', sg_uuid) return # we have to create a new rule LOG.debug('Creating missing DRS rule %s with members %s', @@ -2405,7 +2451,6 @@ def _sync_sync_server_group(context, sg_uuid): self._session, self._cluster, rule) LOG.info('Created missing DRS rule %s with members %s', rule_name, ', '.join(expected_members)) - LOG.debug('Sync for server-group %s done', sg_uuid) return if sg.policy == 'soft-affinity': @@ -2415,7 +2460,6 @@ def _sync_sync_server_group(context, sg_uuid): self._session, self._cluster, rule) LOG.info('Deleted DRS rule %s with policy soft-affinity.', rule_name) - LOG.debug('Sync for server-group %s done', sg_uuid) return if len(expected_members) < 2: @@ -2424,7 +2468,6 @@ def _sync_sync_server_group(context, sg_uuid): cluster_util.delete_rule( self._session, self._cluster, rule) LOG.info('Deleted DRS rule %s with < 2 members.', rule_name) - LOG.debug('Sync for server-group %s done', sg_uuid) return if not rule.enabled: @@ -2439,7 +2482,6 @@ def _sync_sync_server_group(context, sg_uuid): existing_moref_values = set(vutil.get_moref_value(m) for m in rule.vm) if expected_moref_values == existing_moref_values: - LOG.debug('Sync for server-group %s done', sg_uuid) return # we have to update the DRS rule to contain the right members @@ -2449,6 +2491,5 @@ def _sync_sync_server_group(context, sg_uuid): cluster_util.update_rule(self._session, self._cluster, rule) LOG.info('Updated DRS rule %s with members %s', rule_name, ', '.join(expected_members)) - LOG.debug('Sync for server-group %s done', sg_uuid) _sync_sync_server_group(context, sg_uuid) From 5e60f956817b64cb9b5d51a7a23ea8132bfe28ed Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 3 Aug 2023 10:01:25 +0200 Subject: [PATCH 37/92] vmware: Delete duplicate DRS rules Cross-cluster vMotion creates DRS rules in the target cluster, if the VM was part of them in the source cluster. This can create duplicate DRS rules (by name), as it seems VMware looks at the UUID of the rule not the name. Since multiple rules for the same VM cannot be enabled at the same time and `cluster_util.get_rule()` only returns the first matching rule, Nova might update the wrong rule and the anti-affinity might not actually work. To fix this, we get all the rules with the same name during `_update_rule()` to check for duplicates. We keep the first rule we find and delete all others. If the rule is disabled, Nova will enable it already. Since Nova always updates its rule with the VMs that should be in there, any VM being in a deleted duplicate rule is handled, too. Change-Id: Ie5958a1645e8133f303e386353526ec83e646882 (cherry picked from commit 843f988d7f44e90653635d7b8ffcdfa52b631991) --- nova/virt/vmwareapi/vmops.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 2526aa32b13..2979802da6d 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -2434,8 +2434,23 @@ def _sync_sync_server_group(context, sg_uuid): LOG.debug('Sync for server-group %s done', sg_uuid) def _update_rule(rule_name, expected_members, sg): - rule = cluster_util.get_rule( - self._session, self._cluster, rule_name) + # we need to get by "prefix", to get all rules matching our name, + # as there can be duplication happening with automatically + # vSphere-created rules during vMotion + rules = [r for r in cluster_util.get_rules_by_prefix( + self._session, self._cluster, rule_name) + if r.name == rule_name] + + rule = rules[0] if rules else None + + # if we have duplicates (with the same name), delete them + for dupl_rule in rules[1:]: + LOG.debug('Deleting DRS rule %s with key %s as duplicate', + dupl_rule.name, dupl_rule.key) + cluster_util.delete_rule( + self._session, self._cluster, dupl_rule) + LOG.info('Deleted rule %s with key %s as duplicate', + dupl_rule.name, dupl_rule.key) if not rule: if len(expected_members) < 2 or sg.policy == 'soft-affinity': From d4ced4eb4693fda233efc331a429d14c7cdd62b8 Mon Sep 17 00:00:00 2001 From: Csaba Seres Date: Fri, 6 Sep 2019 09:44:53 +0200 Subject: [PATCH 38/92] [vmware] Allow root-disk > flavor.root_gb for BFV When booting from volume, the driver should ignore the volume size and not error out of the root-disk is bigger than what the flavor would have created as ephemeral disk. (cherry picked from commit 533c5039ba7a2d0684edc97caa9c7bde3264cedd) (cherry picked from commit bae6465d859ec9efbbfdab6f7e611ba1bce174a7) Change-Id: I44340d55f696aa421eaa5b46c8f97a0e4a440655 (cherry picked from commit ca5c9a471c33160288aca766de86c3ca0b7d74d7) --- nova/virt/vmwareapi/vmops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 2979802da6d..b513c7746dc 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -573,11 +573,11 @@ def _cache_iso_image(self, vi, tmp_image_ds_loc): tmp_image_ds_loc.parent, vi.cache_image_folder) - def _get_vm_config_info(self, instance, image_info, - extra_specs): + def _get_vm_config_info(self, instance, image_info, extra_specs): """Captures all relevant information from the spawn parameters.""" if (instance.flavor.root_gb != 0 and + instance.image_ref and image_info.file_size > instance.flavor.root_gb * units.Gi): reason = _("Image disk size greater than requested disk size") raise exception.InstanceUnacceptable(instance_id=instance.uuid, From d5e15c5aa2519c743dfebb7359ecc142aeb9cd81 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 16 Oct 2019 09:13:50 +0200 Subject: [PATCH 39/92] [vmware] Don't rely on empty image_ref for BFV detection It's possible to create a boot-from-volume instance with instance.image_ref set to something - even something different than the actually booted volume's image according to https://bugs.launchpad.net/nova/+bug/1648501. Therefore, we cannot rely on an empty image_ref for BFV detection and instead use nova.compute.utils.is_volume_backed_instance now. This prohibits the vmware driver from creating a disk on ephemeral store in addition to using the volume and booting from the volume. (cherry picked from commit 395c76d2b128f442f2f60fd4c8c71b6d5247a0c6) (cherry picked from commit b08bf77d23ac9bbb6c2af62cfe28eafd281e9a3b) Change-Id: I6067dd9ce8a67f4c59b47b3f2c0931794daa68f3 (cherry picked from commit 5872f327f1cb1535ba93037dfbd1beab14bc6874) --- .../unit/virt/vmwareapi/test_configdrive.py | 2 +- .../unit/virt/vmwareapi/test_driver_api.py | 24 +++-- nova/tests/unit/virt/vmwareapi/test_vmops.py | 100 +++++++++++++----- nova/virt/vmwareapi/driver.py | 3 +- nova/virt/vmwareapi/vmops.py | 26 +++-- 5 files changed, 108 insertions(+), 47 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_configdrive.py b/nova/tests/unit/virt/vmwareapi/test_configdrive.py index 7b1b4142185..57ce91b730e 100644 --- a/nova/tests/unit/virt/vmwareapi/test_configdrive.py +++ b/nova/tests/unit/virt/vmwareapi/test_configdrive.py @@ -33,7 +33,7 @@ from nova.virt.vmwareapi import vmops -class ConfigDriveTestCase(test.NoDBTestCase): +class ConfigDriveTestCase(test.TestCase): REQUIRES_LOCKING = True diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 797fd1d7fe5..3100e926347 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -351,8 +351,10 @@ def _create_instance(self, node=None, set_image_ref=True, self.instance = fake_instance.fake_instance_obj( self.context, **values) - def _create_vm(self, node=None, num_instances=1, uuid=None, - flavor='m1.large', powered_on=True, + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) + def _create_vm(self, fake_is_volume_backed, node=None, num_instances=1, + uuid=None, flavor='m1.large', powered_on=True, ephemeral=None, bdi=None, flavor_updates=None): """Create and spawn the VM.""" if not node: @@ -1098,6 +1100,8 @@ def fake_call_method(module, method, *args, **kwargs): self._check_vm_info(info, power_state.RUNNING) self.assertTrue(self.exception) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=True) @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' 'update_cluster_placement') @mock.patch.object(vm_util, 'relocate_vm') @@ -1113,6 +1117,7 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, mock_attach_volume, mock_relocate_vm, mock_update_cluster_placement, + mock_is_volume_backed, set_image_ref=True): self._create_instance(set_image_ref=set_image_ref) @@ -1133,6 +1138,8 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, mock_attach_volume.assert_called_once_with(connection_info, self.instance, constants.DEFAULT_ADAPTER_TYPE) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' 'update_cluster_placement') @mock.patch('nova.virt.vmwareapi.volumeops.VMwareVolumeOps.' @@ -1143,7 +1150,8 @@ def test_spawn_attach_volume_iscsi(self, mock_info_get_mapping, mock_block_volume_in_mapping, mock_attach_volume, - mock_update_cluster_placement): + mock_update_cluster_placement, + mock_is_volume_backed): self._create_instance() connection_info = self._test_vmdk_connection_info('iscsi') root_disk = [{'connection_info': connection_info, @@ -2322,9 +2330,13 @@ def test_detach_interface_with_exception(self, mock_get_device): def test_resize_to_smaller_disk(self): self._create_vm(flavor='m1.large') flavor = self._get_flavor_by_name('m1.small') - self.assertRaises(exception.InstanceFaultRollback, - self.conn.migrate_disk_and_power_off, self.context, - self.instance, 'fake_dest', flavor, None) + with test.nested( + mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False)): + self.assertRaises(exception.InstanceFaultRollback, + self.conn.migrate_disk_and_power_off, + self.context, self.instance, 'fake_dest', flavor, + None) def test_spawn_attach_volume_vmdk(self): self._spawn_attach_volume_vmdk() diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index c5902714fc5..31d065e42d3 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -31,6 +31,7 @@ from nova.network import model as network_model from nova import objects from nova import test +from nova.tests.unit import fake_block_device from nova.tests.unit import fake_flavor from nova.tests.unit import fake_instance from nova.tests.unit.virt.vmwareapi import fake as vmwareapi_fake @@ -1151,6 +1152,8 @@ def test_migrate_disk_and_power_off_disk_shrink(self): self._test_migrate_disk_and_power_off, flavor_root_gb=self._instance.flavor.root_gb - 1) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vmops.VMwareVMOps, "_remove_ephemerals_and_swap") @mock.patch.object(vm_util, 'get_vmdk_info') @mock.patch.object(vmops.VMwareVMOps, "_resize_disk") @@ -1162,6 +1165,7 @@ def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, fake_power_off, fake_resize_vm, fake_resize_disk, fake_get_vmdk_info, fake_remove_ephemerals_and_swap, + fake_is_volume_backed, flavor_root_gb): vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', 'fake-adapter', @@ -1174,8 +1178,7 @@ def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, self._vmops.migrate_disk_and_power_off(self._context, self._instance, None, - flavor, - None) + flavor) fake_get_vm_ref.assert_called_once_with(self._session, self._instance) @@ -1192,6 +1195,7 @@ def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, for i in range(4)] fake_progress.assert_has_calls(calls) + @mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid') @mock.patch.object(vmops.VMwareVMOps, "_remove_ephemerals_and_swap") @mock.patch.object(vm_util, 'get_vmdk_info') @mock.patch.object(vmops.VMwareVMOps, "_resize_disk") @@ -1203,24 +1207,52 @@ def test_migrate_disk_and_power_off_root_block_device(self, fake_get_vm_ref, fake_progress, fake_power_off, fake_resize_vm, fake_resize_disk, fake_get_vmdk_info, - fake_remove_ephemerals_and_swap): + fake_remove_ephemerals_and_swap, + fake_bdm_get_by_instance_uuid): # shrinking the root-disk should be ignored flavor_root_gb = self._instance.flavor.root_gb - 1 - self._instance.image_ref = None - connection_info1 = {'data': 'fake-data1', 'serial': 'volume-fake-id1'} - connection_info2 = {'data': 'fake-data2', 'serial': 'volume-fake-id2'} - connection_info3 = {'data': 'fake-data3', 'serial': 'volume-fake-id3'} - bdm = [{'boot_index': 0, - 'connection_info': connection_info1, - 'disk_bus': constants.ADAPTER_TYPE_IDE}, - {'boot_index': 1, - 'connection_info': connection_info2, - 'disk_bus': constants.DEFAULT_ADAPTER_TYPE}, - {'boot_index': 2, - 'connection_info': connection_info3, - 'disk_bus': constants.ADAPTER_TYPE_LSILOGICSAS}] - bdi = {'block_device_mapping': bdm} + bdms = objects.block_device.block_device_make_list_from_dicts( + self._context, [ + fake_block_device.FakeDbBlockDeviceDict( + {'id': 1, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/sda', 'tag': "db", + 'volume_id': uuids.volume_1, + 'boot_index': 0}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 2, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/hda', 'tag': "nfvfunc1", + 'volume_id': uuids.volume_2}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 3, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/sdb', 'tag': "nfvfunc2", + 'volume_id': uuids.volume_3}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 4, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/hdb', + 'volume_id': uuids.volume_4}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 5, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/vda', 'tag': "nfvfunc3", + 'volume_id': uuids.volume_5}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 6, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/vdb', 'tag': "nfvfunc4", + 'volume_id': uuids.volume_6}), + fake_block_device.FakeDbBlockDeviceDict( + {'id': 7, + 'source_type': 'volume', 'destination_type': 'volume', + 'device_name': '/dev/vdc', 'tag': "nfvfunc5", + 'volume_id': uuids.volume_7}), + ] + ) + fake_bdm_get_by_instance_uuid.return_value = bdms vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', 'fake-adapter', @@ -1233,8 +1265,7 @@ def test_migrate_disk_and_power_off_root_block_device(self, self._vmops.migrate_disk_and_power_off(self._context, self._instance, None, - flavor, - bdi) + flavor) fake_get_vm_ref.assert_called_once_with(self._session, self._instance) @@ -1284,6 +1315,8 @@ def test_prepare_for_spawn_invalid_ram(self): self.assertRaises(exception.InstanceUnacceptable, self._vmops.prepare_for_spawn, instance) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=True) @mock.patch('nova.image.glance.API.get') @mock.patch.object(vmops.LOG, 'debug') @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_if_missing') @@ -1294,7 +1327,8 @@ def test_prepare_for_spawn_invalid_ram(self): def test_spawn_mask_block_device_info_password(self, mock_lock, mock_update_cluster_placement, mock_build_virtual_machine, mock_get_vm_config_info, - mock_fetch_image_if_missing, mock_debug, mock_glance): + mock_fetch_image_if_missing, mock_debug, mock_glance, + mock_is_volume_backed): # Very simple test that just ensures block_device_info auth_password # is masked when logged; the rest of the test just fails out early. data = {'auth_password': 'scrubme'} @@ -1355,6 +1389,8 @@ def _get_metadata(self, is_image_used=True): 'image_id': uuids.image if is_image_used else None, 'version': version.version_string_with_package()}) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vm_util, 'rename_vm') @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') @@ -1377,7 +1413,8 @@ def test_spawn_non_root_block_device(self, from_image, use_disk_image, power_on_instance, create_folders, - rename_vm): + rename_vm, + is_volume_backed_instance): self._instance.flavor = self._flavor extra_specs = get_extra_specs.return_value connection_info1 = {'data': 'fake-data1', 'serial': 'volume-fake-id1'} @@ -1423,6 +1460,8 @@ def test_spawn_non_root_block_device(self, from_image, connection_info2, self._instance, constants.DEFAULT_ADAPTER_TYPE) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=True) @mock.patch.object(vm_util, 'rename_vm') @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') @@ -1439,7 +1478,8 @@ def test_spawn_with_no_image_and_block_devices(self, from_image, build_virtual_machine, power_on_instance, create_folders, - rename_vm): + rename_vm, + is_volume_backed_instance): self._instance.image_ref = None self._instance.flavor = self._flavor extra_specs = get_extra_specs.return_value @@ -1489,6 +1529,8 @@ def test_spawn_with_no_image_and_block_devices(self, from_image, connection_info3, self._instance, constants.ADAPTER_TYPE_LSILOGICSAS) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') @mock.patch('nova.virt.vmwareapi.vm_util.power_on_instance') @@ -1503,7 +1545,8 @@ def test_spawn_unsupported_hardware(self, from_image, update_cluster_placement, build_virtual_machine, power_on_instance, - create_folders): + create_folders, + is_volume_backed_instance): self._instance.image_ref = None self._instance.flavor = self._flavor extra_specs = get_extra_specs.return_value @@ -1725,6 +1768,8 @@ def _verify_spawn_method_calls(self, mock_call_method, extras=None): recorded_methods = [c[1][1] for c in mock_call_method.mock_calls] self.assertEqual(expected_methods, recorded_methods) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @@ -1766,6 +1811,7 @@ def _test_spawn(self, mock_update_vnic_index, mock_update_cluster_placement, mock_create_folders, + mock_is_volume_backed, block_device_info=None, extra_specs=None, config_drive=False): @@ -1910,11 +1956,14 @@ def _test_spawn(self, mock_update_vnic_index.assert_called_once_with( self._context, self._instance, network_info) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(ds_util, 'get_datastore') @mock.patch.object(vmops.VMwareVMOps, 'get_datacenter_ref_and_name') def _test_get_spawn_vm_config_info(self, mock_get_datacenter_ref_and_name, mock_get_datastore, + mock_is_volume_backed, image_size_bytes=0): image_info = images.VMwareImage( image_id=self._image_id, @@ -2001,6 +2050,8 @@ def test_spawn_with_block_device_info_swap(self): 'device_name': '/dev/sdb'}} self._test_spawn(block_device_info=block_device_info) + @mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) @mock.patch.object(vm_util, 'rename_vm') @mock.patch('nova.virt.vmwareapi.vm_util.power_on_instance') @mock.patch.object(vmops.VMwareVMOps, '_create_and_attach_thin_disk') @@ -2023,7 +2074,8 @@ def test_spawn_with_ephemerals_and_swap(self, from_image, use_disk_image, create_and_attach_thin_disk, power_on_instance, - rename_vm): + rename_vm, + is_volume_backed_instance): self._instance.flavor = objects.Flavor(vcpus=1, memory_mb=512, name="m1.tiny", root_gb=1, ephemeral_gb=1, swap=512, diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 55046e4c4cf..da9715cf5ec 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -282,8 +282,7 @@ def migrate_disk_and_power_off(self, context, instance, dest, """ # TODO(PhilDay): Add support for timeout (clean shutdown) return self._vmops.migrate_disk_and_power_off(context, instance, - dest, flavor, - block_device_info) + dest, flavor) def confirm_migration(self, context, migration, instance, network_info): """Confirms a resize, destroying the source VM.""" diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index b513c7746dc..b9aac2090ae 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -40,6 +40,7 @@ from nova.compute import api as compute from nova.compute import power_state from nova.compute import task_states +from nova.compute import utils as compute_utils import nova.conf from nova.console import type as ctype from nova import context as nova_context @@ -576,8 +577,10 @@ def _cache_iso_image(self, vi, tmp_image_ds_loc): def _get_vm_config_info(self, instance, image_info, extra_specs): """Captures all relevant information from the spawn parameters.""" + boot_from_volume = compute_utils.is_volume_backed_instance( + instance._context, instance) if (instance.flavor.root_gb != 0 and - instance.image_ref and + not boot_from_volume and image_info.file_size > instance.flavor.root_gb * units.Gi): reason = _("Image disk size greater than requested disk size") raise exception.InstanceUnacceptable(instance_id=instance.uuid, @@ -783,6 +786,8 @@ def spawn(self, context, instance, image_meta, injected_files, vi = self._get_vm_config_info(instance, image_info, extra_specs) + boot_from_volume = compute_utils.is_volume_backed_instance( + instance._context, instance) metadata = self._get_instance_metadata(context, instance) # Creates the virtual machine. The virtual machine reference returned # is unique within Virtual Center. @@ -820,7 +825,7 @@ def spawn(self, context, instance, image_meta, injected_files, block_device_mapping = driver.block_device_info_get_mapping( block_device_info) - if instance.image_ref: + if instance.image_ref and not boot_from_volume: self._imagecache.enlist_image( image_info.image_id, vi.datastore, vi.dc_info.ref) self._fetch_image_if_missing(context, vi) @@ -1507,25 +1512,18 @@ def _resize_create_ephemerals_and_swap(self, vm_ref, instance, self._create_swap(block_device_info, instance, vm_ref, dc_info, datastore, folder, vmdk.adapter_type) - def migrate_disk_and_power_off(self, context, instance, dest, - flavor, block_device_info): + def migrate_disk_and_power_off(self, context, instance, dest, flavor): """Transfers the disk of a running instance in multiple phases, turning off the instance before the end. """ vm_ref = vm_util.get_vm_ref(self._session, instance) vmdk = vm_util.get_vmdk_info(self._session, vm_ref) - def _is_volume_backed(bdi): - # this contains anything with _valid_destination = 'volume', - # ephemerals have their own list - bdm = driver.block_device_info_get_mapping(bdi) - for disk in bdm: - if disk.get('boot_index') == 0: - return True - return False + boot_from_volume = compute_utils.is_volume_backed_instance(context, + instance) # Checks if the migration needs a disk resize down. - if (not _is_volume_backed(block_device_info) and ( + if (not boot_from_volume and ( flavor.root_gb < instance.flavor.root_gb or (flavor.root_gb != 0 and flavor.root_gb < vmdk.capacity_in_bytes / units.Gi))): @@ -1554,7 +1552,7 @@ def _is_volume_backed(bdi): total_steps=RESIZE_TOTAL_STEPS) # 3.Reconfigure the disk properties - if not _is_volume_backed(block_device_info): + if not boot_from_volume: self._resize_disk(instance, vm_ref, vmdk, flavor) self._update_instance_progress(context, instance, step=3, From 277dba04988ad36c2dd7d6078478b73d3a7e8a53 Mon Sep 17 00:00:00 2001 From: Marius Leustean Date: Wed, 20 Nov 2019 16:04:53 +0200 Subject: [PATCH 40/92] [vmware] allow to relocate a VM to another cluster while resizing The relocate should happen after the _revert_migration_update_disks() has been executed, otherwise reverting to the initial disk size could fail. We use `migration.dest_host` and `migration.source_host` to determine if we should move the instance to another cluster We fail if a vif is not found on the VM while relocating. When attaching the volumes after a migration, we must guarantee the order of the boot devices. Otherwise, the instance may not boot due to the boot disk not being attached as first disk. To facilitate this, we use the `boot_index` for ordering - volumes having `boot_index` set to 0 will be attached first. When a relocate fails and we roll back, we have to attach any volumes we detached in the process. (cherry picked from commit dac4481c19bcb2eabc9ad5eff431cacf49204d25) (cherry picked from commit 4a1957a219ef4fcd4dbdd11f5f33bbf7e09c5ed4) (cherry picked from commit 4e7a3dee1b267383a0b98bb4ebf3b21de442b92d) (cherry picked from commit c8f96d01039e89188320029ec26eb7d30c09a3a1) (cherry picked from commit ed2db87278b102a8be34151bccaab81458b96b1c) (cherry picked from commit 7b525c30f3513ef28af29c13c9246240a24235b3) (cherry picked from commit 5472f5e73ac9ba73cdee6732447bf89fa4fa9d42) (cherry picked from commit 53c241c2a26da03c6a5a8555188156f05de1b3ea) (cherry picked from commit 458376cb956da0a12b3714154bfc67a279d4a833) (cherry picked from commit 81f10702f1c24d1da190925dbd7a6fa0647e821d) (cherry picked from commit adff90c39a1e7044bc474bca714ec24654a8d784) (cherry picked from commit 748d0d938b33cbb07d9ec180f52f2aba5f675497) (cherry picked from commit f4b0146cd82fe3186709b0906cf1d9db0fedff91) Change-Id: I73deecbd9e870a2c000772be719a2f06a2f6d264 (cherry picked from commit eacd99228b4be4f3e4f1ab5756d515782f0a9e8c) --- nova/tests/unit/virt/vmwareapi/test_vmops.py | 297 +++++++++++++------ nova/virt/vmwareapi/vm_util.py | 70 ++--- nova/virt/vmwareapi/vmops.py | 177 +++++++++-- 3 files changed, 393 insertions(+), 151 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 31d065e42d3..76fd9ee7261 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -48,6 +48,7 @@ from nova.virt.vmwareapi import vim_util from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops +from nova.virt.vmwareapi import volumeops CONF = nova.conf.CONF @@ -113,9 +114,11 @@ def setUp(self): root_gb=10, ephemeral_gb=0, swap=0, extra_specs={}) self._instance.flavor = self._flavor - - self._vmops = vmops.VMwareVMOps(self._session, self._virtapi, None, - mock.Mock, cluster=cluster.obj) + self._volumeops = volumeops.VMwareVolumeOps(self._session) + self._vmops = vmops.VMwareVMOps(self._session, self._virtapi, + self._volumeops, + mock.Mock, + cluster=cluster.obj) self._cluster = cluster self._image_meta = objects.ImageMeta.from_dict({'id': self._image_id}) subnet_4 = network_model.Subnet(cidr='192.168.0.1/24', @@ -598,7 +601,7 @@ def test_clean_shutdown_no_vmwaretools(self): succeeds=False) def _test_finish_migration(self, power_on=True, resize_instance=False, - migration=None): + migration=None, relocate_fails=False): with test.nested( mock.patch.object(self._vmops, '_resize_create_ephemerals_and_swap'), @@ -607,39 +610,98 @@ def _test_finish_migration(self, power_on=True, resize_instance=False, mock.patch.object(vm_util, "get_vm_ref", return_value='fake-ref'), mock.patch.object(vmops.VMwareVMOps, - "update_cluster_placement") + "_remove_ephemerals_and_swap"), + mock.patch.object(vm_util, 'get_vmdk_info'), + mock.patch.object(vmops.VMwareVMOps, "_resize_vm"), + mock.patch.object(vmops.VMwareVMOps, "_resize_disk"), + mock.patch.object(vmops.VMwareVMOps, "_relocate_vm"), + mock.patch.object(vmops.VMwareVMOps, "_detach_volumes"), + mock.patch.object(vmops.VMwareVMOps, "_attach_volumes"), + mock.patch.object(vmops.VMwareVMOps, + "update_cluster_placement"), + mock.patch('nova.compute.utils.is_volume_backed_instance', + return_value=False) ) as (fake_resize_create_ephemerals_and_swap, fake_update_instance_progress, fake_power_on, fake_get_vm_ref, - fake_update_cluster_placement): + fake_remove_ephemerals_and_swap, fake_get_vmdk_info, + fake_resize_vm, fake_resize_disk, fake_relocate_vm, + fake_detach_volumes, fake_attach_volumes, + fake_update_cluster_placement, fake_is_volume_backed): migration = migration or objects.Migration(dest_compute="nova", - source_compute="nova", uuid=uuids.migration) - self._vmops.finish_migration(context=self._context, - migration=None, - instance=self._instance, - disk_info=None, - network_info=None, - block_device_info=None, - resize_instance=resize_instance, - image_meta=None, - power_on=power_on) - fake_resize_create_ephemerals_and_swap.assert_called_once_with( - 'fake-ref', self._instance, None) - if power_on: - fake_power_on.assert_called_once_with(self._session, - self._instance, - vm_ref='fake-ref') - else: - self.assertFalse(fake_power_on.called) - + source_compute="nova") + if relocate_fails: + fake_relocate_vm.side_effect = test.TestingException + vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', + 'fake-adapter', + 'fake-disk', + self._instance.flavor.root_gb * units.Gi, + 'fake-device') + fake_get_vmdk_info.return_value = vmdk + vm_ref_calls = [mock.call(self._session, self._instance)] + try: + self._vmops.finish_migration(context=self._context, + migration=migration, + instance=self._instance, + disk_info=None, + network_info=None, + block_device_info=None, + resize_instance=resize_instance, + image_meta=None, + power_on=power_on) + except test.TestingException: + pass + relocate_failed = False if migration.dest_compute != migration.source_compute: - fake_update_cluster_placement.assert_called_once_with( - self._context, self._instance) - calls = [ - mock.call(self._context, self._instance, step=5, - total_steps=vmops.RESIZE_TOTAL_STEPS), - mock.call(self._context, self._instance, step=6, - total_steps=vmops.RESIZE_TOTAL_STEPS)] - fake_update_instance_progress.assert_has_calls(calls) + fake_relocate_vm.\ + assert_called_once_with('fake-ref', self._context, + self._instance, None, None) + fake_detach_volumes.assert_called_once_with(self._instance, + None) + fake_attach_volumes.assert_called_once_with(self._instance, + None) + if not relocate_fails: + fake_update_cluster_placement.assert_called_once_with( + self._context, self._instance) + else: + fake_update_cluster_placement.assert_not_called() + relocate_failed = True + + if not relocate_failed: + fake_resize_create_ephemerals_and_swap.assert_called_once_with( + 'fake-ref', self._instance, None) + fake_remove_ephemerals_and_swap.assert_called_once_with( + 'fake-ref') + if power_on: + fake_power_on.assert_called_once_with(self._session, + self._instance, + vm_ref='fake-ref') + else: + self.assertFalse(fake_power_on.called) + + fake_resize_vm.assert_called_once_with(self._context, + self._instance, + 'fake-ref', + self._instance.flavor, + mock.ANY) + if resize_instance: + fake_resize_disk.\ + assert_called_once_with(self._instance, + 'fake-ref', + vmdk, + self._instance.flavor) + + calls = [mock.call(self._context, self._instance, step=i, + total_steps=vmops.RESIZE_TOTAL_STEPS) + for i in range(2, 7)] + fake_update_instance_progress.assert_has_calls(calls) + else: + fake_resize_create_ephemerals_and_swap.assert_not_called() + fake_remove_ephemerals_and_swap.assert_not_called() + fake_power_on.assert_not_called() + fake_resize_vm.assert_not_called() + fake_resize_disk.assert_not_called() + fake_update_instance_progress.assert_not_called() + fake_get_vm_ref.assert_has_calls(vm_ref_calls) def test_finish_migration_power_on(self): self._test_finish_migration(power_on=True, resize_instance=False) @@ -650,6 +712,17 @@ def test_finish_migration_power_off(self): def test_finish_migration_power_on_resize(self): self._test_finish_migration(power_on=True, resize_instance=True) + def test_finish_migration_another_cluster(self, relocate_fails=False): + self._test_finish_migration(power_on=True, + resize_instance=False, + relocate_fails=relocate_fails, + migration=objects.Migration( + dest_compute="dest", + source_compute="source")) + + def test_finish_migration_relocate_fails(self): + self.test_finish_migration_another_cluster(relocate_fails=True) + @mock.patch.object(vmops.VMwareVMOps, '_create_swap') @mock.patch.object(vmops.VMwareVMOps, '_create_ephemeral') @mock.patch.object(ds_obj, 'get_datastore_by_ref', @@ -701,6 +774,7 @@ def test_resize_create_ephemerals_no_root(self): vmdk = vm_util.VmdkInfo(None, None, None, 0, None) self._test_resize_create_ephemerals(vmdk, None) + @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(vmops.VMwareVMOps, '_resize_create_ephemerals_and_swap') @mock.patch.object(vmops.VMwareVMOps, '_remove_ephemerals_and_swap') @@ -716,7 +790,13 @@ def test_resize_create_ephemerals_no_root(self): @mock.patch.object(vm_util, 'power_off_instance') @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake-ref') @mock.patch.object(vm_util, 'power_on_instance') - def _test_finish_revert_migration(self, fake_power_on, + @mock.patch.object(vmops.VMwareVMOps, '_detach_volumes') + @mock.patch.object(vmops.VMwareVMOps, '_attach_volumes') + @mock.patch.object(vmops.VMwareVMOps, '_relocate_vm') + @mock.patch.object(vmops.VMwareVMOps, 'list_instances') + def _test_finish_revert_migration(self, fake_list_instances, + fake_relocate_vm, fake_attach_volumes, + fake_detach_volumes, fake_power_on, fake_get_vm_ref, fake_power_off, fake_resize_spec, fake_reconfigure_vm, fake_get_browser, @@ -725,7 +805,9 @@ def _test_finish_revert_migration(self, fake_power_on, fake_remove_ephemerals_and_swap, fake_resize_create_ephemerals_and_swap, fake_get_extra_specs, - power_on): + fake_update_cluster_placement, + power_on, instances_list=None, + relocate_fails=False): """Tests the finish_revert_migration method on vmops.""" datastore = ds_obj.Datastore(ref='fake-ref', name='fake') device = vmwareapi_fake.DataObject() @@ -750,14 +832,21 @@ def _test_finish_revert_migration(self, fake_power_on, self._vmops._volumeops = mock.Mock() mock_attach_disk = self._vmops._volumeops.attach_disk_to_vm mock_detach_disk = self._vmops._volumeops.detach_disk_from_vm + instances_list = instances_list or [self._instance.uuid] + fake_list_instances.return_value = instances_list + same_cluster = self._instance.uuid in instances_list + if relocate_fails: + fake_relocate_vm.side_effect = test.TestingException() + try: + self._vmops.finish_revert_migration(self._context, + instance=self._instance, + network_info=None, + block_device_info=None, + power_on=power_on) + except test.TestingException: + pass - self._vmops.finish_revert_migration(self._context, - instance=self._instance, - network_info=None, - block_device_info=None, - power_on=power_on) - fake_get_vm_ref.assert_called_once_with(self._session, - self._instance) + vm_ref_calls = [mock.call(self._session, self._instance)] fake_power_off.assert_called_once_with(self._session, self._instance, 'fake-ref') @@ -812,7 +901,21 @@ def _test_finish_revert_migration(self, fake_power_on, fake_remove_ephemerals_and_swap.assert_called_once_with('fake-ref') fake_resize_create_ephemerals_and_swap.assert_called_once_with( 'fake-ref', self._instance, None) - if power_on: + if not same_cluster: + fake_detach_volumes.assert_called_once_with(self._instance, + None) + fake_relocate_vm.\ + assert_called_once_with('fake-ref', self._context, + self._instance, None) + fake_attach_volumes.assert_called_once_with(self._instance, + None) + if not relocate_fails: + fake_update_cluster_placement.assert_called_once_with( + self._context, self._instance) + else: + fake_update_cluster_placement.assert_not_called() + fake_get_vm_ref.assert_has_calls(vm_ref_calls) + if power_on and not relocate_fails: fake_power_on.assert_called_once_with(self._session, self._instance) else: @@ -943,6 +1046,32 @@ def test_live_migration(self, mock_find_host, mock_find_datastore, post_method.assert_called_once_with(self._context, self._instance, 'fake-host', False, migrate_data) + def test_finish_revert_migration_another_cluster(self, + relocate_fails=False): + instances_list = ["fake_uuid_foo_bar"] + self._test_finish_revert_migration(power_on=True, + instances_list=instances_list, + relocate_fails=relocate_fails) + + def test_finish_revert_migration_relocate_fails(self): + self.test_finish_revert_migration_another_cluster(relocate_fails=True) + + @mock.patch.object(volumeops.VMwareVolumeOps, 'attach_volume') + def test_attach_volumes(self, fake_attach_volume): + block_device_info = { + 'block_device_mapping': [ + {'boot_index': -1, 'connection_info': {'id': 'c'}}, + {'boot_index': 1, 'connection_info': {'id': 'b'}}, + {'boot_index': 0, 'connection_info': {'id': 'a'}}, + ] + } + self._vmops._attach_volumes(self._instance, block_device_info) + fake_attach_volume.assert_has_calls([ + mock.call({'id': 'a'}, self._instance), + mock.call({'id': 'b'}, self._instance), + mock.call({'id': 'c'}, self._instance), + ]) + @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(vm_util, 'reconfigure_vm') @@ -1054,18 +1183,20 @@ def test_resize_disk(self, fake_disk_copy, fake_disk_move, extra_specs = vm_util.ExtraSpecs() fake_get_extra_specs.return_value = extra_specs + instance = self._instance.obj_clone() + instance.old_flavor = instance.flavor.obj_clone() flavor = fake_flavor.fake_flavor_obj(self._context, root_gb=self._instance.flavor.root_gb + 1) - self._vmops._resize_disk(self._instance, 'fake-ref', vmdk, flavor) + self._vmops._resize_disk(instance, 'fake-ref', vmdk, flavor) fake_get_dc_ref_and_name.assert_called_once_with(datastore.ref) fake_disk_copy.assert_called_once_with( self._session, dc_info.ref, '[fake] uuid/root.vmdk', '[fake] uuid/resized.vmdk') mock_detach_disk.assert_called_once_with('fake-ref', - self._instance, + instance, device) fake_extend.assert_called_once_with( - self._instance, flavor['root_gb'] * units.Mi, + instance, flavor['root_gb'] * units.Mi, '[fake] uuid/resized.vmdk', dc_info.ref) calls = [ mock.call(self._session, dc_info.ref, @@ -1077,7 +1208,7 @@ def test_resize_disk(self, fake_disk_copy, fake_disk_move, fake_disk_move.assert_has_calls(calls) mock_attach_disk.assert_called_once_with( - 'fake-ref', self._instance, 'fake-adapter', 'fake-disk', + 'fake-ref', instance, 'fake-adapter', 'fake-disk', '[fake] uuid/root.vmdk', disk_io_limits=extra_specs.disk_io_limits) @@ -1154,17 +1285,12 @@ def test_migrate_disk_and_power_off_disk_shrink(self): @mock.patch('nova.compute.utils.is_volume_backed_instance', return_value=False) - @mock.patch.object(vmops.VMwareVMOps, "_remove_ephemerals_and_swap") @mock.patch.object(vm_util, 'get_vmdk_info') - @mock.patch.object(vmops.VMwareVMOps, "_resize_disk") - @mock.patch.object(vmops.VMwareVMOps, "_resize_vm") @mock.patch.object(vm_util, 'power_off_instance') @mock.patch.object(vmops.VMwareVMOps, "_update_instance_progress") @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake-ref') def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, - fake_power_off, fake_resize_vm, - fake_resize_disk, fake_get_vmdk_info, - fake_remove_ephemerals_and_swap, + fake_power_off, fake_get_vmdk_info, fake_is_volume_backed, flavor_root_gb): vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', @@ -1186,32 +1312,29 @@ def _test_migrate_disk_and_power_off(self, fake_get_vm_ref, fake_progress, fake_power_off.assert_called_once_with(self._session, self._instance, 'fake-ref') - fake_resize_vm.assert_called_once_with(self._context, self._instance, - 'fake-ref', flavor, mock.ANY) - fake_resize_disk.assert_called_once_with(self._instance, 'fake-ref', - vmdk, flavor) calls = [mock.call(self._context, self._instance, step=i, total_steps=vmops.RESIZE_TOTAL_STEPS) - for i in range(4)] + for i in range(2)] fake_progress.assert_has_calls(calls) @mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid') + @mock.patch.object(vmops.VMwareVMOps, '_resize_create_ephemerals_and_swap') @mock.patch.object(vmops.VMwareVMOps, "_remove_ephemerals_and_swap") @mock.patch.object(vm_util, 'get_vmdk_info') @mock.patch.object(vmops.VMwareVMOps, "_resize_disk") @mock.patch.object(vmops.VMwareVMOps, "_resize_vm") - @mock.patch.object(vm_util, 'power_off_instance') + @mock.patch.object(vm_util, 'power_on_instance') @mock.patch.object(vmops.VMwareVMOps, "_update_instance_progress") @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake-ref') - def test_migrate_disk_and_power_off_root_block_device(self, - fake_get_vm_ref, fake_progress, - fake_power_off, fake_resize_vm, - fake_resize_disk, fake_get_vmdk_info, - fake_remove_ephemerals_and_swap, - fake_bdm_get_by_instance_uuid): + def test_finish_migration_root_block_device(self, fake_get_vm_ref, + fake_progress, fake_power_on, + fake_resize_vm, + fake_resize_disk, + fake_get_vmdk_info, + fake_remove_eph_and_swap, + fake_resize_create_eph_swap, + fake_bdm_get_by_instance_uuid): # shrinking the root-disk should be ignored - flavor_root_gb = self._instance.flavor.root_gb - 1 - bdms = objects.block_device.block_device_make_list_from_dicts( self._context, [ fake_block_device.FakeDbBlockDeviceDict( @@ -1254,31 +1377,33 @@ def test_migrate_disk_and_power_off_root_block_device(self, ) fake_bdm_get_by_instance_uuid.return_value = bdms - vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', - 'fake-adapter', - 'fake-disk', - self._instance.flavor.root_gb * units.Gi, - 'fake-device') - fake_get_vmdk_info.return_value = vmdk - flavor = fake_flavor.fake_flavor_obj(self._context, - root_gb=flavor_root_gb) - self._vmops.migrate_disk_and_power_off(self._context, - self._instance, - None, - flavor) - + migration = objects.Migration(source_compute="nova", + dest_compute="nova") + self._vmops.finish_migration(context=self._context, + migration=migration, + instance=self._instance, + disk_info=None, + network_info=None, + block_device_info=None, + resize_instance=True, + image_meta=self._image_meta, + power_on=True) fake_get_vm_ref.assert_called_once_with(self._session, self._instance) - - fake_power_off.assert_called_once_with(self._session, - self._instance, - 'fake-ref') + fake_resize_create_eph_swap.assert_called_once_with( + 'fake-ref', self._instance, None) + fake_remove_eph_and_swap.assert_called_once_with('fake-ref') + fake_power_on.assert_called_once_with(self._session, self._instance, + vm_ref='fake-ref') fake_resize_vm.assert_called_once_with(self._context, self._instance, - 'fake-ref', flavor, mock.ANY) + 'fake-ref', + self._instance.flavor, + self._image_meta) + fake_get_vmdk_info.assert_not_called() fake_resize_disk.assert_not_called() calls = [mock.call(self._context, self._instance, step=i, total_steps=vmops.RESIZE_TOTAL_STEPS) - for i in range(4)] + for i in range(2, 7)] fake_progress.assert_has_calls(calls) @mock.patch.object(vutil, 'get_inventory_path', return_value='fake_path') diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 7caf3bb22aa..b798ad89544 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -441,9 +441,40 @@ def _create_vif_spec(client_factory, vif_info, vif_limits=None, offset=0): # NOTE(asomya): Only works on ESXi if the portgroup binding is set to # ephemeral. Invalid configuration if set to static and the NIC does # not come up on boot if set to dynamic. + mac_address = vif_info['mac_address'] + set_net_device_backing(client_factory, net_device, vif_info) + connectable_spec = client_factory.create('ns0:VirtualDeviceConnectInfo') + connectable_spec.startConnected = True + connectable_spec.allowGuestControl = True + connectable_spec.connected = True + + net_device.connectable = connectable_spec + + # The Server assigns a Key to the device. Here we pass a -ve temporary key. + # -ve because actual keys are +ve numbers and we don't + # want a clash with the key that server might associate with the device + net_device.key = -47 - offset + net_device.addressType = "manual" + net_device.macAddress = mac_address + net_device.wakeOnLanEnabled = True + + # vnic limits are only supported from version 6.0 + if vif_limits and vif_limits.has_limits(): + if hasattr(net_device, 'resourceAllocation'): + net_device.resourceAllocation = _get_allocation_info( + client_factory, vif_limits, + 'ns0:VirtualEthernetCardResourceAllocation') + else: + msg = _('Limits only supported from vCenter 6.0 and above') + raise exception.Invalid(msg) + + network_spec.device = net_device + return network_spec + + +def set_net_device_backing(client_factory, net_device, vif_info): network_ref = vif_info['network_ref'] network_name = vif_info['network_name'] - mac_address = vif_info['mac_address'] backing = None if network_ref and network_ref['type'] == 'OpaqueNetwork': backing = client_factory.create( @@ -475,36 +506,8 @@ def _create_vif_spec(client_factory, vif_info, vif_limits=None, offset=0): backing = client_factory.create( 'ns0:VirtualEthernetCardNetworkBackingInfo') backing.deviceName = network_name - - connectable_spec = client_factory.create('ns0:VirtualDeviceConnectInfo') - connectable_spec.startConnected = True - connectable_spec.allowGuestControl = True - connectable_spec.connected = True - - net_device.connectable = connectable_spec net_device.backing = backing - # The Server assigns a Key to the device. Here we pass a -ve temporary key. - # -ve because actual keys are +ve numbers and we don't - # want a clash with the key that server might associate with the device - net_device.key = -47 - offset - net_device.addressType = "manual" - net_device.macAddress = mac_address - net_device.wakeOnLanEnabled = True - - # vnic limits are only supported from version 6.0 - if vif_limits and vif_limits.has_limits(): - if hasattr(net_device, 'resourceAllocation'): - net_device.resourceAllocation = _get_allocation_info( - client_factory, vif_limits, - 'ns0:VirtualEthernetCardResourceAllocation') - else: - msg = _('Limits only supported from vCenter 6.0 and above') - raise exception.Invalid(msg) - - network_spec.device = net_device - return network_spec - def get_network_attach_config_spec(client_factory, vif_info, index, vif_limits=None): @@ -998,10 +1001,11 @@ def clone_vm_spec(client_factory, location, def relocate_vm_spec(client_factory, res_pool=None, datastore=None, host=None, disk_move_type="moveAllDiskBackingsAndAllowSharing", - devices=None): + devices=None, folder=None): rel_spec = client_factory.create('ns0:VirtualMachineRelocateSpec') rel_spec.datastore = datastore rel_spec.host = host + rel_spec.folder = folder rel_spec.pool = res_pool rel_spec.diskMoveType = disk_move_type if devices is not None: @@ -1011,10 +1015,10 @@ def relocate_vm_spec(client_factory, res_pool=None, datastore=None, host=None, def relocate_vm(session, vm_ref, res_pool=None, datastore=None, host=None, disk_move_type="moveAllDiskBackingsAndAllowSharing", - devices=None): + devices=None, spec=None): client_factory = session.vim.client.factory - rel_spec = relocate_vm_spec(client_factory, res_pool, datastore, host, - disk_move_type, devices) + rel_spec = spec or relocate_vm_spec(client_factory, res_pool, datastore, + host, disk_move_type, devices) relocate_task = session._call_method(session.vim, "RelocateVM_Task", vm_ref, spec=rel_spec) session._wait_for_task(relocate_task) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index b9aac2090ae..a475eb39159 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -69,7 +69,7 @@ LOG = logging.getLogger(__name__) -RESIZE_TOTAL_STEPS = 6 +RESIZE_TOTAL_STEPS = 7 class VirtualMachineInstanceConfigInfo(object): @@ -346,15 +346,20 @@ def _get_extra_specs(self, flavor, image_meta=None): if video_ram and max_vram: extra_specs.hw_video_ram = video_ram * units.Mi / units.Ki - if CONF.vmware.pbm_enabled: - storage_policy = flavor.extra_specs.get('vmware:storage_policy', - CONF.vmware.pbm_default_policy) + storage_policy = self._get_storage_policy(flavor) + if storage_policy: extra_specs.storage_policy = storage_policy topology = hardware.get_best_cpu_topology(flavor, image_meta, allow_threads=False) extra_specs.cores_per_socket = topology.cores return extra_specs + def _get_storage_policy(self, flavor): + if CONF.vmware.pbm_enabled: + return flavor.extra_specs.get('vmware:storage_policy', + CONF.vmware.pbm_default_policy) + return None + def _get_esx_host_and_cookies(self, datastore, dc_path, file_path): hosts = datastore.get_connected_hosts(self._session) host = ds_obj.Datastore.choose_host(hosts) @@ -1461,7 +1466,7 @@ def _resize_vm(self, context, instance, vm_ref, flavor, image_meta): def _resize_disk(self, instance, vm_ref, vmdk, flavor): extra_specs = self._get_extra_specs(instance.flavor, instance.image_meta) - if (flavor.root_gb > instance.flavor.root_gb and + if ((flavor.root_gb > instance.old_flavor.root_gb) and flavor.root_gb > vmdk.capacity_in_bytes / units.Gi): root_disk_in_kb = flavor.root_gb * units.Mi ds_ref = vmdk.device.backing.datastore @@ -1531,8 +1536,6 @@ def migrate_disk_and_power_off(self, context, instance, dest, flavor): raise exception.InstanceFaultRollback( exception.ResizeError(reason=reason)) - # TODO(garyk): treat dest parameter. Migration needs to be treated. - # 0. Zero out the progress to begin self._update_instance_progress(context, instance, step=0, @@ -1544,26 +1547,6 @@ def migrate_disk_and_power_off(self, context, instance, dest, flavor): step=1, total_steps=RESIZE_TOTAL_STEPS) - # 2. Reconfigure the VM properties - self._resize_vm(context, instance, vm_ref, flavor, instance.image_meta) - - self._update_instance_progress(context, instance, - step=2, - total_steps=RESIZE_TOTAL_STEPS) - - # 3.Reconfigure the disk properties - if not boot_from_volume: - self._resize_disk(instance, vm_ref, vmdk, flavor) - self._update_instance_progress(context, instance, - step=3, - total_steps=RESIZE_TOTAL_STEPS) - - # 4. Purge ephemeral and swap disks - self._remove_ephemerals_and_swap(vm_ref) - self._update_instance_progress(context, instance, - step=4, - total_steps=RESIZE_TOTAL_STEPS) - def confirm_migration(self, migration, instance, network_info): """Confirms a resize, destroying the source VM.""" vm_ref = vm_util.get_vm_ref(self._session, instance) @@ -1640,6 +1623,23 @@ def finish_revert_migration(self, context, instance, network_info, if vmdk.device: self._revert_migration_update_disks(vm_ref, instance, vmdk, block_device_info) + # Relocate the instance back, if needed + if instance.uuid not in self.list_instances(): + self._detach_volumes(instance, block_device_info) + LOG.debug("Relocating VM for reverting migration", + instance=instance) + try: + self._relocate_vm(vm_ref, context, instance, network_info) + LOG.debug("Relocated VM for reverting migration", + instance=instance) + except Exception as e: + with excutils.save_and_reraise_exception(): + LOG.error("Relocating the VM failed: %s", e, + instance=instance) + else: + self.update_cluster_placement(context, instance) + finally: + self._attach_volumes(instance, block_device_info) if power_on: vm_util.power_on_instance(self._session, instance) @@ -1650,20 +1650,133 @@ def finish_migration(self, context, migration, instance, disk_info, """Completes a resize, turning on the migrated instance.""" vm_ref = vm_util.get_vm_ref(self._session, instance) - # 5. Update ephemerals if necessary + flavor = instance.flavor + boot_from_volume = compute_utils.is_volume_backed_instance(context, + instance) + reattach_volumes = False + # 2. Relocate the VM if necessary + # If the dest_compute is different from the source_compute, it means we + # need to relocate the VM here since we are running on the dest_compute + if migration.source_compute != migration.dest_compute: + self._detach_volumes(instance, block_device_info) + reattach_volumes = True + LOG.debug("Relocating VM for migration to %s", + migration.dest_compute, instance=instance) + try: + self._relocate_vm(vm_ref, context, instance, network_info, + image_meta) + LOG.debug("Relocated VM to %s", migration.dest_compute, + instance=instance) + except Exception as e: + with excutils.save_and_reraise_exception(): + LOG.error("Relocating the VM failed with error: %s", e, + instance=instance) + self._attach_volumes(instance, block_device_info) + + self.update_cluster_placement(context, instance) + + self._update_instance_progress(context, instance, + step=2, + total_steps=RESIZE_TOTAL_STEPS) + # 3.Reconfigure the VM and disk + self._resize_vm(context, instance, vm_ref, flavor, image_meta) + if not boot_from_volume and resize_instance: + vmdk = vm_util.get_vmdk_info(self._session, vm_ref, + uuid=instance.uuid) + self._resize_disk(instance, vm_ref, vmdk, flavor) + self._update_instance_progress(context, instance, + step=3, + total_steps=RESIZE_TOTAL_STEPS) + + # 4. Purge ephemeral and swap disks + self._remove_ephemerals_and_swap(vm_ref) + self._update_instance_progress(context, instance, + step=4, + total_steps=RESIZE_TOTAL_STEPS) + # 5. Update ephemerals self._resize_create_ephemerals_and_swap(vm_ref, instance, block_device_info) - self._update_instance_progress(context, instance, step=5, total_steps=RESIZE_TOTAL_STEPS) - # 6. Start VM - if power_on: - vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) + # 6. Attach the volumes (if necessary) + if reattach_volumes: + self._attach_volumes(instance, block_device_info) self._update_instance_progress(context, instance, step=6, total_steps=RESIZE_TOTAL_STEPS) + # 7. Start VM + if power_on: + vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) + self._update_instance_progress(context, instance, + step=7, + total_steps=RESIZE_TOTAL_STEPS) + + def _relocate_vm(self, vm_ref, context, instance, network_info, + image_meta=None): + image_meta = image_meta or instance.image_meta + storage_policy = self._get_storage_policy(instance.flavor) + allowed_ds_types = ds_util.get_allowed_datastore_types( + image_meta.properties.hw_disk_type) + datastore = ds_util.get_datastore(self._session, self._cluster, + self._datastore_regex, + storage_policy, + allowed_ds_types) + dc_info = self.get_datacenter_ref_and_name(datastore.ref) + folder = self._get_project_folder(dc_info, instance.project_id, + 'Instances') + + client_factory = self._session.vim.client.factory + spec = vm_util.relocate_vm_spec(client_factory, + res_pool=self._root_resource_pool, + folder=folder, datastore=datastore.ref) + + # Iterate over the network adapters and update the backing + if network_info: + spec.deviceChange = [] + vif_model = image_meta.properties.hw_vif_model + hardware_devices = self._session._call_method( + vutil, + "get_object_property", + vm_ref, + "config.hardware.device") + vif_infos = vmwarevif.get_vif_info(self._session, + self._cluster, + vif_model, + network_info) + for vif_info in vif_infos: + device = vmwarevif.get_network_device(hardware_devices, + vif_info['mac_address']) + if not device: + msg = _("No device with MAC address %s exists on the " + "VM") % vif_info['mac_address'] + raise exception.NotFound(msg) + + # Update the network device backing + config_spec = client_factory.create( + 'ns0:VirtualDeviceConfigSpec') + vm_util.set_net_device_backing( + client_factory, device, vif_info) + config_spec.operation = "edit" + config_spec.device = device + spec.deviceChange.append(config_spec) + + vm_util.relocate_vm(self._session, vm_ref, spec=spec) + + def _detach_volumes(self, instance, block_device_info): + block_devices = driver.block_device_info_get_mapping(block_device_info) + for disk in block_devices: + self._volumeops.detach_volume(disk['connection_info'], instance) + + def _attach_volumes(self, instance, block_device_info): + disks = driver.block_device_info_get_mapping(block_device_info) + # make sure the disks are attached by the boot_index order (if any) + for disk in sorted(disks, + key=lambda d: d['boot_index'] + if 'boot_index' in d and d['boot_index'] > -1 + else len(disks)): + self._volumeops.attach_volume(disk['connection_info'], instance) def _find_esx_host(self, cluster_ref, ds_ref): """Find ESX host in the specified cluster which is also connected to From f0737eb97ed50260ecd4872722b700559721b32c Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 25 Jun 2020 16:11:55 +0200 Subject: [PATCH 41/92] vmware: Use right adapter-type for resize volume attach When we resize, we detach all volumes from the VM. If this is a BfV VM, it has no disks left and thus loses it's SCSI adapter. Since we previously didn't explicitly set an adapter type on re-attaching the volumes, the default was used. Now, it might happen, that the OS does not have all drivers in the initrd and thus can't find its disk anymore. This happened with SLES12p4 images, which define a paravirtual adapter type but got a LsiLogic one after resize. Change-Id: I0dbbd5ca4851d01cfb03b95dcd54af582975d598 (cherry picked from commit 33193bf5dd5a15f5e437cafa7dbaf27ba1704c97) (cherry picked from commit 65ab9dbd185ce5813082f4dad0e9d8e225dd404c) (cherry picked from commit 294a6612c5563cefa1cd25e76367f57997df5b8c) --- nova/tests/unit/virt/vmwareapi/test_vmops.py | 19 ++++++++++-------- nova/virt/vmwareapi/vmops.py | 21 +++++++++++++++----- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 76fd9ee7261..0d2e77561bc 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -658,7 +658,8 @@ def _test_finish_migration(self, power_on=True, resize_instance=False, fake_detach_volumes.assert_called_once_with(self._instance, None) fake_attach_volumes.assert_called_once_with(self._instance, - None) + None, + vmdk.adapter_type) if not relocate_fails: fake_update_cluster_placement.assert_called_once_with( self._context, self._instance) @@ -815,7 +816,7 @@ def _test_finish_revert_migration(self, fake_list_instances, backing.datastore = datastore.ref device.backing = backing vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', - 'fake-adapter', + mock.sentinel.adapter_type, 'fake-disk', 'fake-capacity', device) @@ -895,7 +896,7 @@ def _test_finish_revert_migration(self, fake_list_instances, '[fake] uuid/original.vmdk', '[fake] uuid/root.vmdk') mock_attach_disk.assert_called_once_with( - 'fake-ref', self._instance, 'fake-adapter', 'fake-disk', + 'fake-ref', self._instance, vmdk.adapter_type, 'fake-disk', '[fake] uuid/root.vmdk', disk_io_limits=extra_specs.disk_io_limits) fake_remove_ephemerals_and_swap.assert_called_once_with('fake-ref') @@ -908,7 +909,8 @@ def _test_finish_revert_migration(self, fake_list_instances, assert_called_once_with('fake-ref', self._context, self._instance, None) fake_attach_volumes.assert_called_once_with(self._instance, - None) + None, + vmdk.adapter_type) if not relocate_fails: fake_update_cluster_placement.assert_called_once_with( self._context, self._instance) @@ -1065,11 +1067,12 @@ def test_attach_volumes(self, fake_attach_volume): {'boot_index': 0, 'connection_info': {'id': 'a'}}, ] } - self._vmops._attach_volumes(self._instance, block_device_info) + self._vmops._attach_volumes(self._instance, block_device_info, + mock.sentinel.adapter_type) fake_attach_volume.assert_has_calls([ - mock.call({'id': 'a'}, self._instance), - mock.call({'id': 'b'}, self._instance), - mock.call({'id': 'c'}, self._instance), + mock.call({'id': 'a'}, self._instance, mock.sentinel.adapter_type), + mock.call({'id': 'b'}, self._instance, mock.sentinel.adapter_type), + mock.call({'id': 'c'}, self._instance, mock.sentinel.adapter_type), ]) @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index a475eb39159..de39749210a 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -1625,6 +1625,9 @@ def finish_revert_migration(self, context, instance, network_info, block_device_info) # Relocate the instance back, if needed if instance.uuid not in self.list_instances(): + # Get the root disk vmdk object's adapter type + adapter_type = vmdk.adapter_type + self._detach_volumes(instance, block_device_info) LOG.debug("Relocating VM for reverting migration", instance=instance) @@ -1639,7 +1642,7 @@ def finish_revert_migration(self, context, instance, network_info, else: self.update_cluster_placement(context, instance) finally: - self._attach_volumes(instance, block_device_info) + self._attach_volumes(instance, block_device_info, adapter_type) if power_on: vm_util.power_on_instance(self._session, instance) @@ -1658,6 +1661,11 @@ def finish_migration(self, context, migration, instance, disk_info, # If the dest_compute is different from the source_compute, it means we # need to relocate the VM here since we are running on the dest_compute if migration.source_compute != migration.dest_compute: + # Get the root disk vmdk object's adapter type + vmdk = vm_util.get_vmdk_info(self._session, vm_ref, + uuid=instance.uuid) + adapter_type = vmdk.adapter_type + self._detach_volumes(instance, block_device_info) reattach_volumes = True LOG.debug("Relocating VM for migration to %s", @@ -1671,7 +1679,8 @@ def finish_migration(self, context, migration, instance, disk_info, with excutils.save_and_reraise_exception(): LOG.error("Relocating the VM failed with error: %s", e, instance=instance) - self._attach_volumes(instance, block_device_info) + self._attach_volumes(instance, block_device_info, + adapter_type) self.update_cluster_placement(context, instance) @@ -1702,7 +1711,7 @@ def finish_migration(self, context, migration, instance, disk_info, # 6. Attach the volumes (if necessary) if reattach_volumes: - self._attach_volumes(instance, block_device_info) + self._attach_volumes(instance, block_device_info, adapter_type) self._update_instance_progress(context, instance, step=6, total_steps=RESIZE_TOTAL_STEPS) @@ -1769,14 +1778,16 @@ def _detach_volumes(self, instance, block_device_info): for disk in block_devices: self._volumeops.detach_volume(disk['connection_info'], instance) - def _attach_volumes(self, instance, block_device_info): + def _attach_volumes(self, instance, block_device_info, adapter_type): disks = driver.block_device_info_get_mapping(block_device_info) # make sure the disks are attached by the boot_index order (if any) for disk in sorted(disks, key=lambda d: d['boot_index'] if 'boot_index' in d and d['boot_index'] > -1 else len(disks)): - self._volumeops.attach_volume(disk['connection_info'], instance) + adapter_type = disk.get('disk_bus') or adapter_type + self._volumeops.attach_volume(disk['connection_info'], instance, + adapter_type) def _find_esx_host(self, cluster_ref, ds_ref): """Find ESX host in the specified cluster which is also connected to From 0f448c4ced9cacb89fc51c59db0eb1a7d1e4db66 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 22 Jul 2020 11:06:58 +0200 Subject: [PATCH 42/92] vmware: Support memoryAllocation change on resize When resizing a VM, the driver only updated the `cpuAllocation` but not the `memoryAllocation` attribute of a VM. But since a resize to another flavor might change the reserved memory, we also need to update that attribute. Change-Id: Ie7e21fe3703688534d39cb7256a1b92c629452fa (cherry picked from commit c84419fa8977b0e76c4fa56517b18885f5d07ef0) (cherry picked from commit 4e99a85bdf176ca707713965b688ca3692823701) (cherry picked from commit 8e8ca9873790b0162e8d3c4cdd9ebff794a10563) --- nova/tests/unit/virt/vmwareapi/test_vm_util.py | 18 +++++++++++++++++- nova/virt/vmwareapi/vm_util.py | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 376d77d2373..e479ba87327 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -197,6 +197,13 @@ def test_get_resize_spec(self): cpuAllocation.shares.level = 'normal' cpuAllocation.shares.shares = 0 expected.cpuAllocation = cpuAllocation + memoryAllocation = fake_factory.create('ns0:ResourceAllocationInfo') + memoryAllocation.reservation = 0 + memoryAllocation.limit = -1 + memoryAllocation.shares = fake_factory.create('ns0:SharesInfo') + memoryAllocation.shares.level = 'normal' + memoryAllocation.shares.shares = 0 + expected.memoryAllocation = memoryAllocation expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -206,7 +213,9 @@ def test_get_resize_spec_with_limits(self): memory_mb = 2048 cpu_limits = vm_util.Limits(limit=7, reservation=6) - extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits) + memory_limits = vm_util.Limits(reservation=127) + extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits, + memory_limits=memory_limits) fake_factory = fake.FakeFactory() result = vm_util.get_vm_resize_spec(fake_factory, vcpus, memory_mb, extra_specs, @@ -221,6 +230,13 @@ def test_get_resize_spec_with_limits(self): cpuAllocation.shares.level = 'normal' cpuAllocation.shares.shares = 0 expected.cpuAllocation = cpuAllocation + memoryAllocation = fake_factory.create('ns0:ResourceAllocationInfo') + memoryAllocation.reservation = 127 + memoryAllocation.limit = -1 + memoryAllocation.shares = fake_factory.create('ns0:SharesInfo') + memoryAllocation.shares.level = 'normal' + memoryAllocation.shares.shares = 0 + expected.memoryAllocation = memoryAllocation expected.memoryReservationLockedToMax = True self.assertEqual(expected, result) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index b798ad89544..416f3bc5ede 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -368,6 +368,9 @@ def get_vm_resize_spec(client_factory, vcpus, memory_mb, extra_specs, resize_spec.cpuAllocation = _get_allocation_info( client_factory, extra_specs.cpu_limits, 'ns0:ResourceAllocationInfo') + resize_spec.memoryAllocation = _get_allocation_info( + client_factory, extra_specs.memory_limits, + 'ns0:ResourceAllocationInfo') resize_spec.memoryReservationLockedToMax = memory_reservation_locked if metadata: From 360964c6b7180ee005d8745e9bea6dea6aa33904 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 5 Aug 2021 11:49:49 +0200 Subject: [PATCH 43/92] VmWare: Create ServiceLocatorSpec For cross-vcenter migrations (live or not), we need to create ServiceLocatorSpec. This change provides the required functions Change-Id: I38503815a9a2ddc8939520c29117cbd8d9d13628 (cherry picked from commit 079e666214184dd69620bbad1fa58ee4de193541) --- nova/virt/vmwareapi/vm_util.py | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 416f3bc5ede..cace226af1c 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -20,6 +20,10 @@ import collections import copy +import hashlib +import socket +import ssl +from urllib.parse import urlparse from oslo_log import log as logging from oslo_service import loopingcall @@ -1700,3 +1704,60 @@ def detach_fcd(session, vm_ref, fcd_id): task = session._call_method( session.vim, "DetachDisk_Task", vm_ref, diskId=disk_id) session._wait_for_task(task) + + +def create_service_locator_name_password(client_factory, username, password): + """Creates a ServiceLocatorNamePassword object, which in turn is + derived of ServiceLocatorCredential + """ + o = client_factory.create('ns0:ServiceLocatorNamePassword') + o.username = username + o.password = password + return o + + +def get_sha1_ssl_thumbprint(url, timeout=1): + """Returns the sha-1 thumbprint of the ssl cert of the vcenter or None + """ + try: + parsed_url = urlparse(url) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + wrappedSocket = ssl.wrap_socket(sock) + thumb = None + wrappedSocket.connect((parsed_url.hostname, parsed_url.port or 443)) + + der_cert_bin = wrappedSocket.getpeercert(True) + + thumb = hashlib.sha1(der_cert_bin).hexdigest().upper() + t = iter(thumb) + return ':'.join(a + b for a, b in zip(t, t)) + except (OSError, ValueError, socket.timeout): + return + + +def create_service_locator(client_factory, url, vcenter_instance_uuid, + credential, ssl_thumbprint=None): + """Creates a ServiceLocator WSDL-object + + :param client_factory: - factory for creating the object + :param url: - url to the vcenter api + :param vcenter_instance_uuid: - uuid of the vcenter instance + :param credential: - An instance of ServiceLocatorCredential + (See: create_service_locator_name_password) + :param ssl_thumbprint: - The sha1 thumbprint of the cert of the instance + (See: get_sha1_ssl_thumbprint) + :returns: A ServiceLocator WSDL-object + """ + + # While it is nominally optional, operations seems to fail without it + # We will do a best effort + if not ssl_thumbprint: + ssl_thumbprint = get_sha1_ssl_thumbprint(url) + + sl = client_factory.create('ns0:ServiceLocator') + sl.url = url + sl.instanceUuid = vcenter_instance_uuid + sl.credential = credential + sl.sslThumbprint = ssl_thumbprint + return sl From 6da13e372024077b1a1d70c1c36078a912e87f17 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Sat, 2 Oct 2021 13:09:51 +0200 Subject: [PATCH 44/92] Vmware: Live-Migration of VMs Tested with attached VMs for old cinder API and new (3.44+) block-device-mapping. Requires patches to cinder to "migrate" the volumes between vCenter by creating an empty shadow-vm. Nova takes care of the rest - deleting the outdated shadow-vms - Attaching the volumes to the new shadow-vms PlaceVM API is called to chose the host, but the initial placement doesn't take the server groups into account, they are applied after the migration. BigVMs are missing the probably needed special treatment (emptying of the host) The code is written in a way, that if anything goes wrong during the migration, such as - the conductor - one of the compute nodes - one of the vCenter APIs being offline/restarted, that simply a second migration can be started. The migration may happen in the background of vCenter, and will be picked up when finished. The necessary book-keeping will be updated after the migration. The migration data will only be kept in memory, and will not be recovered on restart. Since the VM is gone after a migration, it won't be possible to update the vm-group membership and thus we do not call `update_cluster_placement()` but only sync the server group instead. We have to catch all exceptions in the post-migration steps, otherwise a roll-back will be initiated which we cannot do properly as the VM has already been migrated with the vSphere API. Instead the VM will be set in error state as it will require a manual inspection and intervention by an operator. Apparently, the volume-id is not consistently stored as volume_id in connection_info. Use the block_device.get_volume_id function to handle the fallback. Co-Authored-By: Jakob Karge Co-Authored-By: Marius Leustean Change-Id: If8c7265c53b64f00292e6689d1f6860ff29c671e (cherry picked from commit ee0d89c7106a494948bf6d08f3285842b4e09e56) Incorporated-Change-Id: I04c62f400a522bf9fe5828199c0dff80a1004f42 Incorporated-Change-Id: I75faecfdd48c9f40d243aecdd2b90b89e5158335 Incorporated-Change-Id: If5a8527578db8e4690595524e0785ee8b4de1d79 Incorporated-Change-Id: I192209d90603c43cb0c1bb4cb32e55c6638d9a3e (cherry picked from commit 68121cf7efcb3b90cd38e1c3b29022cbebd8e53b) --- nova/tests/unit/virt/vmwareapi/fake.py | 2 + .../unit/virt/vmwareapi/test_driver_api.py | 163 +++++++- nova/tests/unit/virt/vmwareapi/test_vmops.py | 32 -- nova/virt/vmwareapi/driver.py | 381 +++++++++++++++--- nova/virt/vmwareapi/vmops.py | 221 +++++++--- nova/virt/vmwareapi/volumeops.py | 118 ++++++ 6 files changed, 753 insertions(+), 164 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/fake.py b/nova/tests/unit/virt/vmwareapi/fake.py index 8f9b109dd75..fd717f56d74 100644 --- a/nova/tests/unit/virt/vmwareapi/fake.py +++ b/nova/tests/unit/virt/vmwareapi/fake.py @@ -438,6 +438,7 @@ def __init__(self, **kwargs): self.set("summary.config.memorySizeMB", kwargs.get("mem", 1)) self.set("summary.config.instanceUuid", kwargs.get("instanceUuid")) self.set("version", kwargs.get("version")) + self.set("resourcePool", kwargs.get("resourcePool")) devices = _create_array_of_type('VirtualDevice') devices.VirtualDevice = kwargs.get("virtual_device", []) @@ -1058,6 +1059,7 @@ def create_vm(uuid=None, name=None, "extra_config": extraConfig, "virtual_device": devices, "instanceUuid": uuid, + "resourcePool": res_pool_ref, "version": version} vm = VirtualMachine(**vm_dict) _create_object(vm) diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 3100e926347..df3ba5c7116 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -2361,25 +2361,158 @@ def test_datastore_dc_map(self): # currently there are 2 data stores self.assertEqual(2, len(ds_util._DS_DC_MAPPING)) - def test_pre_live_migration(self): - migrate_data = objects.VMwareLiveMigrateData() - migrate_data.cluster_name = 'fake-cluster' - migrate_data.datastore_regex = 'datastore1' - ret = self.conn.pre_live_migration(self.context, 'fake-instance', - 'fake-block-dev-info', 'fake-net-info', - 'fake-disk-info', migrate_data) - self.assertIs(migrate_data, ret) + def _create_live_migrate_data(self): + data = objects.migrate_data.VMwareLiveMigrateData() + + data.dest_cluster_ref = "cluster-0" + data.instance_already_migrated = False + data.is_same_vcenter = False + data.relocate_defaults = { + "relocate_spec": { + "pool": { + "_type": "Resgroup", + "value": "resgroup-0", + }, + "datastore": { + "_type": "Datastore", + "value": "datastore-0", + }, + "host": { + "_type": "Host", + "value": "host-0", + }, + "folder": { + "_type": "Folder", + "value": "folder-0", + } + }, + "service": { + "url": "https://otherhost.test", + "instance_uuid": "uuid", + "ssl_thumbprint": "sha1thumbprint", + "credentials": { + "_type": "ServiceLocatorNamePassword", + "username": "username", + "password": "password", + } + } + } + + return data + + def _create_block_device_info(self): + return {} + + def _create_placement_result(self, *args, **kwargs): + relocate_spec = mock.NonCallableMagicMock( + host=mock.sentinel.placement_host, + datastore=mock.sentinel.placement_datastore + ) + + recommendation = mock.NonCallableMagicMock( + reason="xvmotionPlacement", + rating=5, + action=[mock.NonCallableMagicMock(relocateSpec=relocate_spec)] + ) + + return mock.NonCallableMagicMock( + recommendations=[recommendation] + ) + + @mock.patch.object(vmops.VMwareVMOps, 'place_vm') + def test_pre_live_migration(self, mock_place_vm): + mock_place_vm.side_effect = self._create_placement_result + self._create_instance() + migrate_data = self._create_live_migrate_data() + block_device_info = self._create_block_device_info() + result = self.conn.pre_live_migration(self.context, + self.instance, block_device_info, 'fake_network_info', + 'fake_disk_info', migrate_data) + self.assertEqual(result, migrate_data) + + @mock.patch.object(vm_util, 'relocate_vm') + @mock.patch.object(vm_util, 'create_service_locator') + @mock.patch.object(vm_util, 'get_hardware_devices') + @mock.patch.object(driver.VMwareVCDriver, '_get_volume_mappings', + returns=[]) + def test_live_migration(self, get_volume_mappings, get_hardware_devices, + service_locator, relocate_vm): + self._create_instance() + migrate_data = self._create_live_migrate_data() + + post_method = mock.Mock() + recover_method = mock.Mock() + get_hardware_devices.returns = [] + + with mock.patch.object(vm_util, 'get_vm_ref', + return_value=mock.sentinel.vm_ref): + + self.conn.live_migration(self.context, self.instance, + 'fake_dest', post_method, recover_method, + migrate_data=migrate_data) + + service_locator.assert_called() + get_hardware_devices.assert_called() + relocate_vm.assert_called() + post_method.assert_called() + recover_method.assert_not_called() + + @mock.patch.object(vm_util, 'create_service_locator') + @mock.patch.object(volumeops.VMwareVolumeOps, + 'map_volumes_to_devices', returns=[]) + @mock.patch.object(vmops.VMwareVMOps, + 'live_migration', side_effect=Exception) + def test_live_migration_failure_rollback(self, mock_live_migration, + volumes_to_devices, mock_service_locator): + self._create_instance() + migrate_data = self._create_live_migrate_data() + + post_method = mock.Mock() + recover_method = mock.Mock() + + exception = Exception + + with test.nested( + mock.patch.object(vm_util, 'get_vm_ref', + return_value=mock.sentinel.vm_ref), + ) as (mock_vm_ref,): + + self.assertRaises(exception, self.conn.live_migration, + self.context, self.instance, + 'fake_dest', post_method, recover_method, + migrate_data=migrate_data) + + post_method.assert_not_called() + recover_method.assert_called() def test_rollback_live_migration_at_destination(self): - with mock.patch.object(self.conn, "destroy") as mock_destroy: - self.conn.rollback_live_migration_at_destination(self.context, - "instance", [], None) - mock_destroy.assert_called_once_with(self.context, - "instance", [], None) + self._create_instance() + block_device_info = self._create_block_device_info() + migrate_data = self._create_live_migrate_data() + with test.nested( + mock.patch.object(self.conn._volumeops, 'delete_shadow_vms'), + ) as (mock_delete_shadow_vms, ): + + self.conn.rollback_live_migration_at_destination( + self.context, self.instance, 'fake_network_info', + block_device_info, migrate_data=migrate_data) + mock_delete_shadow_vms.assert_called_once() def test_post_live_migration(self): - self.assertIsNone(self.conn.post_live_migration(self.context, - 'fake_instance', 'fake_block_device_info')) + self._create_instance() + migrate_data = self._create_live_migrate_data() + block_device_info = self._create_block_device_info() + with test.nested( + mock.patch.object(self.conn._volumeops, 'delete_shadow_vms'), + mock.patch.object(self.conn._vmops, + 'sync_instance_server_group') + ) as (mock_delete_shadow_vms, mock_sync_instance_server_group): + + self.conn.post_live_migration( + self.context, self.instance, block_device_info, + migrate_data) + mock_delete_shadow_vms.assert_called_once() + mock_sync_instance_server_group.assert_called_once() def test_get_instance_disk_info_is_implemented(self): # Ensure that the method has been implemented in the driver diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 0d2e77561bc..babd20e8ce8 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -1016,38 +1016,6 @@ def mock_call_method(module, method, *args, **kwargs): mock_get_ds.assert_called_once_with(self._session, 'cluster_ref', None) - @mock.patch.object(vm_util, 'relocate_vm') - @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake_vm') - @mock.patch.object(vm_util, 'get_cluster_ref_by_name', - return_value='fake_cluster') - @mock.patch.object(vm_util, 'get_res_pool_ref', return_value='fake_pool') - @mock.patch.object(vmops.VMwareVMOps, '_find_datastore_for_migration') - @mock.patch.object(vmops.VMwareVMOps, '_find_esx_host', - return_value='fake_host') - def test_live_migration(self, mock_find_host, mock_find_datastore, - mock_get_respool, mock_get_cluster, mock_get_vm, - mock_relocate): - post_method = mock.MagicMock() - migrate_data = objects.VMwareLiveMigrateData() - migrate_data.cluster_name = 'fake-cluster' - migrate_data.datastore_regex = 'ds1|ds2' - mock_find_datastore.return_value = ds_obj.Datastore('ds_ref', 'ds') - with mock.patch.object(self._session, '_call_method', - return_value='hardware-devices'): - self._vmops.live_migration( - self._context, self._instance, 'fake-host', - post_method, None, False, migrate_data) - mock_get_vm.assert_called_once_with(self._session, self._instance) - mock_get_cluster.assert_called_once_with(self._session, 'fake-cluster') - mock_find_datastore.assert_called_once_with(self._instance, 'fake_vm', - 'fake_cluster', mock.ANY) - mock_find_host.assert_called_once_with('fake_cluster', 'ds_ref') - mock_relocate.assert_called_once_with(self._session, 'fake_vm', - 'fake_pool', 'ds_ref', 'fake_host', - devices=[]) - post_method.assert_called_once_with(self._context, self._instance, - 'fake-host', False, migrate_data) - def test_finish_revert_migration_another_cluster(self, relocate_fails=False): instances_list = ["fake_uuid_foo_bar"] diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index da9715cf5ec..77889be0c05 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -18,7 +18,8 @@ """ A connection to the VMware vCenter platform. """ - +import contextlib +from operator import attrgetter import os import random import re @@ -28,6 +29,7 @@ import os_resource_classes as orc import os_traits as ot from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_utils import excutils from oslo_utils import units from oslo_utils import versionutils as v_utils @@ -46,6 +48,7 @@ from nova import objects import nova.privsep.path from nova import utils +from nova.virt import block_device from nova.virt import driver from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants @@ -53,6 +56,7 @@ from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import host from nova.virt.vmwareapi import session +from nova.virt.vmwareapi import vif as vmwarevif from nova.virt.vmwareapi import vim_util as nova_vim_util from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops @@ -217,7 +221,6 @@ def init_host(self, host): self._session._create_session() self._vmops.set_compute_host(host) - LOG.debug("Starting green server-group sync-loop thread") utils.spawn(self._server_group_sync_loop, host) @@ -303,65 +306,6 @@ def finish_migration(self, context, migration, instance, disk_info, network_info, image_meta, resize_instance, block_device_info, power_on) - def pre_live_migration(self, context, instance, block_device_info, - network_info, disk_info, migrate_data): - return migrate_data - - def post_live_migration_at_source(self, context, instance, network_info): - pass - - def post_live_migration_at_destination(self, context, instance, - network_info, - block_migration=False, - block_device_info=None): - pass - - def cleanup_live_migration_destination_check(self, context, - dest_check_data): - pass - - def live_migration(self, context, instance, dest, - post_method, recover_method, block_migration=False, - migrate_data=None): - """Live migration of an instance to another host.""" - self._vmops.live_migration(context, instance, dest, post_method, - recover_method, block_migration, - migrate_data) - - def check_can_live_migrate_source(self, context, instance, - dest_check_data, block_device_info=None): - cluster_name = dest_check_data.cluster_name - cluster_ref = vm_util.get_cluster_ref_by_name(self._session, - cluster_name) - if cluster_ref is None: - msg = (_("Cannot find destination cluster %s for live migration") % - cluster_name) - raise exception.MigrationPreCheckError(reason=msg) - res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref) - if res_pool_ref is None: - msg = _("Cannot find destination resource pool for live migration") - raise exception.MigrationPreCheckError(reason=msg) - return dest_check_data - - def check_can_live_migrate_destination(self, context, instance, - src_compute_info, dst_compute_info, - block_migration=False, - disk_over_commit=False): - # the information that we need for the destination compute node - # is the name of its cluster and datastore regex - data = objects.VMwareLiveMigrateData() - data.cluster_name = CONF.vmware.cluster_name - data.datastore_regex = CONF.vmware.datastore_regex - return data - - def rollback_live_migration_at_destination(self, context, instance, - network_info, - block_device_info, - destroy_disks=True, - migrate_data=None): - """Clean up destination node after a failed live migration.""" - self.destroy(context, instance, network_info, block_device_info) - def get_instance_disk_info(self, instance, block_device_info=None): pass @@ -591,6 +535,11 @@ def attach_volume(self, context, connection_info, instance, mountpoint, def detach_volume(self, context, connection_info, instance, mountpoint, encryption=None): """Detach volume storage to VM instance.""" + if not self._vmops.is_instance_in_resource_pool(instance): + volume_id = block_device.get_volume_id(connection_info) + LOG.debug("Not detaching %s, vm is in different cluster", + volume_id, instance=instance) + return True # NOTE(claudiub): if context parameter is to be used in the future, # the _detach_instance_volumes method will have to be updated as well. return self._volumeops.detach_volume(connection_info, instance) @@ -832,3 +781,313 @@ def _server_group_sync_loop(self, compute_host): LOG.debug('Finished server-group sync-loop') time.sleep(CONF.vmware.server_group_sync_loop_spacing) + + def check_can_live_migrate_destination(self, context, instance, + src_compute_info, dst_compute_info, + block_migration=False, + disk_over_commit=False): + """Check if it is possible to execute live migration.""" + data = objects.migrate_data.VMwareLiveMigrateData() + + data.instance_already_migrated = ( + self.instance_exists(instance) and # In this vcenter + self._vmops.is_instance_in_resource_pool(instance)) + + url = "https://{}:{}".format(CONF.vmware.host_ip, + CONF.vmware.host_port) + + data.cluster_name = self._cluster_name + data.dest_cluster_ref = vim_util.get_moref_value(self._cluster_ref) + data.datastore_regex = CONF.vmware.datastore_regex + + data.relocate_defaults = { + "service": { + "url": url, + "instance_uuid": self._vcenter_uuid, + "ssl_thumbprint": vm_util.get_sha1_ssl_thumbprint(url), + "credentials": { + "_type": "ServiceLocatorNamePassword", + "username": CONF.vmware.host_username, + "password": CONF.vmware.host_password, + } + } + } + + return data + + def cleanup_live_migration_destination_check(self, context, + dest_check_data): + """Do required cleanup on dest host after check_can_live_migrate calls + """ + + def check_can_live_migrate_source(self, context, instance, + dest_check_data, block_device_info=None): + """Check if it is possible to execute live migration.""" + + defaults = dest_check_data.relocate_defaults + dest_vcenter_uuid = defaults["service"]["instance_uuid"] + + dest_check_data.is_same_vcenter = ( + dest_vcenter_uuid == self._vcenter_uuid) + + if dest_check_data.instance_already_migrated: + return dest_check_data + + if dest_check_data.is_same_vcenter: + # Drop the service-credentials, no need to check the volumes, + # as we do not need to mess around with them + defaults = dest_check_data.relocate_defaults + defaults.pop("service") + dest_check_data.relocate_defaults = defaults + + # Remove the DRS constraints so we can actually move it. + # We have to do it in the check, because the next step + # is the pre_live_migration and that is on the destination + # host + self._vmops.update_cluster_placement(context, instance, + remove=True) + else: + # Validate that we have all necessary information for the + # _old_ volume attachments + self._get_checked_volumes(context, instance, [ + 'volume', # For deleting the old shadow-vms + ]) + + return dest_check_data + + def pre_live_migration(self, context, instance, block_device_info, + network_info, disk_info, migrate_data): + """Prepare an instance for live migration (on the destination host)""" + if migrate_data.instance_already_migrated: + return migrate_data + + return self._pre_live_migration(context, instance, + block_device_info, network_info, disk_info, migrate_data) + + def _pre_live_migration(self, context, instance, block_device_info, + network_info, disk_info, migrate_data): + result = self._vmops.place_vm(context, instance) + + if hasattr(result, 'drsFault'): + LOG.error("Placement Error: %s", vim_util.serialize_object( + result.drsFault), instance=instance) + + if (not hasattr(result, 'recommendations') or + not result.recommendations): + raise exception.MigrationError( + reason="PlaceVM did not give any recommendations") + + rs = sorted([r for r in result.recommendations + if r.reason == "xvmotionPlacement" and + r.action], + key=attrgetter("rating")) + if not rs: + raise exception.MigrationError( + reason="Did not get any xvmotionPlacement") + + relocate_spec = rs[0].action[0].relocateSpec + + # Should never happen, but if it does we rather want an error + # here, than sometime down the line + if not relocate_spec.host: + raise exception.MigrationError( + reason="No host with enough resources") + + # Samere here: Should never happen + if not relocate_spec.datastore: + raise exception.MigrationError( + reason="No datastore with enough resources") + + # relocate_defaults are serialized/deserialized on put/get + defaults = migrate_data.relocate_defaults + spec = vim_util.serialize_object(relocate_spec) + defaults["relocate_spec"] = spec + # Writing the values back + migrate_data.relocate_defaults = defaults + + return migrate_data + + def _get_checked_volumes(self, context, instance, required_values, + exc=exception.MigrationError): + volumes = self._get_volume_mappings(context, instance) + for key, volume in volumes.items(): + for v in required_values: + if v not in volume: + message = "Missing {} for volume {} (Device {})".format( + v, volume.get("volume_id"), key + ) + raise exc(message) + return volumes + + def live_migration(self, context, instance, dest, + post_method, recover_method, block_migration=False, + migrate_data=None): + """Live migration of an instance to another host.""" + if not migrate_data: + LOG.error("live_migration() called without migration_data" + " - cannot continue operations", instance=instance) + recover_method(context, instance, dest, migrate_data) + raise ValueError("Missing migrate_data") + + if migrate_data.instance_already_migrated: + LOG.info("Recovering migration", instance=instance) + post_method(context, instance, dest, block_migration, migrate_data) + return + + # We require the target-datastore for all volume-attachment + required_volume_attributes = ["datastore_ref"] + if migrate_data.is_same_vcenter: + dest_session = self._session + else: + # For the shadow vm fixup after migration + dest_session = self._create_dest_session(migrate_data) + required_volume_attributes.append('volume') + + try: + # Validate that we have all necessary information for the + # new volume attachments + # This cannot be done in pre-check, as the new volume attachments + # are created prior the live-migration + volumes = self._get_checked_volumes(context, instance, + required_volume_attributes) + self._set_vif_infos(migrate_data, dest_session) + self._vmops.live_migration(instance, migrate_data, volumes) + LOG.info("Migration operation completed", instance=instance) + post_method(context, instance, dest, block_migration, migrate_data) + except Exception: + LOG.exception("Failed due to an exception", instance=instance) + with excutils.save_and_reraise_exception(): + # We are still in the task-state migrating, so cannot + # recover the DRS settings. We rely on the sync to do that + LOG.debug("Calling live migration recover_method " + "for instance: %s", instance["name"], + instance=instance) + recover_method(context, instance, dest, migrate_data) + + def _get_volume_mappings(self, context, instance): + """Returns a mapping for all the devices with volumes to + their connection_info.data, where possible. + It is up to the caller to verify that the required information + is available. + It uses the _current_ block-device mapping. + """ + volume_infos = [] + for bdm in objects.BlockDeviceMappingList.get_by_instance_uuid( + context, instance.uuid): + if not bdm.is_volume: + continue + # If we have volume_id, `map_volumes_to_devices` will map it to + # a device. + data = {"volume_id": bdm.volume_id} + try: + connection_info = jsonutils.loads(bdm.connection_info) + data.update(connection_info.get("data", {})) + except ValueError as e: + message = ("Could not parse connection_info for volume {}." + " Reason: {}" + ).format(bdm.volume_id, e) + LOG.warning(message, instance=instance) + + # Normalize the datastore reference + # As it depends on the caller, if actually need the + # value, we do not raise an error here + datastore = data.get("datastore") or \ + data.get("ds_ref_val") + if datastore: + data["datastore_ref"] = vim_util.get_moref(datastore, + "Datastore") + volume_infos.append(data) + return self._volumeops.map_volumes_to_devices(instance, volume_infos) + + def _set_vif_infos(self, migrate_data, session): + if 'vifs' not in migrate_data or not migrate_data.vifs: + migrate_data.vif_infos = [] + return + + cluster_ref = vim_util.get_moref(migrate_data.dest_cluster_ref, + "ClusterComputeResource") + dest_network_info = [ + vif.get_dest_vif() + for vif in migrate_data.vifs + ] + + vif_model = None # We are not reading that value + migrate_data.vif_infos = vmwarevif.get_vif_info(session, + cluster_ref, utils.is_neutron(), vif_model, dest_network_info) + + def _create_dest_session(self, migrate_data): + defaults = migrate_data.relocate_defaults + service = defaults["service"] + credentials = service["credentials"] + dest_url = urllib.parse.urlparse(service["url"]) + + dest_session = session.VMwareAPISession( + host_ip=dest_url.hostname, + host_port=dest_url.port, + username=credentials["username"], + password=credentials["password"], + # TODO(fwiesel): SSL Settings + ) + + return dest_session + + def rollback_live_migration_at_destination(self, context, instance, + network_info, + block_device_info, + destroy_disks=True, + migrate_data=None): + """Clean up destination node after a failed live migration.""" + LOG.info("rollback_live_migration_at_destination %s", + block_device_info, instance=instance) + if not migrate_data.is_same_vcenter: + self._volumeops.delete_shadow_vms(block_device_info, instance) + + @contextlib.contextmanager + def _error_out_instance_on_exception(self, instance, message): + try: + yield + except Exception as error: + LOG.exception("Failed to %s, setting to ERROR state", + message, + instance=instance, error=error) + instance.vm_state = vm_states.ERROR + instance.save() + + def post_live_migration(self, context, instance, block_device_info, + migrate_data=None): + """Post operation of live migration at source host.""" + if not migrate_data.is_same_vcenter: + with self._error_out_instance_on_exception(instance, + "delete shadow vms"): + self._volumeops.delete_shadow_vms(block_device_info, instance) + + with self._error_out_instance_on_exception(instance, + "sync server groups"): + self._vmops.sync_instance_server_group(context, instance) + + def post_live_migration_at_source(self, context, instance, network_info): + # This is mostly for network related cleanup tasks at the source + # There is nothing to do for us + pass + + def post_live_migration_at_destination(self, context, instance, + network_info, + block_migration=False, + block_device_info=None): + """Post operation of live migration at destination host.""" + with self._error_out_instance_on_exception(instance, + "disable drs"): + self._vmops.disable_drs_if_needed(instance) + + with self._error_out_instance_on_exception(instance, + "update cluster placement"): + self._vmops.update_cluster_placement(context, instance) + + with self._error_out_instance_on_exception(instance, + "fixup shadow vms"): + volumes = self._get_volume_mappings(context, instance) + LOG.debug("Fixing shadow vms %s", volumes, instance=instance) + self._volumeops.fixup_shadow_vms(instance, volumes) + + self._vmops._clean_up_after_special_spawning( + context, instance.memory_mb, instance.flavor) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index de39749210a..cb6ad6674f9 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -764,16 +764,23 @@ def sync_instance_server_group(self, context, instance): except nova.exception.InstanceGroupNotFound: pass - def update_admin_vm_group_membership(self, instance, remove=False): + @staticmethod + def _get_admin_group_name_for_instance(instance): vm_group_name = CONF.vmware.special_spawning_vm_group if not vm_group_name: - return + return None needs_empty_host = utils.vm_needs_special_spawning( int(instance.memory_mb), instance.flavor) if needs_empty_host: - return + return None + + return vm_group_name + def update_admin_vm_group_membership(self, instance, remove=False): + vm_group_name = self._get_admin_group_name_for_instance(instance) + if not vm_group_name: + return vm_ref = vm_util.get_vm_ref(self._session, instance) cluster_util.update_vm_group_membership(self._session, self._cluster, vm_group_name, vm_ref, @@ -1388,6 +1395,19 @@ def _clean_shutdown(self, instance, timeout, retry_interval): return False + def is_instance_in_resource_pool(self, instance): + try: + vm_ref = vm_util.get_vm_ref(self._session, instance) + res_pool = self._session._call_method(vutil, "get_object_property", + vm_ref, "resourcePool") + + return vutil.get_moref_value(res_pool) == \ + vutil.get_moref_value(self._root_resource_pool) + except (exception.InstanceNotFound, + vexc.ManagedObjectNotFoundException): + LOG.debug("Failed to find instance", instance=instance) + return False + def _get_instance_props(self, vm_ref): lst_properties = ["summary.guest.toolsStatus", "runtime.powerState", @@ -1773,6 +1793,88 @@ def _relocate_vm(self, vm_ref, context, instance, network_info, vm_util.relocate_vm(self._session, vm_ref, spec=spec) + def live_migration(self, instance, migrate_data, volume_mapping): + defaults = migrate_data.relocate_defaults + relocate_spec_defaults = defaults["relocate_spec"] + disk_move_type = relocate_spec_defaults.get("diskMoveType", + "moveAllDiskBackingsAndDisallowSharing") + + def moref(item): + v = relocate_spec_defaults[item] + return vutil.get_moref(v["value"], v["_type"]) + + client_factory = self._session.vim.client.factory + datastore = moref("datastore") + relocate_spec = vm_util.relocate_vm_spec(client_factory, + res_pool=moref("pool"), + datastore=datastore, + host=moref("host"), + disk_move_type=disk_move_type, + folder=moref("folder")) + + service = defaults.get("service") + if service: + credentials_dict = service.pop("credentials") + credentials = client_factory.create( + "ns0:" + credentials_dict.pop("_type")) + + for k, v in credentials_dict.items(): + setattr(credentials, k, v) + + relocate_spec.service = vm_util.create_service_locator( + client_factory, + service["url"], + service["instance_uuid"], + credentials, + service["ssl_thumbprint"], + ) + + vm_ref = vm_util.get_vm_ref(self._session, instance) + + device_config_spec = [] + relocate_spec.deviceChange = device_config_spec + disks = [] + relocate_spec.disk = disks + + netdevices = [] + for device in vm_util.get_hardware_devices(self._session, vm_ref): + class_name = device.__class__.__name__ + if class_name in vm_util.ALL_SUPPORTED_NETWORK_DEVICES: + netdevices.append(device) + elif class_name == "VirtualDisk": + locator = client_factory.create( + "ns0:VirtualMachineRelocateSpecDiskLocator") + locator.diskId = device.key + target = volume_mapping.get(device.key) + if not target: # Not a volume + locator.datastore = datastore + else: + locator.datastore = target["datastore_ref"] + profile_id = target.get("profile_id") + if profile_id: + profile_spec = client_factory.create( + "ns0:VirtualMachineDefinedProfileSpec") + profile_spec.profileId = profile_id + locator.profile = [profile_spec] + disks.append(locator) + + for vif_info in migrate_data.vif_infos: + device = vmwarevif.get_network_device(netdevices, + vif_info["mac_address"]) + if not device: + msg = _("No device with MAC address %s exists on the " + "VM") % vif_info["mac_address"] + raise exception.NotFound(msg) + + # Update the network device backing + config_spec = client_factory.create("ns0:VirtualDeviceConfigSpec") + vm_util.set_net_device_backing(client_factory, device, vif_info) + config_spec.operation = "edit" + config_spec.device = device + device_config_spec.append(config_spec) + + vm_util.relocate_vm(self._session, vm_ref, spec=relocate_spec) + def _detach_volumes(self, instance, block_device_info): block_devices = driver.block_device_info_get_mapping(block_device_info) for disk in block_devices: @@ -1834,59 +1936,6 @@ def _find_datastore_for_migration(self, instance, vm_ref, cluster_ref, return ds_util.get_datastore(self._session, cluster_ref, datastore_regex) - def live_migration(self, context, instance, dest, - post_method, recover_method, block_migration, - migrate_data): - LOG.debug("Live migration data %s", migrate_data, instance=instance) - vm_ref = vm_util.get_vm_ref(self._session, instance) - cluster_name = migrate_data.cluster_name - cluster_ref = vm_util.get_cluster_ref_by_name(self._session, - cluster_name) - datastore_regex = re.compile(migrate_data.datastore_regex) - res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref) - # find a datastore where the instance will be migrated to - ds = self._find_datastore_for_migration(instance, vm_ref, cluster_ref, - datastore_regex) - if ds is None: - LOG.error("Cannot find datastore", instance=instance) - raise exception.HostNotFound(host=dest) - LOG.debug("Migrating instance to datastore %s", ds.name, - instance=instance) - # find ESX host in the destination cluster which is connected to the - # target datastore - esx_host = self._find_esx_host(cluster_ref, ds.ref) - if esx_host is None: - LOG.error("Cannot find ESX host for live migration, cluster: %s, " - "datastore: %s", migrate_data.cluster_name, ds.name, - instance=instance) - raise exception.HostNotFound(host=dest) - # Update networking backings - network_info = instance.get_network_info() - client_factory = self._session.vim.client.factory - devices = [] - hardware_devices = vm_util.get_hardware_devices(self._session, vm_ref) - vif_model = instance.image_meta.properties.get('hw_vif_model', - constants.DEFAULT_VIF_MODEL) - for vif in network_info: - vif_info = vmwarevif.get_vif_dict( - self._session, cluster_ref, vif_model, vif) - device = vmwarevif.get_network_device(hardware_devices, - vif['address']) - devices.append(vm_util.update_vif_spec(client_factory, vif_info, - device)) - - LOG.debug("Migrating instance to cluster '%s', datastore '%s' and " - "ESX host '%s'", cluster_name, ds.name, esx_host, - instance=instance) - try: - vm_util.relocate_vm(self._session, vm_ref, res_pool_ref, - ds.ref, esx_host, devices=devices) - LOG.info("Migrated instance to host %s", dest, instance=instance) - except Exception: - with excutils.save_and_reraise_exception(): - recover_method(context, instance, dest, migrate_data) - post_method(context, instance, dest, block_migration, migrate_data) - def poll_rebooting_instances(self, timeout, instances): """Poll for rebooting instances.""" ctxt = nova_context.get_admin_context() @@ -2630,3 +2679,63 @@ def _update_rule(rule_name, expected_members, sg): rule_name, ', '.join(expected_members)) _sync_sync_server_group(context, sg_uuid) + + def place_vm(self, context, instance): + # We currently only fill the bare-minimum to get a placement. + # The datastore for the VM is selected as on instance creation, + # instead of allowing placevm to decide it (which may be better) + # We also do not pass the information about the NICs and other + # attached disks, which may give the placement a better information + client_factory = self._session.vim.client.factory + flavor = instance.flavor + image_meta = instance.image_meta + image_info = images.VMwareImage.from_image(context, + instance.image_ref, + image_meta) + + extra_specs = self._get_extra_specs(flavor, image_meta) + + vi = self._get_vm_config_info(instance, image_info, extra_specs) + + vm_folder = self._get_project_folder(vi.dc_info, + project_id=instance.project_id, type_='Instances') + + relocate_spec = vm_util.relocate_vm_spec( + client_factory, + res_pool=self._root_resource_pool, + datastore=vi.datastore.ref, + disk_move_type="moveAllDiskBackingsAndDisallowSharing", + folder=vm_folder) + + placement_spec = client_factory.create("ns0:PlacementSpec") + placement_spec.placementType = "relocate" + # Maybe we want to allow that? Default is True + # placement_spec.disallowPrerequisiteMoves = False + placement_spec.relocateSpec = relocate_spec + # So we do not place the vm on a failover host + placement_spec.hosts, _ = \ + vm_util.get_hosts_and_reservations_for_cluster( + self._session, self._cluster) + + # Sets cpuAllocation, memoryAllocation, numCPUs, memoryMB + config_spec = vm_util.get_vm_resize_spec(client_factory, + int(flavor.vcpus), + int(flavor.memory_mb), + extra_specs) + # The last mandatory field for config_spec per doc + config_spec.version = extra_specs.hw_version + placement_spec.configSpec = config_spec + + vm_group_name = self._get_admin_group_name_for_instance(instance) + if vm_group_name: + placement_rules = [] + placement_spec.rules = placement_rules + cluster_rules = cluster_util.fetch_cluster_rules(self._session, + self._cluster) + for rule in cluster_rules.values(): + if getattr(rule, "vmGroupName", None) == vm_group_name: + placement_rules.append(rule) + + result = self._session._call_method(self._session.vim, "PlaceVm", + self._cluster, placementSpec=placement_spec) + return result diff --git a/nova/virt/vmwareapi/volumeops.py b/nova/virt/vmwareapi/volumeops.py index 9bc195f192d..1f668dc48ac 100644 --- a/nova/virt/vmwareapi/volumeops.py +++ b/nova/virt/vmwareapi/volumeops.py @@ -25,6 +25,7 @@ import nova.conf from nova import exception from nova.i18n import _ +from nova.virt import driver from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import session from nova.virt.vmwareapi import vm_util @@ -671,3 +672,120 @@ def attach_root_volume(self, connection_info, instance, # we don't do that. This comment should help us detect upstream changes # to the function as merging should fail. self.attach_volume(connection_info, instance, adapter_type) + + def fixup_shadow_vms(self, instance, shadow_vms): + """The function ensures that all volumes attached to a VM are also + attached at the corresponding shadow vms. + + As the vcenter controls both VMs and volumes, and you can only move + them together in a single migration operation between vcenters, + Cinder cannot move the volumes independently. + It only can create the shadow-vms in the target vcenter. + The backing then gets moved through the live-migration and needs to + be reassociated with (attached to) the corresponding shadow-vms + + It handles the case that the shadow-vm is without backing (normal-case) + But also that there is a pre-existing backing (e.g. we retry a failed + migration) + In the latter case, we discard the existing attachment of the + shadow-vm, as the attached volume is the one with the more current data + + :param instance: The instance moved via a live-migration and to which + the volumes are attached to + :param shadow_vms: A mapping of devices with volumes to the + connection_info.data information, most importantly holding the "volume" + field, which is a managed-object reference to a shadow-vm + """ + # This should sensibly moved out to cinder: + # Cinder can delete the old shadow-vm, as soon as the attachment + # for the vm prior the vmotion gets deleted + vm_ref = vm_util.get_vm_ref(self._session, instance) + for device in vm_util.get_hardware_devices(self._session, vm_ref): + class_name = device.__class__.__name__ + + if class_name != "VirtualDisk": + continue + + data = shadow_vms.get(device.key) + if not data: + continue + + try: + current_device_path = device.backing.fileName + volume = data["volume"] + volume_ref = vutil.get_moref(volume, "VirtualMachine") + original_device = self._get_vmdk_base_volume_device(volume_ref) + if original_device: + original_device_path = original_device.backing.fileName + if original_device_path == current_device_path: + # Already attached + continue + + # That should not happen + LOG.warning("Shadow-vm %s already has a disk" + " attached at %s replacing it with %s", + volume, original_device_path, current_device_path, + instance=instance + ) + self.detach_disk_from_vm(volume_ref, instance, + original_device, destroy_disk=True) + + disk_type = vm_util._get_device_disk_type(device) + self.attach_disk_to_vm(volume_ref, + instance, + constants.DEFAULT_ADAPTER_TYPE, + disk_type, + current_device_path + ) + except Exception: + LOG.exception("Failed to attach volume %s. Device %s", + data["volume_id"], device.key, + instance=instance) + + def delete_shadow_vms(self, block_device_info, instance=None): + # We need to delete the migrated shadow vms + # (until we implement it in cinder) + block_device_mapping = driver.block_device_info_get_mapping( + block_device_info) + + if not block_device_mapping: + return + + session = self._session + deleted = [] + for disk in block_device_mapping: + connection_info = disk["connection_info"] + try: + data = connection_info["data"] + volume_ref = self._get_volume_ref(data) + destroy_task = session._call_method(session.vim, + "Destroy_Task", + volume_ref) + session._wait_for_task(destroy_task) + deleted.append("{volume_id} ({volume})".format(**data)) + except oslo_vmw_exceptions.ManagedObjectNotFoundException: + LOG.debug("Volume %s already deleted", + data.get("volume_id"), instance=instance) + except Exception: + LOG.exception("Failed to delete volume %s", + data.get("volume_id"), instance=instance) + + LOG.info("Deleted %s", deleted, instance=instance) + + def map_volumes_to_devices(self, instance, disk_infos): + """Maps a connection_info.data to a device of the instance by its key + :param instance: The instance the volumes are attached to + :param disk_infos: An iterable over a dict containing at least + 'volume_id' + :returns: A dict {device_key: disk_info} mapping the passed disk_info + dicts to a a device by the stored volume_id + """ + remapped = {} + vm_ref = vm_util.get_vm_ref(self._session, instance) + # TODO(fwiesel) Create a function + # _get_vmdk_backed_disk_devices (plural) + # so we do not have two calls for each device + for disk_info in disk_infos: + device = self._get_vmdk_backed_disk_device(vm_ref, disk_info) + remapped[device.key] = disk_info + return remapped From 850d4485ea67c97ec4283d419e63e283a5c0d5ab Mon Sep 17 00:00:00 2001 From: Jakob Karge Date: Fri, 5 Jan 2024 12:59:13 +0100 Subject: [PATCH 45/92] vmware: Clear MOref cache on source after migration When a VM moves away from a compute node, the MOref cache in nova.virt.vmwareapi.vm_util._VM_REFS_CACHE is not cleaned up. If the VM is then migrated back to the same node it once came from, _and_ it has volumes attached, the attachment creation fails in the VCenter API, because the MOref used is the stale one. Clean up the cache after moving away, so that if the VM comes back, the new MOref value is fetched and volume attachments work. Change-Id: Ib1f4b7cdd8c28924b674032a9ea92171b796bdb9 (cherry picked from commit 3a401be1acfee28e46f6d2cb87a0274906a7344d) --- nova/virt/vmwareapi/driver.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 77889be0c05..a9ee29045bb 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -1066,9 +1066,12 @@ def post_live_migration(self, context, instance, block_device_info, self._vmops.sync_instance_server_group(context, instance) def post_live_migration_at_source(self, context, instance, network_info): - # This is mostly for network related cleanup tasks at the source - # There is nothing to do for us - pass + # We have to clean up our VM-moref cache, so that when a VM with + # volume attachments comes back to this compute node after being moved + # away, the moref is not incorrect/stale and the VMware API, on volume + # attachment creation, doesn't complain that the VM + # "has already been deleted or has not been completely created". + vm_util.vm_ref_cache_delete(instance.uuid) def post_live_migration_at_destination(self, context, instance, network_info, From 8594130def71b75cd57680daa0d1185058e33dd2 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 4 Oct 2021 16:00:14 +0200 Subject: [PATCH 46/92] Vmwareapi: Live-migration changes on conductor level. Split out the changes for live-migration which are affecting the conductor Basing it on the upstream VMWareLiveMigrateData 1.0 version in the hope that we have an easier path merging the code with upstream eventually Change-Id: I5d8417c836d735122e033eeb72f7671b2558afc4 (cherry picked from commit a2c335fd4c51ad26776a73fe52b22b07359f5f9d) (cherry picked from commit 6ee8e60a1473d0bd5e3dd55860f1afb05f8a9ed2) --- nova/conductor/tasks/live_migrate.py | 5 ++- nova/objects/migrate_data.py | 46 +++++++++++++++++++++++-- nova/tests/unit/objects/test_objects.py | 2 +- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/nova/conductor/tasks/live_migrate.py b/nova/conductor/tasks/live_migrate.py index 6ff834dca93..d0bce004896 100644 --- a/nova/conductor/tasks/live_migrate.py +++ b/nova/conductor/tasks/live_migrate.py @@ -172,7 +172,10 @@ def rollback(self, ex): self.migration) def _check_instance_is_active(self): - if self.instance.power_state not in (power_state.RUNNING, + # NOSTATE, as the VM might be already migrated, + # but we missed to update the db and we want to retry + if self.instance.power_state not in (power_state.NOSTATE, + power_state.RUNNING, power_state.PAUSED): raise exception.InstanceInvalidState( instance_uuid=self.instance.uuid, diff --git a/nova/objects/migrate_data.py b/nova/objects/migrate_data.py index 1340c34d971..e213c67438e 100644 --- a/nova/objects/migrate_data.py +++ b/nova/objects/migrate_data.py @@ -373,16 +373,58 @@ def obj_make_compatible(self, primitive, target_version): class VMwareLiveMigrateData(LiveMigrateData): # Version 1.0: Initial version # Version 1.1: Inherited pci_dev_map_src_dst from LiveMigrateData - VERSION = '1.1' + # Version 1.2: Added dest_cluster_ref, is_same_vcenter, + # instance_already_migrated, relocate_defaults_json, + # vif_infos_json for cross-vcenter migration + VERSION = '1.2' fields = { 'cluster_name': fields.StringField(nullable=False), 'datastore_regex': fields.StringField(nullable=False), + 'dest_cluster_ref': fields.StringField(nullable=False), + 'is_same_vcenter': fields.BooleanField(default=True), + 'instance_already_migrated': fields.BooleanField(default=False), + 'relocate_defaults_json': fields.SensitiveStringField(default="[]"), + 'vif_infos_json': fields.StringField(default="[]"), } + @property + def relocate_defaults(self): + return jsonutils.loads(self.relocate_defaults_json) + + @relocate_defaults.setter + def relocate_defaults(self, relocate_defaults_dict): + self.relocate_defaults_json = jsonutils.dumps(relocate_defaults_dict) + + @property + def vif_infos(self): + return jsonutils.loads(self.vif_infos_json) + + @vif_infos.setter + def vif_infos(self, vif_infos): + self.vif_infos_json = jsonutils.dumps(vif_infos) + + def to_legacy_dict(self, pre_migration_result=False): + legacy = super(VMwareLiveMigrateData, self).to_legacy_dict() + for field in self.fields: + if self.obj_attr_is_set(field): + legacy[field] = getattr(self, field) + return legacy + + def from_legacy_dict(self, legacy): + super(VMwareLiveMigrateData, self).from_legacy_dict(legacy) + for field in self.fields: + if field in legacy: + setattr(self, field, legacy[field]) + def obj_make_compatible(self, primitive, target_version): super(VMwareLiveMigrateData, self).obj_make_compatible( primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) - if (target_version < (1, 1)): + if target_version < (1, 2): + for k in ('dest_cluster_ref', 'is_same_vcenter', + 'instance_already_migrated', 'relocate_defaults_json', + 'vif_infos_json'): + primitive.pop(k, None) + if target_version < (1, 1): primitive.pop('pci_dev_map_src_dst', None) diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 2b99df6f960..70f82d83215 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1183,7 +1183,7 @@ def obj_name(cls): 'VirtCPUTopology': '1.0-fc694de72e20298f7c6bab1083fd4563', 'VirtualInterface': '1.3-efd3ca8ebcc5ce65fff5a25f31754c54', 'VirtualInterfaceList': '1.0-9750e2074437b3077e46359102779fc6', - 'VMwareLiveMigrateData': '1.1-14a63b00f1b8880453df6d19a883bc73', + 'VMwareLiveMigrateData': '1.2-69ac58f0c1f20686ddfdfae03ceb0929', 'VolumeUsage': '1.0-6c8190c46ce1469bb3286a1f21c2e475', 'XenDeviceBus': '1.0-272a4f899b24e31e42b2b9a7ed7e9194', } From d6add34d0a3c8e3e3be45686c47db599786d9a27 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 8 Nov 2019 11:08:45 +0100 Subject: [PATCH 47/92] Add `_create_host_group_spec()` to cluster_util We will use that to create/update hostgroups like we do for vmgroups already. (cherry picked from commit 95e1b638c79c797249748fc82e3a82e1a7781f11) (cherry picked from commit 3f3ccf7fdb076209e0e1ebd3aafa8ad41725d505) Change-Id: I558fce33e10786c7e46097a96c89f26b66ba9541 (cherry picked from commit 4385063b84d0fdf7a53a8754082f410d27b116f3) --- nova/virt/vmwareapi/cluster_util.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 380a451905f..ceefb84b9ca 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -74,6 +74,22 @@ def create_group_spec(client_factory, group, operation): return group_spec +def _create_host_group_spec(client_factory, name, host_refs, operation="add", + group=None): + group = group or client_factory.create('ns0:ClusterHostGroup') + group.name = name + + if hasattr(group, 'host'): + group.host += host_refs + else: + group.host = host_refs + + group_spec = client_factory.create('ns0:ClusterGroupSpec') + group_spec.operation = operation + group_spec.info = group + return group_spec + + def _get_vm_group(cluster_config, group_name): if not hasattr(cluster_config, 'group'): return None From 88e17ab34bbec6754ddcf6dd5cf8c021dddc3e08 Mon Sep 17 00:00:00 2001 From: mmidolesov Date: Thu, 5 Apr 2018 01:05:39 -0700 Subject: [PATCH 48/92] [vmwareapi] Fix resizing of images We need to compare the actual file size to the flavor-requested one. (cherry picked from commit 845a11e9ffda291dc688b975dc8f9e6a4c3f1259) (cherry picked from commit de9c332230ff7108478c935a3d18199eaa917c4e) Change-Id: Iac69bf10e5321f60fefeaf854cf6f1d99a79a3ca (cherry picked from commit 3dc16f6b1eb37b68df14aae7b6dccc76546344e5) --- .../unit/virt/vmwareapi/test_driver_api.py | 12 +++- nova/tests/unit/virt/vmwareapi/test_vmops.py | 66 ++++++++++++------- nova/virt/vmwareapi/vmops.py | 13 +++- 3 files changed, 65 insertions(+), 26 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index df3ba5c7116..01200c7001f 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -804,9 +804,19 @@ def _fake_extend(instance, requested_size, name, dc_ref): vmwareapi_fake.assertPathExists(self, str(root)) self.assertEqual(1, fake_extend_virtual_disk[0].call_count) + @mock.patch.object(ds_util, 'get_datastore') @mock.patch.object(nova.virt.vmwareapi.images.VMwareImage, 'from_image') - def test_spawn_disk_extend_sparse(self, mock_from_image): + def test_spawn_disk_extend_sparse(self, mock_from_image, + mock_get_datastore): + fake_ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds1') + fake_ds = ds_obj.Datastore( + ref=fake_ds_ref, name='ds1', + capacity=10 * units.Gi, + freespace=10 * units.Gi) + + mock_get_datastore.return_value = fake_ds + img_props = images.VMwareImage( image_id=self.fake_image_uuid, file_size=units.Ki, diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index babd20e8ce8..feca5c4199c 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -61,7 +61,7 @@ def __eq__(self, ds_path_param): return str(ds_path_param) == self.expected_ds_path_str -class VMwareVMOpsTestCase(test.NoDBTestCase): +class VMwareVMOpsTestCase(test.TestCase): def setUp(self): super(VMwareVMOpsTestCase, self).setUp() ds_util.dc_cache_reset() @@ -1420,18 +1420,19 @@ def test_prepare_for_spawn_invalid_ram(self): @mock.patch.object(vmops.VMwareVMOps, 'build_virtual_machine') @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.lockutils, 'lock') - def test_spawn_mask_block_device_info_password(self, mock_lock, - mock_update_cluster_placement, - mock_build_virtual_machine, mock_get_vm_config_info, - mock_fetch_image_if_missing, mock_debug, mock_glance, - mock_is_volume_backed): + @mock.patch.object(ds_util, 'get_datastore') + def test_spawn_mask_block_device_info_password(self, mock_get_datastore, + mock_lock, mock_update_cluster_placement, + mock_build_virtual_machine, mock_get_vm_config_info, + mock_fetch_image_if_missing, mock_debug, mock_glance, + mock_is_volume_backed): # Very simple test that just ensures block_device_info auth_password # is masked when logged; the rest of the test just fails out early. data = {'auth_password': 'scrubme'} bdm = [{'boot_index': 0, 'disk_bus': constants.DEFAULT_ADAPTER_TYPE, 'connection_info': {'data': data}}] bdi = {'block_device_mapping': bdm} - + mock_get_datastore.return_value = self._ds self.password_logged = False # Tests that the parameters to the to_xml method are sanitized for @@ -1445,11 +1446,26 @@ def fake_debug(*args, **kwargs): self.flags(flat_injected=False) self.flags(enabled=False, group='vnc') - mock_vi = mock.Mock() - mock_vi.root_gb = 1 - mock_vi.ii.file_size = 2 * units.Gi - mock_vi.instance.flavor.root_gb = 1 - mock_get_vm_config_info.return_value = mock_vi + extra_specs = vm_util.ExtraSpecs() + flavor_fits_image = False + file_size = 10 * units.Gi if flavor_fits_image else 5 * units.Gi + image_info = images.VMwareImage( + image_id=self._image_id, + file_size=file_size, + linked_clone=False) + + cache_root_folder = self._ds.build_path("vmware_base", self._image_id) + mock_imagecache = mock.Mock() + mock_imagecache.get_image_cache_folder.return_value = cache_root_folder + dc_info = ds_util.DcInfo( + ref=self._cluster, name='fake_dc', + vmFolder=vmwareapi_fake.ManagedObjectReference( + name='Folder', + value='fake_vm_folder')) + vi = vmops.VirtualMachineInstanceConfigInfo( + self._instance, image_info, + self._ds, dc_info, mock_imagecache, extra_specs) + mock_get_vm_config_info.return_value = vi # Call spawn(). We don't care what it does as long as it generates # the log message, which we check below. @@ -1692,9 +1708,11 @@ def test_get_ds_browser(self): @mock.patch.object( vmops.VMwareVMOps, '_sized_image_exists', return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_extend_virtual_disk') + @mock.patch.object(vmops.VMwareVMOps, '_extend_if_required') @mock.patch.object(vm_util, 'copy_virtual_disk') def _test_use_disk_image_as_linked_clone(self, mock_copy_virtual_disk, + mock_extend_if_required, mock_extend_virtual_disk, mock_sized_image_exists, flavor_fits_image=False): @@ -1725,11 +1743,10 @@ def _test_use_disk_image_as_linked_clone(self, str(vi.cache_image_path), str(sized_cached_image_ds_loc)) - if not flavor_fits_image: - mock_extend_virtual_disk.assert_called_once_with( - self._instance, vi.root_gb * units.Mi, - str(sized_cached_image_ds_loc), - self._dc_info.ref) + mock_extend_if_required.assert_called_once_with( + self._dc_info, + vi.ii, + vi.instance, str(sized_cached_image_ds_loc)) mock_attach_disk_to_vm.assert_called_once_with( "fake_vm_ref", self._instance, vi.ii.adapter_type, @@ -1745,9 +1762,11 @@ def test_use_disk_image_as_linked_clone_flavor_fits_image(self): self._test_use_disk_image_as_linked_clone(flavor_fits_image=True) @mock.patch.object(vmops.VMwareVMOps, '_extend_virtual_disk') + @mock.patch.object(vmops.VMwareVMOps, '_extend_if_required') @mock.patch.object(vm_util, 'copy_virtual_disk') def _test_use_disk_image_as_full_clone(self, mock_copy_virtual_disk, + mock_extend_if_required, mock_extend_virtual_disk, flavor_fits_image=False): extra_specs = vm_util.ExtraSpecs() @@ -1776,10 +1795,10 @@ def _test_use_disk_image_as_full_clone(self, str(vi.cache_image_path), fake_path) - if not flavor_fits_image: - mock_extend_virtual_disk.assert_called_once_with( - self._instance, vi.root_gb * units.Mi, - fake_path, self._dc_info.ref) + mock_extend_if_required.assert_called_once_with( + self._dc_info, + vi.ii, + vi.instance, fake_path) mock_attach_disk_to_vm.assert_called_once_with( "fake_vm_ref", self._instance, vi.ii.adapter_type, @@ -1950,10 +1969,11 @@ def _test_spawn(self, mock.patch.object(self._vmops, '_get_extra_specs', return_value=extra_specs), mock.patch.object(self._vmops, '_get_instance_metadata', - return_value='fake-metadata') + return_value='fake-metadata'), + mock.patch.object(ds_util, 'file_size', return_value=0) ) as (_wait_for_task, _call_method, _generate_uuid, _fetch_image, _get_img_svc, _get_inventory_path, _get_extra_specs, - _get_instance_metadata): + _get_instance_metadata, file_size): self._vmops.spawn(self._context, self._instance, image, injected_files='fake_files', admin_password='password', diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index cb6ad6674f9..21277ba79ab 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -208,8 +208,17 @@ def _delete_datastore_file(self, datastore_path, dc_ref): def _extend_if_required(self, dc_info, image_info, instance, root_vmdk_path): - """Increase the size of the root vmdk if necessary.""" - if instance.flavor.root_gb * units.Gi > image_info.file_size: + root_vmdk = ds_obj.DatastorePath.parse( + root_vmdk_path.replace(".vmdk", "-flat.vmdk")) + + datastore = ds_util.get_datastore(self._session, dc_info.ref, + re.compile(r"^{}$".format( + root_vmdk.datastore))) + ds_browser = self._get_ds_browser(datastore.ref) + actual_file_size = ds_util.file_size(self._session, ds_browser, + root_vmdk.parent, + root_vmdk.basename) + if instance.flavor.root_gb * units.Gi > actual_file_size: size_in_kb = instance.flavor.root_gb * units.Mi self._extend_virtual_disk(instance, size_in_kb, root_vmdk_path, dc_info.ref) From f13b01cabe94f3879d0261f8b826c5794262acaf Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Tue, 8 Feb 2022 15:51:44 +0100 Subject: [PATCH 49/92] Vmware: Use a more generic serialisation of specs to json The previous version was doing it hand-coded, while this one uses the type-system, which allows us to serialise only the necessary fields (non-null), and also serialise polymorphic objects. Change-Id: I0dcafb74b3494185b3b58d78cb501069675aea33 (cherry picked from commit d77cdcbad19a7a482a9be980113efc7877efa312) (cherry picked from commit 6129d5ea6503550084b7383131a0e9ab45527557) --- nova/objects/migrate_data.py | 2 +- nova/tests/unit/objects/test_objects.py | 2 +- .../unit/virt/vmwareapi/test_driver_api.py | 25 +++--- nova/virt/vmwareapi/driver.py | 58 ++++++------ nova/virt/vmwareapi/vim_util.py | 88 +++++++++++++++++++ nova/virt/vmwareapi/vmops.py | 41 +++------ 6 files changed, 147 insertions(+), 69 deletions(-) diff --git a/nova/objects/migrate_data.py b/nova/objects/migrate_data.py index e213c67438e..a2516eb1582 100644 --- a/nova/objects/migrate_data.py +++ b/nova/objects/migrate_data.py @@ -384,7 +384,7 @@ class VMwareLiveMigrateData(LiveMigrateData): 'dest_cluster_ref': fields.StringField(nullable=False), 'is_same_vcenter': fields.BooleanField(default=True), 'instance_already_migrated': fields.BooleanField(default=False), - 'relocate_defaults_json': fields.SensitiveStringField(default="[]"), + 'relocate_defaults_json': fields.SensitiveStringField(default="{}"), 'vif_infos_json': fields.StringField(default="[]"), } diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 70f82d83215..f8aa712b284 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1183,7 +1183,7 @@ def obj_name(cls): 'VirtCPUTopology': '1.0-fc694de72e20298f7c6bab1083fd4563', 'VirtualInterface': '1.3-efd3ca8ebcc5ce65fff5a25f31754c54', 'VirtualInterfaceList': '1.0-9750e2074437b3077e46359102779fc6', - 'VMwareLiveMigrateData': '1.2-69ac58f0c1f20686ddfdfae03ceb0929', + 'VMwareLiveMigrateData': '1.2-093beb7b7866660440a44c441166c0f3', 'VolumeUsage': '1.0-6c8190c46ce1469bb3286a1f21c2e475', 'XenDeviceBus': '1.0-272a4f899b24e31e42b2b9a7ed7e9194', } diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 01200c7001f..8bb5a4b7cc4 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -2429,8 +2429,9 @@ def _create_placement_result(self, *args, **kwargs): recommendations=[recommendation] ) + @mock.patch.object(vim_util, 'serialize_object') @mock.patch.object(vmops.VMwareVMOps, 'place_vm') - def test_pre_live_migration(self, mock_place_vm): + def test_pre_live_migration(self, mock_place_vm, mock_serialize_object): mock_place_vm.side_effect = self._create_placement_result self._create_instance() migrate_data = self._create_live_migrate_data() @@ -2441,12 +2442,14 @@ def test_pre_live_migration(self, mock_place_vm): self.assertEqual(result, migrate_data) @mock.patch.object(vm_util, 'relocate_vm') - @mock.patch.object(vm_util, 'create_service_locator') @mock.patch.object(vm_util, 'get_hardware_devices') + @mock.patch.object(vim_util, 'deserialize_object') @mock.patch.object(driver.VMwareVCDriver, '_get_volume_mappings', returns=[]) - def test_live_migration(self, get_volume_mappings, get_hardware_devices, - service_locator, relocate_vm): + @mock.patch.object(driver.VMwareVCDriver, '_create_dest_session') + def test_live_migration(self, create_dest_session, get_volume_mappings, + deserialize_object, + get_hardware_devices, relocate_vm): self._create_instance() migrate_data = self._create_live_migrate_data() @@ -2461,26 +2464,26 @@ def test_live_migration(self, get_volume_mappings, get_hardware_devices, 'fake_dest', post_method, recover_method, migrate_data=migrate_data) - service_locator.assert_called() + deserialize_object.assert_called() get_hardware_devices.assert_called() relocate_vm.assert_called() post_method.assert_called() recover_method.assert_not_called() - @mock.patch.object(vm_util, 'create_service_locator') - @mock.patch.object(volumeops.VMwareVolumeOps, - 'map_volumes_to_devices', returns=[]) + @mock.patch.object(driver.VMwareVCDriver, '_create_dest_session') + @mock.patch.object(driver.VMwareVCDriver, '_get_volume_mappings', + returns=[]) @mock.patch.object(vmops.VMwareVMOps, - 'live_migration', side_effect=Exception) + 'live_migration', side_effect=test.TestingException) def test_live_migration_failure_rollback(self, mock_live_migration, - volumes_to_devices, mock_service_locator): + get_volume_mappings, create_dest_session): self._create_instance() migrate_data = self._create_live_migrate_data() post_method = mock.Mock() recover_method = mock.Mock() - exception = Exception + exception = test.TestingException with test.nested( mock.patch.object(vm_util, 'get_vm_ref', diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index a9ee29045bb..af2436ed90a 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -799,18 +799,16 @@ def check_can_live_migrate_destination(self, context, instance, data.cluster_name = self._cluster_name data.dest_cluster_ref = vim_util.get_moref_value(self._cluster_ref) data.datastore_regex = CONF.vmware.datastore_regex + client_factory = self._session.client.factory + + service = vm_util.create_service_locator(client_factory, url, + self._vcenter_uuid, + vm_util.create_service_locator_name_password( + username=CONF.vmware.host_username, + password=CONF.vmware.host_password)) data.relocate_defaults = { - "service": { - "url": url, - "instance_uuid": self._vcenter_uuid, - "ssl_thumbprint": vm_util.get_sha1_ssl_thumbprint(url), - "credentials": { - "_type": "ServiceLocatorNamePassword", - "username": CONF.vmware.host_username, - "password": CONF.vmware.host_password, - } - } + "service": nova_vim_util.serialize_object(service, typed=True) } return data @@ -824,11 +822,14 @@ def check_can_live_migrate_source(self, context, instance, dest_check_data, block_device_info=None): """Check if it is possible to execute live migration.""" + client_factory = self._session.client.factory defaults = dest_check_data.relocate_defaults - dest_vcenter_uuid = defaults["service"]["instance_uuid"] + service_locator = nova_vim_util.deserialize_object(client_factory, + defaults["service"], + "ServiceLocator") dest_check_data.is_same_vcenter = ( - dest_vcenter_uuid == self._vcenter_uuid) + service_locator.instanceUuid == self._vcenter_uuid) if dest_check_data.instance_already_migrated: return dest_check_data @@ -836,7 +837,6 @@ def check_can_live_migrate_source(self, context, instance, if dest_check_data.is_same_vcenter: # Drop the service-credentials, no need to check the volumes, # as we do not need to mess around with them - defaults = dest_check_data.relocate_defaults defaults.pop("service") dest_check_data.relocate_defaults = defaults @@ -869,8 +869,8 @@ def _pre_live_migration(self, context, instance, block_device_info, result = self._vmops.place_vm(context, instance) if hasattr(result, 'drsFault'): - LOG.error("Placement Error: %s", vim_util.serialize_object( - result.drsFault), instance=instance) + LOG.error("Placement Error: %s", nova_vim_util.serialize_object( + result.drsFault, typed=True), instance=instance) if (not hasattr(result, 'recommendations') or not result.recommendations): @@ -900,7 +900,7 @@ def _pre_live_migration(self, context, instance, block_device_info, # relocate_defaults are serialized/deserialized on put/get defaults = migrate_data.relocate_defaults - spec = vim_util.serialize_object(relocate_spec) + spec = nova_vim_util.serialize_object(relocate_spec, typed=True) defaults["relocate_spec"] = spec # Writing the values back migrate_data.relocate_defaults = defaults @@ -936,14 +936,14 @@ def live_migration(self, context, instance, dest, # We require the target-datastore for all volume-attachment required_volume_attributes = ["datastore_ref"] - if migrate_data.is_same_vcenter: - dest_session = self._session - else: - # For the shadow vm fixup after migration - dest_session = self._create_dest_session(migrate_data) - required_volume_attributes.append('volume') - try: + if migrate_data.is_same_vcenter: + dest_session = self._session + else: + # For the shadow vm fixup after migration + dest_session = self._create_dest_session(migrate_data) + required_volume_attributes.append('volume') + # Validate that we have all necessary information for the # new volume attachments # This cannot be done in pre-check, as the new volume attachments @@ -1017,15 +1017,17 @@ def _set_vif_infos(self, migrate_data, session): def _create_dest_session(self, migrate_data): defaults = migrate_data.relocate_defaults - service = defaults["service"] - credentials = service["credentials"] - dest_url = urllib.parse.urlparse(service["url"]) + client_factory = self._session.client.factory + service_locator = nova_vim_util.deserialize_object(client_factory, + defaults["service"], + "ServiceLocator") + dest_url = urllib.parse.urlparse(service_locator.url) dest_session = session.VMwareAPISession( host_ip=dest_url.hostname, host_port=dest_url.port, - username=credentials["username"], - password=credentials["password"], + username=service_locator.credential.username, + password=service_locator.credential.password, # TODO(fwiesel): SSL Settings ) diff --git a/nova/virt/vmwareapi/vim_util.py b/nova/virt/vmwareapi/vim_util.py index 699660de40f..10df0c9524d 100644 --- a/nova/virt/vmwareapi/vim_util.py +++ b/nova/virt/vmwareapi/vim_util.py @@ -168,3 +168,91 @@ def get_array_items(array_obj): if hasattr(array_obj, attr_name): return getattr(array_obj, attr_name) return array_obj + + +# To be moved to oslo_vmware +def _get_child_type(sxtype, name): + if not sxtype: + return None + child, _ = sxtype.get_child(name) + return child.type[0] + + +# To be moved to oslo_vmware +def serialize_object(obj, typed=False, expected_type=None): + if obj is None: + return None + + if (obj.__class__.__name__ == 'ManagedObjectReference' and + not obj.value): + # Special case for suds, which creates "empty" references + # as a value object with an empty string + return None + + d = {} + if not typed or "sxtype" not in obj.__metadata__: + sxtype = None + else: + sxtype = obj.__metadata__["sxtype"] + if expected_type and sxtype.name != expected_type: + d["class"] = sxtype.name + + for k, v in dict(obj).items(): + if hasattr(v, "__keylist__"): + expected_type = _get_child_type(sxtype, k) + ser = serialize_object(v, + typed=typed, + expected_type=expected_type) + if not typed or ser: + d[k] = ser + elif isinstance(v, list): + expected_type = _get_child_type(sxtype, k) + ser = [] + for item in v: + if hasattr(item, "__keylist__"): + ser.append( + serialize_object(item, typed=typed, + expected_type=expected_type)) + else: + ser.append(item) + if not typed or ser: + d[k] = ser + else: + if not typed or v is not None: + d[k] = v + return d + + +def deserialize_object(factory, ser, typename): + if ser is None: + return ser + + if typename == "ManagedObjectReference": + # Special case: It is a simpleContent object (the only in the wsdl) + return vutil.get_moref(ser["value"], ser["_type"]) + + typename = ser.get("class", typename) + qtypename = "ns0:{}".format(typename) + obj = factory.create(qtypename) + try: + sxtype = obj.__metadata__["sxtype"] + except AttributeError: + sxtype = factory.resolver.find(qtypename) + + for child, _ in sxtype.children(): + v = ser.get(child.name, None) + + if isinstance(v, list): + if not v or not isinstance(v[0], dict): + deser = v + else: + type_ = child.type[0] + deser = [deserialize_object(factory, i, type_) for i in v] + elif isinstance(v, dict): + type_ = child.type[0] + deser = deserialize_object(factory, v, type_) + else: + deser = v + + setattr(obj, child.name, deser) + return obj diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 21277ba79ab..bc2a27cd78a 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -1804,39 +1804,24 @@ def _relocate_vm(self, vm_ref, context, instance, network_info, def live_migration(self, instance, migrate_data, volume_mapping): defaults = migrate_data.relocate_defaults - relocate_spec_defaults = defaults["relocate_spec"] - disk_move_type = relocate_spec_defaults.get("diskMoveType", - "moveAllDiskBackingsAndDisallowSharing") - - def moref(item): - v = relocate_spec_defaults[item] - return vutil.get_moref(v["value"], v["_type"]) client_factory = self._session.vim.client.factory - datastore = moref("datastore") - relocate_spec = vm_util.relocate_vm_spec(client_factory, - res_pool=moref("pool"), - datastore=datastore, - host=moref("host"), - disk_move_type=disk_move_type, - folder=moref("folder")) + relocate_spec = vim_util.deserialize_object(client_factory, + defaults["relocate_spec"], "VirtualMachineRelocateSpec") - service = defaults.get("service") - if service: - credentials_dict = service.pop("credentials") - credentials = client_factory.create( - "ns0:" + credentials_dict.pop("_type")) + if not migrate_data.is_same_vcenter: + disk_move_type = "moveAllDiskBackingsAndDisallowSharing" + else: + disk_move_type = "moveAllDiskBackingsAndAllowSharing" - for k, v in credentials_dict.items(): - setattr(credentials, k, v) + relocate_spec.diskMoveType = disk_move_type - relocate_spec.service = vm_util.create_service_locator( - client_factory, - service["url"], - service["instance_uuid"], - credentials, - service["ssl_thumbprint"], - ) + datastore = relocate_spec.datastore + + service = defaults.get("service") + if service: + relocate_spec.service = vim_util.deserialize_object( + client_factory, service, "ServiceLocator") vm_ref = vm_util.get_vm_ref(self._session, instance) From 350a3831a5e2c4d3ebde0082f1a6fe31d6744987 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Tue, 24 Oct 2023 15:24:42 +0200 Subject: [PATCH 50/92] Expose volume sizes to scheduler during resize/migration We want to allow customers to prevent resizes between shard during resizes, if the sum of the volume sizes hit a certain, Nova-configured threshold. Since we only need volume sizes for that, we just provide volumes sizes in the scheduler hint. The information we store in the "volume_sizes" scheduler-hint can change at any time so is not worth storing in the DB after the scheduling is done. Therefore, we remove it again once the resize got triggered and before saving the `RequestSpec`. Change-Id: If1ba024dd2337f26a38148179b16a7cfd82de64c (cherry picked from commit 4f84353ad27da45f3143cddd204166587146ad78) --- nova/conductor/manager.py | 14 +++++++ nova/tests/unit/conductor/test_conductor.py | 45 +++++++++++++++------ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/nova/conductor/manager.py b/nova/conductor/manager.py index 32b2fe8ec2a..b7a86c10967 100644 --- a/nova/conductor/manager.py +++ b/nova/conductor/manager.py @@ -348,6 +348,14 @@ def _get_request_spec_for_cold_migrate(context, instance, flavor, sh = filter_properties.setdefault('scheduler_hints', {}) sh.update(new_scheduler_hints) + volume_sizes = [bdm.volume_size for bdm in instance.get_bdms() + if bdm.destination_type == 'volume'] + sh = filter_properties.setdefault('scheduler_hints', {}) + # NOTE(jkulik): We have to update the "volume_sizes" key even if there + # are no volumes attached currently, because the "scheduler_hints" are + # persisted in the DB and thus could contain old information. + sh.update({'volume_sizes': volume_sizes}) + # NOTE(sbauza): If a reschedule occurs when prep_resize(), then # it only provides filter_properties legacy dict back to the # conductor with no RequestSpec part of the payload for Date: Mon, 7 Dec 2020 15:39:20 +0100 Subject: [PATCH 51/92] vmware: Add option for setting default hw_version This is necessary with multi-version clusters where migrating e.g. on resize needs to be possible. Since the vmware driver was explicitly overriding the hw_version attribute on ExtraSpecs and didn't use __init__(), we still ended up with no hw_version if the flavor didn't set one. Change-Id: Idc287c4dfa2b5d6a6a837a5014063417c8e13768 (cherry picked from commit 09b25470174798d1c5a0660dc5461995d6ee300c) (cherry picked from commit 5ee7da68162e91cd42f57f107983a8c0c112f9f2) --- nova/conf/vmware.py | 13 +++++++++++++ .../tests/unit/virt/vmwareapi/test_vm_util.py | 19 +++++++++++++++++++ nova/tests/unit/virt/vmwareapi/test_vmops.py | 15 +++++++++++++++ nova/virt/vmwareapi/vm_util.py | 2 +- nova/virt/vmwareapi/vmops.py | 3 ++- 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index cd8225d1ea9..50ec71d62ad 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -343,6 +343,19 @@ Possible values: * integer >= time in seconds to sleep between runs * integer < 0: disable the sync-loop +"""), + cfg.StrOpt('default_hw_version', + default=None, + help=""" +Set a default hardware version for VMs, that can be overridden in flavors + +This version is especially useful in multi-cluster environments where clusters +get upgraded individually but movement of VMs is necessary between them +nonetheless. It is recommend to set this to the highest version supported by +all clusters in the environment. + +Example: "vmx-13" for vSphere 6.5. +Versions can be looked up here: https://kb.vmware.com/s/article/1003746 """), ] diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index e479ba87327..130e819c9c2 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -25,6 +25,7 @@ from oslo_vmware import pbm from oslo_vmware import vim_util as vutil +import nova.conf from nova import exception from nova.network import model as network_model from nova import test @@ -35,6 +36,8 @@ from nova.virt.vmwareapi import session as vmware_session from nova.virt.vmwareapi import vm_util +CONF = nova.conf.CONF + class partialObject(object): def __init__(self, path='fake-path'): @@ -1038,6 +1041,22 @@ def test_get_vm_create_spec_with_firmware(self): expected.tools.beforeGuestStandby = True self.assertEqual(expected, result) + def test_get_vm_create_spec_with_default_hw_version(self): + CONF.set_override('default_hw_version', 'vmx-13', + 'vmware') + extra_specs = vm_util.ExtraSpecs() + fake_factory = fake.FakeFactory() + result = vm_util.get_vm_create_spec(fake_factory, + self._instance, + 'fake-datastore', [], + extra_specs) + + expected = self._create_vm_config_spec() + expected.memoryReservationLockedToMax = False + expected.version = 'vmx-13' + + self.assertEqual(expected, result) + def test_create_vm(self): def fake_call_method(module, method, *args, **kwargs): diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index feca5c4199c..1c16b66758a 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -2503,6 +2503,8 @@ def _validate_extra_specs(self, expected, actual): actual.cpu_limits.shares_level) self.assertEqual(expected.cpu_limits.shares_share, actual.cpu_limits.shares_share) + self.assertEqual(expected.hw_version, + actual.hw_version) def _validate_flavor_extra_specs(self, flavor_extra_specs, expected): # Validate that the extra specs are parsed correctly @@ -2670,6 +2672,19 @@ def test_extra_specs_vif_shares_with_invalid_level(self): self.assertRaises(exception.InvalidInput, self._validate_flavor_extra_specs, flavor_extra_specs, extra_specs) + def test_extra_specs_hw_version_override(self): + CONF.set_override('default_hw_version', 'vmx-13', + 'vmware') + flavor_extra_specs = {'vmware:hw_version': 'vmx-14'} + extra_specs = vm_util.ExtraSpecs(hw_version='vmx-14') + self._validate_flavor_extra_specs(flavor_extra_specs, extra_specs) + + def test_extra_specs_hw_version_override_empty(self): + CONF.set_override('default_hw_version', 'vmx-13', + 'vmware') + extra_specs = vm_util.ExtraSpecs(hw_version='vmx-13') + self._validate_flavor_extra_specs({}, extra_specs) + def _make_vm_config_info(self, is_iso=False, is_sparse_disk=False, vsphere_location=None): disk_type = (constants.DISK_TYPE_SPARSE if is_sparse_disk diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index cace226af1c..eff6e666af1 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -112,7 +112,7 @@ def __init__(self, cpu_limits=None, hw_version=None, self.memory_limits = memory_limits or Limits() self.disk_io_limits = disk_io_limits or Limits() self.vif_limits = vif_limits or Limits() - self.hw_version = hw_version + self.hw_version = hw_version or CONF.vmware.default_hw_version self.storage_policy = storage_policy self.cores_per_socket = cores_per_socket self.hv_enabled = hv_enabled diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index bc2a27cd78a..6293f94d2c8 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -341,7 +341,8 @@ def _get_extra_specs(self, flavor, image_meta=None): extra_specs.firmware = 'efi' elif hw_firmware_type == fields.FirmwareType.BIOS: extra_specs.firmware = 'bios' - hw_version = flavor.extra_specs.get('vmware:hw_version') + hw_version = flavor.extra_specs.get('vmware:hw_version', + CONF.vmware.default_hw_version) hv_enabled = flavor.extra_specs.get('vmware:hv_enabled') extra_specs.hv_enabled = hv_enabled extra_specs.hw_version = hw_version From a0eaffea6e43b329536ab99ab83dbb4af923c2f9 Mon Sep 17 00:00:00 2001 From: Andrew Karpow Date: Fri, 4 Jan 2019 15:14:40 +0100 Subject: [PATCH 52/92] flatten power_sync_state to intervall duration (cherry picked from commit a78c03e788b14a79df993dc68bd4c8fd805d92e9) (cherry picked from commit 70e96767c9a65b573b49b418a2119fe3066eff5f) Change-Id: Iae0593824a25f97aa2804d489f9dceb49e7d2ffa (cherry picked from commit 599e44399c8a3a38a19dea0d36ebd8cd3d41d043) --- nova/compute/manager.py | 3 +++ nova/tests/unit/compute/test_compute.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 425b7342c1d..56862590531 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -32,6 +32,7 @@ import functools import inspect import math +import random import sys import time import traceback @@ -11008,6 +11009,8 @@ def query_driver_power_state_and_sync(): self._query_driver_power_state_and_sync(context, db_instance) try: + greenthread.sleep(random.randint( + 1, CONF.sync_power_state_interval)) query_driver_power_state_and_sync() except Exception: LOG.exception("Periodic sync_power_state task had an " diff --git a/nova/tests/unit/compute/test_compute.py b/nova/tests/unit/compute/test_compute.py index b0f7de18429..43f333fd061 100644 --- a/nova/tests/unit/compute/test_compute.py +++ b/nova/tests/unit/compute/test_compute.py @@ -1671,6 +1671,9 @@ def setUp(self): 'm1.small') self.tiny_flavor = objects.Flavor.get_by_name(self.context, 'm1.tiny') + self.stub_out('eventlet.greenthread.sleep', + lambda *a, **kw: None) + def fake_share_info(self): share_mapping = {} share_mapping['id'] = 1 From fd5c30fdad6b6fb6f197ecb580921ea14ef82796 Mon Sep 17 00:00:00 2001 From: Martin Midolesov Date: Fri, 22 Nov 2019 06:30:12 -0800 Subject: [PATCH 53/92] Remove @safe_connect from _get_provider_aggregates Backport of fix that can be found here: https://review.opendev.org/#/c/678958/ ATTENTION: There are more changes in that fix, that we didn't have to backport for queens. Please check the other changes when re-applying to newer versions of nova. (cherry picked from commit 4790b8fda6b03abb284a370e7d945d03d9624c6b) (cherry picked from commit a3b78aff65f6582b9b8c51a96d942e366fe433ce) (cherry picked from commit 6d963b88c07b5055201e7d869e7f4feddf5a42dd) --- nova/scheduler/client/report.py | 1 - nova/tests/unit/scheduler/client/test_report.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/client/report.py b/nova/scheduler/client/report.py index 8f9abb9c936..dab692d36f5 100644 --- a/nova/scheduler/client/report.py +++ b/nova/scheduler/client/report.py @@ -364,7 +364,6 @@ def get_allocation_candidates(self, context, resources): LOG.error(msg, args) return None, None, None - @safe_connect def _get_provider_aggregates(self, context, rp_uuid): """Queries the placement API for a resource provider's aggregates. diff --git a/nova/tests/unit/scheduler/client/test_report.py b/nova/tests/unit/scheduler/client/test_report.py index ed43b7843da..6075e13a57c 100644 --- a/nova/tests/unit/scheduler/client/test_report.py +++ b/nova/tests/unit/scheduler/client/test_report.py @@ -2859,6 +2859,17 @@ def test_get_provider_aggregates_error(self, log_mock): self.ks_adap_mock.get.reset_mock() log_mock.reset_mock() + def test_get_provider_aggregates_conn_failure(self): + self.ks_adap_mock.get.side_effect = ks_exc.EndpointNotFound() + self.assertRaises( + ks_exc.ClientException, + self.client._get_provider_aggregates, self.context, uuids.cn) + expected_url = '/resource_providers/' + uuids.cn + '/aggregates' + + self.ks_adap_mock.get.assert_called_once_with( + expected_url, microversion='1.19', + global_request_id=self.context.global_id) + class TestTraits(SchedulerReportClientTestCase): trait_api_kwargs = {'microversion': '1.6'} From 76a7902740747d95497a023fd46508fa10f2650e Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Tue, 24 Mar 2020 08:49:54 +0100 Subject: [PATCH 54/92] [vmware] Don't rely on defined image's hw_vif_model Not every image has a hw_vif_model attribute defined, as that's not enforced. In this case, we have to take the default model as defined by the constant. We've overlooked this handling. It's already done in other places of the code. Change-Id: I1a729c00791385f7e27e3e9df6cf9757d47eb499 (cherry picked from commit e1a513864fb5061494e0661b67da1e9f99f4d864) (cherry picked from commit 93c0fbe8ca140bf476bfdee20d00175ee0a3af0c) (cherry picked from commit 574bc08fbd714cb66f3ba2548dcf8c8a59563335) --- nova/virt/vmwareapi/vmops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 6293f94d2c8..7be05592624 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -1774,7 +1774,8 @@ def _relocate_vm(self, vm_ref, context, instance, network_info, # Iterate over the network adapters and update the backing if network_info: spec.deviceChange = [] - vif_model = image_meta.properties.hw_vif_model + vif_model = image_meta.properties.get('hw_vif_model', + constants.DEFAULT_VIF_MODEL) hardware_devices = self._session._call_method( vutil, "get_object_property", From 493607c505f0d44d1b5b7edd49c02f2768adfb4f Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 31 Aug 2020 14:09:10 +0200 Subject: [PATCH 55/92] vmware: Let destroy_vm() re-raise InvalidArgument fault There are cases, where we don't want to blindly try and delete a VM, but instead know the outcome for certain types of errors at least. For supporting this case, vm_util.destroy_vm() has to re-raise certain types of exceptions. In the case we're looking for, that's the "InvalidArgument" fault. For this to work like before, code calling destroy_vm() needs to catch these exceptions, too. Change-Id: I28d2f1b94b8439cfea88146746ae6e59d61f087c (cherry picked from commit e112c5e30fde77c44c80ee01069cfae85acf14e0) (cherry picked from commit 73a9a275deab22c93f4673526288f5dfae13f172) (cherry picked from commit d6a0c8f902adc45b0811d09a3be8deef658211eb) --- nova/virt/vmwareapi/vm_util.py | 7 +++++++ nova/virt/vmwareapi/vmops.py | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index eff6e666af1..a05760dcfb9 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -1407,6 +1407,13 @@ def destroy_vm(session, instance, vm_ref=None): vm_ref) session._wait_for_task(destroy_task) LOG.info("Destroyed the VM", instance=instance) + except vexc.VimFaultException as e: + with excutils.save_and_reraise_exception() as ctx: + LOG.exception('Destroy VM failed', instance=instance) + # we need the `InvalidArgument` fault to bubble out of this + # function so it can be acted upon on higher levels + if 'InvalidArgument' not in e.fault_list: + ctx.reraise = False except Exception: LOG.exception('Destroy VM failed', instance=instance) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 7be05592624..3aa51a9d266 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -1104,7 +1104,12 @@ def _get_vm_and_vmdk_attribs(): vmdk_size=vmdk.capacity_in_bytes) finally: if snapshot_vm_ref: - vm_util.destroy_vm(self._session, instance, snapshot_vm_ref) + try: + vm_util.destroy_vm(self._session, instance, + snapshot_vm_ref) + except Exception: + # exception is logged inside the function. we can continue. + pass # Deleting the snapshot after destroying the temporary VM created # based on it allows the instance vm's disks to be consolidated. # TODO(vui) Add handling for when vmdk volume is attached. From d5cf9b1d5a2771e786a174ce7f588026d81ac475 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 9 Sep 2020 15:43:02 +0200 Subject: [PATCH 56/92] vmware: Switch default VIF model Support for the E1000 driver is phasing out in guest operating systems, so we switch to E1000e, which is also faster and has more hardware offloading capabilities. Change-Id: I08ac32f914a57d3eb7328351a07a20a2ef212cb8 (cherry picked from commit 5e7556d880a285b77cfd7bb3c6912eb3b4f8140d) fix unit tests for vmware: Switch default VIF model (cherry picked from commit 425070fedba1cc8abec1c572f9257ae8752f8b9c) (cherry picked from commit a3b173aea351c581558bbf9afcd1dd11f882cdfc) (cherry picked from commit a88e068365219e07fb41a2a0e546a64e969a32cf) --- nova/tests/unit/virt/vmwareapi/test_driver_api.py | 2 +- nova/tests/unit/virt/vmwareapi/test_images.py | 2 +- nova/tests/unit/virt/vmwareapi/test_vmops.py | 6 +++--- nova/virt/vmwareapi/constants.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 8bb5a4b7cc4..9c9f92d1d26 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -415,7 +415,7 @@ def _check_vm_record(self, num_instances=1, powered_on=True, uuid=None): self.assertEqual(vm.get("summary.config.memorySizeMB"), self.type_data['memory_mb']) - self.assertEqual("ns0:VirtualE1000", + self.assertEqual("ns0:VirtualE1000e", vm.get("config.hardware.device").VirtualDevice[2].obj_name) if powered_on: # Check that the VM is running according to Nova diff --git a/nova/tests/unit/virt/vmwareapi/test_images.py b/nova/tests/unit/virt/vmwareapi/test_images.py index 7ec57425474..12ecb8d8c39 100644 --- a/nova/tests/unit/virt/vmwareapi/test_images.py +++ b/nova/tests/unit/virt/vmwareapi/test_images.py @@ -343,7 +343,7 @@ def test_image_defaults(self): self.assertEqual('otherGuest', image.os_type) self.assertEqual('lsiLogic', image.adapter_type) self.assertEqual('preallocated', image.disk_type) - self.assertEqual('e1000', image.vif_model) + self.assertEqual('e1000e', image.vif_model) def test_use_vsphere_location(self): image = self._image_build(None, True, vsphere_location='vsphere://ok') diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 1c16b66758a..16fcb88d7cf 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -2411,7 +2411,7 @@ def test_build_virtual_machine(self, mock_create_folder): # Test that the VM's network is configured as specified devices = vm.get('config.hardware.device').VirtualDevice for device in devices: - if device.obj_name != 'ns0:VirtualE1000': + if device.obj_name != 'ns0:VirtualE1000e': continue self.assertEqual(self._network_values['address'], device.macAddress) @@ -3192,7 +3192,7 @@ def test_attach_interface(self, mock_get_vm_ref, self._vmops._network_api = _network_api vif_info = vif.get_vif_dict(self._session, self._cluster, - 'VirtualE1000', self._network_values) + 'VirtualE1000e', self._network_values) extra_specs = vm_util.ExtraSpecs() mock_extra_specs.return_value = extra_specs self._vmops.attach_interface(self._context, self._instance, @@ -3296,7 +3296,7 @@ def test_attach_interface_with_limits(self, mock_get_vm_ref, self._vmops._network_api = _network_api vif_info = vif.get_vif_dict(self._session, self._cluster, - 'VirtualE1000', self._network_values) + 'VirtualE1000e', self._network_values) vif_limits = vm_util.Limits(shares_level='custom', shares_share=40) extra_specs = vm_util.ExtraSpecs(vif_limits=vif_limits) diff --git a/nova/virt/vmwareapi/constants.py b/nova/virt/vmwareapi/constants.py index f9a4d6d3837..137f82b37d5 100644 --- a/nova/virt/vmwareapi/constants.py +++ b/nova/virt/vmwareapi/constants.py @@ -46,7 +46,7 @@ DATASTORE_TYPE_VSAN = 'vsan' DATASTORE_TYPE_VVOL = 'VVOL' -DEFAULT_VIF_MODEL = network_model.VIF_MODEL_E1000 +DEFAULT_VIF_MODEL = network_model.VIF_MODEL_E1000E DEFAULT_OS_TYPE = "otherGuest" DEFAULT_ADAPTER_TYPE = "lsiLogic" DEFAULT_DISK_TYPE = DISK_TYPE_PREALLOCATED From 31af3f09a2fca8d03ee4d24404bec619146f1799 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 11 Feb 2021 16:43:47 +0100 Subject: [PATCH 57/92] vmware: Optionally attach disconnected serial ports We want to be able to configure new VMs with our currently-disabled vSPC services, because it doesn't seem to be possible to reconfigure a serial port on a running VM - other than setting it to connected and startConnected. Therefore, we add a config option. Change-Id: I0b9d7a7d1445c2017756146068e287628a39bec6 (cherry picked from commit ed2ac5d7954a4905eeceaffd983b307681ee23b1) (cherry picked from commit 04a4482da2827ebf41fdcd2b01f009f0429975f2) --- nova/conf/vmware.py | 12 ++++++++++++ nova/virt/vmwareapi/vm_util.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index 50ec71d62ad..e97a981bf62 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -79,6 +79,18 @@ This option is ignored if serial_port_service_uri is not specified. +* serial_port_service_uri +"""), + cfg.BoolOpt('serial_port_connected_default', + default=True, + help=""" +Configure, whether the added serial port should be connected by default + +This option can be used to enable the possibility to collect logs only for +specific VMs, by reconfiguring and rebooting them in the vCenter. + +Related options: +This option is ignored if serial_port_service_uri is not specified. * serial_port_service_uri """), cfg.StrOpt('serial_log_dir', diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index a05760dcfb9..3ca55d941e0 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -331,9 +331,9 @@ def create_serial_port_spec(client_factory): backing.proxyURI = CONF.vmware.serial_port_proxy_uri connectable_spec = client_factory.create('ns0:VirtualDeviceConnectInfo') - connectable_spec.startConnected = True + connectable_spec.startConnected = CONF.vmware.serial_port_connected_default connectable_spec.allowGuestControl = True - connectable_spec.connected = True + connectable_spec.connected = CONF.vmware.serial_port_connected_default serial_port = client_factory.create('ns0:VirtualSerialPort') serial_port.connectable = connectable_spec From ebd0a5c0f6c4ceac7d6b03ad48656433e36a655c Mon Sep 17 00:00:00 2001 From: Jakob Karge Date: Wed, 24 Feb 2021 17:01:24 +0100 Subject: [PATCH 58/92] VMware: Fix empty vim.get_properties_for_..._objects If the obj_list parameter to the vim_util.get_properties_for_a_collection_of_objects() function is empty, an empty list was returned. The non-error path returns a vim.RetrieveResult object, with an iterable "objects" attribute. Introduce a dummy class "EmptyRetrieveResult" which behaves like vim.RetrieveResult for the empty case, and return an instance if the list of objects to be queried is empty. (cherry picked from commit f509fae19f7a72a2231de13dd047f5f2226d7e82) (cherry picked from commit eabd23ced769962d2442dcb6d1ef53522898a037) --- nova/tests/unit/virt/vmwareapi/test_vim_util.py | 6 ++++++ nova/virt/vmwareapi/vim_util.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vim_util.py b/nova/tests/unit/virt/vmwareapi/test_vim_util.py index b3057a99ac2..42b56fe2169 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vim_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vim_util.py @@ -36,3 +36,9 @@ def test_get_inner_objects(self): self.vim, cluster_ref, 'datastore', 'Datastore', property) datastores = [oc.obj for oc in result.objects] self.assertEqual(expected_ds, datastores) + + def test_get_properties_for_an_empty_collection_of_objects(self): + result = vim_util.get_properties_for_a_collection_of_objects( + self.vim, "Foo", [], ["bar", "baz"]) + self.assertTrue(hasattr(result, "objects")) + self.assertTrue(hasattr(result.objects, "__iter__")) diff --git a/nova/virt/vmwareapi/vim_util.py b/nova/virt/vmwareapi/vim_util.py index 10df0c9524d..72131800c45 100644 --- a/nova/virt/vmwareapi/vim_util.py +++ b/nova/virt/vmwareapi/vim_util.py @@ -25,6 +25,11 @@ CONF = nova.conf.CONF +class EmptyRetrieveResult(object): + def __init__(self): + self.objects = [] + + def object_to_dict(obj, list_depth=1): """Convert Suds object into serializable format. @@ -136,7 +141,7 @@ def get_properties_for_a_collection_of_objects(vim, type, """ client_factory = vim.client.factory if len(obj_list) == 0: - return [] + return EmptyRetrieveResult() prop_spec = get_prop_spec(client_factory, type, properties) lst_obj_specs = [] for obj in obj_list: From 3506d49319589f4a968b4a2cdb93502685d87abe Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 21 Apr 2021 09:56:59 +0200 Subject: [PATCH 59/92] vmware: Add constants for DRS-created vCLS VMs Using these values, one can identify a vCLS VM by looking at the "managedBy" attribute of the VM's config in vSphere. These VMs are special as they're created by VMware's DRS service as an agent VM and any actions done by the user will be reverted by the cluster automatically. See [1] for more details on the why. [1] https://blogs.vmware.com/vsphere/2020/09/vsphere-7-update-1-vsphere-clustering-service-vcls.html Change-Id: I31d1ece3fa514ca42a3ccc1b348da3763b1b1388 (cherry picked from commit 7e0649cfeaf8cd911fc7c750c54393678f159216) (cherry picked from commit 485226e0ff457043ef26b76caccdb12b90c88fc6) --- nova/virt/vmwareapi/constants.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/virt/vmwareapi/constants.py b/nova/virt/vmwareapi/constants.py index 137f82b37d5..5bd5ba96dbd 100644 --- a/nova/virt/vmwareapi/constants.py +++ b/nova/virt/vmwareapi/constants.py @@ -71,6 +71,10 @@ EXTENSION_KEY = 'org.openstack.compute' EXTENSION_TYPE_INSTANCE = 'instance' +# extension and type used by vCLS VMs created by DRS beginning from vSphere 7 +VCLS_EXTENSION_KEY = 'com.vmware.vim.eam' +VCLS_EXTENSION_TYPE_AGENT = 'cluster-agent' + # The max number of devices that can be connected to one adapter # One adapter has 16 slots but one reserved for controller SCSI_MAX_CONNECT_NUMBER = 15 From ebb0cf3c4480c7366504a5ceddc4c658c3ec0d00 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 16 Apr 2018 13:00:09 +0200 Subject: [PATCH 60/92] Do not block on reservation while attaching The attachment operation may potentially be long-running, so a batch of attachments/reservations may be causing a rpc timeout. The compute node still blocks on the lock and creates a bdm regardless. This replaces the instance lock with a bdm specific lock on bdm allocation. That means reserving a device name can run concurrently with detaching a volume. This will likely create a different behaviour, but since the value has a non-zero probability of being wrong anyway, we risk it being wrong more often. Change-Id: I99921bb0f22b02c51377ae276429319639e534df (cherry picked from commit d64c09406ecfd2d0beebffe660742e1cc61a082b) (cherry picked from commit e4e8e6276ab0324704f31d6a3cc4e47f061d25ca) --- nova/compute/manager.py | 24 +++++++++++++++++++++--- nova/virt/driver.py | 1 + nova/virt/libvirt/driver.py | 1 + 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 56862590531..ae724e79ca7 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -2151,8 +2151,11 @@ def _get_device_name_for_instance(self, instance, bdms, block_device_obj): return self.driver.get_device_name_for_instance( instance, bdms, block_device_obj) except NotImplementedError: - return compute_utils.get_device_name_for_instance( - instance, bdms, block_device_obj.get("device_name")) + @utils.synchronized(instance.uuid + "-bdms") + def _do_get_device_name_for_instance(): + return compute_utils.get_device_name_for_instance( + instance, bdms, block_device_obj.get("device_name")) + return _do_get_device_name_for_instance() def _default_block_device_names(self, instance, image_meta, block_devices): """Verify that all the devices have the device_name set. If not, @@ -8052,7 +8055,22 @@ def reserve_block_device_name(self, context, instance, device, raise exception.MultiattachNotSupportedByVirtDriver( volume_id=volume_id) - @utils.synchronized(instance.uuid) + # There is only one driver, which calls out to the hypervisor for that, + # so we need to lock the instance against manipulations + # Otherwise, we can do our own logic and have to synchronise that + # Since we do NOT hold a lock on instance level, we need to introduce + # one on the instance bdm level in _get_device_name_for_instance, so + # that we are protected against concurrency issues with other calls to + # that function + driver_specific_device_name = self.driver.capabilities.get( + "driver_specific_device_name", False) + if driver_specific_device_name: + synchronized = utils.synchronized(instance.uuid) + else: + def synchronized(f): + return f + + @synchronized def do_reserve(): bdms = ( objects.BlockDeviceMappingList.get_by_instance_uuid( diff --git a/nova/virt/driver.py b/nova/virt/driver.py index 2be76fa3806..941c9a37ec3 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -315,6 +315,7 @@ class ComputeDriver(object): "supports_image_type_vhdx": False, "supports_image_type_vmdk": False, "supports_image_type_ploop": False, + "driver_specific_device_name": False } # Indicates if this driver will rebalance nodes among compute service diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 692f368ac67..7faff7b018e 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -471,6 +471,7 @@ def __init__(self, virtapi, read_only=False): self.image_backend.backend().SUPPORTS_LUKS, "supports_ephemeral_encryption_luks": self.image_backend.backend().SUPPORTS_LUKS, + "driver_specific_device_name": True, } super(LibvirtDriver, self).__init__(virtapi) From d8a79aee8304edded5213fb0424fde1cba684a9b Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Thu, 7 Jan 2021 15:49:26 +0100 Subject: [PATCH 61/92] Don't call detach without attachment_id and other attachments When messages time out between nova-api and nova-compute, it can happen, that there's a block-device-mapping that never saw an attachment and wasn't rolled back either. If such a VM is deleted and the volume got attached to another VM in the mean time, a detach call to Cinder without an attachment_id - which cannot exist, because the attachment never got that far - would delete the attachment for the other VM. We now search for a volume-attachment in Cinder if no attachment_id was given. If we don't find one for our instance, but there are attachments, on the volume - which should then belong to other instances - we assume we ran into above's case and don't call detach. Change-Id: I6c6fad88e93fd788e3df1c942fed763c0ad0414f (cherry picked from commit 8a3b9d7935a2c1e2a900fa06a857d022ac5c7e38) (cherry picked from commit 66820cc7118545c9f2f02ce02317f671d03433c4) --- nova/tests/unit/volume/test_cinder.py | 52 ++++++++++++++++++++++++++- nova/volume/cinder.py | 47 +++++++++++++----------- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/nova/tests/unit/volume/test_cinder.py b/nova/tests/unit/volume/test_cinder.py index 5c5914487c5..9b6f322d5d8 100644 --- a/nova/tests/unit/volume/test_cinder.py +++ b/nova/tests/unit/volume/test_cinder.py @@ -670,6 +670,10 @@ def test_detach(self, mock_cinderclient): @mock.patch('nova.volume.cinder.cinderclient') def test_detach_no_attachment_id(self, mock_cinderclient): + """This behaves like the multiattach case. It searches the + attachment_id in the cinder attachments, because we ran into problems + with "ghost attachments" because of MessagingTimeout on attach. + """ attachment = {'server_id': 'fake_uuid', 'attachment_id': 'fakeid' } @@ -683,7 +687,7 @@ def test_detach_no_attachment_id(self, mock_cinderclient): self.api.detach(self.ctx, 'id1', instance_uuid='fake_uuid') mock_cinderclient.assert_called_with(self.ctx, microversion=None) - mock_volumes.detach.assert_called_once_with('id1', None) + mock_volumes.detach.assert_called_once_with('id1', 'fakeid') @mock.patch('nova.volume.cinder.cinderclient') def test_detach_no_attachment_id_multiattach(self, mock_cinderclient): @@ -702,6 +706,52 @@ def test_detach_no_attachment_id_multiattach(self, mock_cinderclient): mock_cinderclient.assert_called_with(self.ctx, microversion=None) mock_volumes.detach.assert_called_once_with('id1', 'fakeid') + @mock.patch('nova.volume.cinder.cinderclient') + def test_detach_no_attachment_id_ghost_attachment_no_others( + self, mock_cinderclient): + """If we have no attachment_id and we can't find an attachment for the + instance_uuid in Cinder and we have not attachments reported by Cinder, + we don't do a detach call without attachment_id. + We call without attachment_id, because we need to reset the state of + the volume in Cinder. Since there's no other attachment reported, we + can't delete an attachment for another instance in Cinder. + """ + mock_volumes = mock.MagicMock() + mock_cinderclient.return_value = mock.MagicMock(version='2', + volumes=mock_volumes) + mock_cinderclient.return_value.volumes.get.return_value = \ + FakeVolume('id1', attachments=[]) + + self.api.detach(self.ctx, 'id1', instance_uuid='fake_uuid') + + mock_cinderclient.assert_called_with(self.ctx, microversion=None) + mock_volumes.detach.assert_called_once_with('id1', None) + + @mock.patch('nova.volume.cinder.cinderclient') + def test_detach_no_attachment_id_ghost_attachment_with_others( + self, mock_cinderclient): + """If we have no attachment_id and we can't find an attachment for the + instance_uuid in Cinder and we have attachments reported by Cinder, we + don't do a detach call. The reason is, that the attachment probably + didn't work because of MessagingTimeout on reserving the block device + name. Then we only have a block device mapping without connection_info, + which also has updated_at set to NULL. + """ + attachment = {'server_id': 'fake_uuid2', + 'attachment_id': 'fakeid' + } + + mock_volumes = mock.MagicMock() + mock_cinderclient.return_value = mock.MagicMock(version='2', + volumes=mock_volumes) + mock_cinderclient.return_value.volumes.get.return_value = \ + FakeVolume('id1', attachments=[attachment]) + + self.api.detach(self.ctx, 'id1', instance_uuid='fake_uuid') + + mock_cinderclient.assert_called_with(self.ctx, microversion=None) + mock_volumes.detach.assert_not_called() + @mock.patch('nova.volume.cinder.cinderclient') def test_detach_internal_server_error(self, mock_cinderclient): mock_cinderclient.return_value.volumes.detach.side_effect = ( diff --git a/nova/volume/cinder.py b/nova/volume/cinder.py index b43d4b551b5..d6f4b2c1164 100644 --- a/nova/volume/cinder.py +++ b/nova/volume/cinder.py @@ -583,28 +583,33 @@ def detach(self, context, volume_id, instance_uuid=None, client = cinderclient(context) if attachment_id is None: volume = self.get(context, volume_id) + attachments = volume.get('attachments', {}) + if instance_uuid: + attachment_id = attachments.get(instance_uuid, {}).\ + get('attachment_id') + else: + LOG.warning("attachment_id couldn't be retrieved for " + "volume %(volume_id)s.", + {'volume_id': volume_id}) if volume['multiattach']: - attachments = volume.get('attachments', {}) - if instance_uuid: - attachment_id = attachments.get(instance_uuid, {}).\ - get('attachment_id') - if not attachment_id: - LOG.warning("attachment_id couldn't be retrieved " - "for volume %(volume_id)s with " - "instance_uuid %(instance_id)s. The " - "volume has the 'multiattach' flag " - "enabled, without the attachment_id " - "Cinder most probably cannot perform " - "the detach.", - {'volume_id': volume_id, - 'instance_id': instance_uuid}) - else: - LOG.warning("attachment_id couldn't be retrieved for " - "volume %(volume_id)s. The volume has the " - "'multiattach' flag enabled, without the " - "attachment_id Cinder most probably " - "cannot perform the detach.", - {'volume_id': volume_id}) + if not attachment_id: + LOG.warning("attachment_id couldn't be retrieved " + "for volume %(volume_id)s with " + "instance_uuid %(instance_id)s. The " + "volume has the 'multiattach' flag " + "enabled, without the attachment_id " + "Cinder most probably cannot perform " + "the detach.", + {'volume_id': volume_id, + 'instance_id': instance_uuid}) + elif not attachment_id and attachments: + # NOTE(jkulik): If we detach unconditionally although we have + # other attachments, we will delete the other attachment's + # attachment entry in Cinder. + LOG.info("Not calling detach, because there are attachments " + "that don't belong to instance %(instance_uuid)s.", + {'instance_uuid': instance_uuid}) + return client.volumes.detach(volume_id, attachment_id) From 11783de678244c0f855916580edaf56b8fa4bf8b Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 7 Feb 2022 10:16:10 +0100 Subject: [PATCH 62/92] Vmwareapi: Use oslo_vmware get_object_property instead of nova one get_network_with_the_name is the last place where the nova function of the same name is used instead of the one implemented in oslo_vmware. Change-Id: I4293f3b2a7551793dd53dd427383583469ed0868 (cherry picked from commit dcaf9c6418bb17568ae357071ba9c7c299ce41d6) (cherry picked from commit 043b6461cd127906373821c6eb493bab0d926d5b) --- .../unit/virt/vmwareapi/test_network_util.py | 89 +++++++------------ nova/virt/vmwareapi/network_util.py | 83 ++++++++--------- nova/virt/vmwareapi/vim_util.py | 24 ----- 3 files changed, 69 insertions(+), 127 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_network_util.py b/nova/tests/unit/virt/vmwareapi/test_network_util.py index b3b5bb15eae..8e6d76a6ccf 100644 --- a/nova/tests/unit/virt/vmwareapi/test_network_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_network_util.py @@ -47,45 +47,21 @@ def _build_cluster_networks(self, networks): """Returns a set of results for a cluster network lookup. This is an example: - (ObjectContent){ - obj = - (obj){ - value = "domain-c7" - _type = "ClusterComputeResource" - } - propSet[] = - (DynamicProperty){ - name = "network" - val = - (ArrayOfManagedObjectReference){ - ManagedObjectReference[] = - (ManagedObjectReference){ - value = "network-54" - _type = "Network" - }, - (ManagedObjectReference){ - value = "dvportgroup-14" - _type = "DistributedVirtualPortgroup" - }, - } - }, + (ArrayOfManagedObjectReference){ + ManagedObjectReference[] = [ + (ManagedObjectReference){ + value = "network-54" + _type = "Network" + }, + (ManagedObjectReference){ + value = "dvportgroup-14" + _type = "DistributedVirtualPortgroup" + }, }] """ - - objects = [] - obj = ObjectContent(obj=vim_util.get_moref("domain-c7", - "ClusterComputeResource"), - propSet=[]) - value = fake.DataObject() - value.ManagedObjectReference = [] - for network in networks: - value.ManagedObjectReference.append(network) - - obj.propSet.append( - DynamicProperty(name='network', - val=value)) - objects.append(obj) - return ResultSet(objects=objects) + array = fake._create_array_of_type("ManagedObjectReference") + array.ManagedObjectReference = networks + return array def test_get_network_no_match(self): net_morefs = [vim_util.get_moref("dvportgroup-135", @@ -94,13 +70,14 @@ def test_get_network_no_match(self): "DistributedVirtualPortgroup")] networks = self._build_cluster_networks(net_morefs) - def mock_call_method(module, method, *args, **kwargs): - if method == 'get_object_properties': + def mock_call_method(module, method, mo_ref, property): + if method != 'get_object_property': + raise RuntimeError('Unexpected method') + if mo_ref == 'fake_cluster': return networks - if method == 'get_object_property': - result = fake.DataObject() - result.name = 'no-match' - return result + result = fake.DataObject() + result.name = 'no-match' + return result with mock.patch.object(self._session, '_call_method', mock_call_method): @@ -114,15 +91,16 @@ def _get_network_dvs_match(self, name): "DistributedVirtualPortgroup")] networks = self._build_cluster_networks(net_morefs) - def mock_call_method(module, method, *args, **kwargs): - if method == 'get_object_properties': + def mock_call_method(module, method, mo_ref, property): + if method != 'get_object_property': + raise RuntimeError('Unexpected method') + if mo_ref == 'fake_cluster': return networks - if method == 'get_object_property': - result = fake.DataObject() - result.name = name - result.key = 'fake_key' - result.distributedVirtualSwitch = 'fake_dvs' - return result + result = fake.DataObject() + result.name = name + result.key = 'fake_key' + result.distributedVirtualSwitch = 'fake_dvs' + return result with mock.patch.object(self._session, '_call_method', mock_call_method): @@ -141,11 +119,12 @@ def test_get_network_network_match(self): net_morefs = [vim_util.get_moref("network-54", "Network")] networks = self._build_cluster_networks(net_morefs) - def mock_call_method(module, method, *args, **kwargs): - if method == 'get_object_properties': + def mock_call_method(module, method, mo_ref, property): + if method != 'get_object_property': + raise RuntimeError('Unexpected method') + if mo_ref == 'fake_cluster': return networks - if method == 'get_object_property': - return 'fake_net' + return 'fake_net' with mock.patch.object(self._session, '_call_method', mock_call_method): diff --git a/nova/virt/vmwareapi/network_util.py b/nova/virt/vmwareapi/network_util.py index a6a84bfdb3e..581146c219c 100644 --- a/nova/virt/vmwareapi/network_util.py +++ b/nova/virt/vmwareapi/network_util.py @@ -23,7 +23,6 @@ from oslo_vmware import vim_util as vutil from nova.virt.vmwareapi import vim_util -from nova.virt.vmwareapi import vm_util LOG = logging.getLogger(__name__) @@ -59,57 +58,45 @@ def _get_network_obj(session, network_objects, network_name): """ network_obj = {} - # network_objects is actually a RetrieveResult object from vSphere API call - for obj_content in network_objects: - # the propset attribute "need not be set" by returning API - if not hasattr(obj_content, 'propSet'): - continue - prop_dict = vm_util.propset_dict(obj_content.propSet) - network_refs = prop_dict.get('network') - if network_refs: - network_refs = network_refs.ManagedObjectReference - for network in network_refs: - # Get network properties - if network._type == 'DistributedVirtualPortgroup': - props = session._call_method(vutil, - "get_object_property", - network, - "config") - # NOTE(asomya): This only works on ESXi if the port binding - # is set to ephemeral - net_name = _get_name_from_dvs_name(props.name) - if network_name in net_name: - network_obj['type'] = 'DistributedVirtualPortgroup' - network_obj['dvpg'] = props.key - dvs_props = session._call_method(vutil, - "get_object_property", - props.distributedVirtualSwitch, - "uuid") - network_obj['dvsw'] = dvs_props - return network_obj - else: - props = session._call_method(vutil, - "get_object_property", - network, - "summary.name") - if props == network_name: - network_obj['type'] = 'Network' - network_obj['name'] = network_name - return network_obj + for network in vim_util.get_array_items(network_objects): + # Get network properties + if network._type == 'DistributedVirtualPortgroup': + props = session._call_method(vutil, + "get_object_property", + network, + "config") + # NOTE(asomya): This only works on ESXi if the port binding + # is set to ephemeral + net_name = _get_name_from_dvs_name(props.name) + if network_name in net_name: + network_obj['type'] = 'DistributedVirtualPortgroup' + network_obj['dvpg'] = props.key + dvs_props = session._call_method(vutil, + "get_object_property", props.distributedVirtualSwitch, + "uuid") + network_obj['dvsw'] = dvs_props + return network_obj + else: + props = session._call_method(vutil, + "get_object_property", + network, + "summary.name") + if props == network_name: + network_obj['type'] = 'Network' + network_obj['name'] = network_name + return network_obj def get_network_with_the_name(session, network_name="vmnet0", cluster=None): """Gets reference to the network whose name is passed as the argument. """ - vm_networks = session._call_method(vim_util, - 'get_object_properties', - None, cluster, - 'ClusterComputeResource', ['network']) - - with vutil.WithRetrieval(session.vim, vm_networks) as network_objs: - network_obj = _get_network_obj(session, network_objs, network_name) - if network_obj: - return network_obj + network_objs = session._call_method(vutil, + "get_object_property", + cluster, + "network") - LOG.debug("Network %s not found on cluster!", network_name) + network_obj = _get_network_obj(session, network_objs, network_name) + if not network_obj: + LOG.debug("Network %s not found on cluster!", network_name) + return network_obj diff --git a/nova/virt/vmwareapi/vim_util.py b/nova/virt/vmwareapi/vim_util.py index 72131800c45..fe4962c2181 100644 --- a/nova/virt/vmwareapi/vim_util.py +++ b/nova/virt/vmwareapi/vim_util.py @@ -56,30 +56,6 @@ def object_to_dict(obj, list_depth=1): return d -def get_object_properties(vim, collector, mobj, type, properties): - """Gets the properties of the Managed object specified.""" - client_factory = vim.client.factory - if mobj is None: - return None - usecoll = collector - if usecoll is None: - usecoll = vim.service_content.propertyCollector - property_filter_spec = client_factory.create('ns0:PropertyFilterSpec') - property_spec = client_factory.create('ns0:PropertySpec') - property_spec.all = (properties is None or len(properties) == 0) - property_spec.pathSet = properties - property_spec.type = type - object_spec = client_factory.create('ns0:ObjectSpec') - object_spec.obj = mobj - object_spec.skip = False - property_filter_spec.propSet = [property_spec] - property_filter_spec.objectSet = [object_spec] - options = client_factory.create('ns0:RetrieveOptions') - options.maxObjects = CONF.vmware.maximum_objects - return vim.RetrievePropertiesEx(usecoll, specSet=[property_filter_spec], - options=options) - - def get_objects(vim, type, properties_to_collect=None, all=False): """Gets the list of objects of the type specified.""" return vutil.get_objects(vim, type, CONF.vmware.maximum_objects, From 5e0417b56330fc71d5d013c553870fe302eeaeb4 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 7 Aug 2023 12:59:14 +0200 Subject: [PATCH 63/92] scheduler: Add method for retrieving all traits The `SchedulerReportClient` do not have a method to list all traits - just one to list the traits for a resource-provider. Since we plan to sync all CUSTOM_* traits into the VMware vCenter, we need the full list available and thus add a method to retrieve them. Change-Id: Ie62792a1247b89f0c83d11f5e70f128b5347614c (cherry picked from commit f020d1cfbd0dbe97856f1b017e6f6b2bbe0ca219) --- nova/scheduler/client/report.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/nova/scheduler/client/report.py b/nova/scheduler/client/report.py index dab692d36f5..74689f10b57 100644 --- a/nova/scheduler/client/report.py +++ b/nova/scheduler/client/report.py @@ -2637,3 +2637,27 @@ def _get_core_usages(usages): self._handle_usages_error_from_placement(resp, project_id, user_id=user_id) return total_counts + + def get_traits(self, context): + """Queries the placement API for all traits. + + :param context: The security context + :return: A set() of trait names + :raise: NovaException on errors. + :raise: keystoneauth1.exceptions.ClientException if placement API + communication fails. + """ + resp = self.get("/traits", version='1.6', + global_request_id=context.global_id) + + if resp.status_code == 200: + json = resp.json() + return set(json['traits']) + + placement_req_id = get_placement_request_id(resp) + LOG.error( + "[%(placement_req_id)s] Failed to retrieve traits from " + "placement API. Got %(status_code)d: %(err_text)s.", + {'placement_req_id': placement_req_id, + 'status_code': resp.status_code, 'err_text': resp.text}) + raise exception.NovaException('Failed to get traits from Placement') From a1621b165a7700b3ca70561798ad70f13b928b25 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Tue, 22 Jun 2021 11:49:28 +0200 Subject: [PATCH 64/92] Idempotent binding creation In case we already created a binding for a live-migration but crashed during the ongoing process, neutron will already hold a port binding for the host. Instead of failing, we can simply take the existing port-binding and continue Change-Id: If84c74e258084d4ab648a6a413896eda087317d7 See: https://specs.openstack.org/openstack/neutron-specs/specs/backlog/pike/portbinding_information_for_nova.html (cherry picked from commit 6caa31f0cc7db6ef28e676d7c5e0facfdf6047fe) --- nova/network/neutron.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/nova/network/neutron.py b/nova/network/neutron.py index 00edf5c3c04..c303a8f532b 100644 --- a/nova/network/neutron.py +++ b/nova/network/neutron.py @@ -61,6 +61,17 @@ _ADMIN_AUTH = None +# NOTE(jkulik): We need to monkeypatch this exception into +# python-neutronclient, because we don't get the error_type from the exception +# back from python-neutronclient other than via a custom class named like it. +class PortBindingAlreadyExistsClient(neutron_client_exc.Conflict): + pass + + +neutron_client_exc.PortBindingAlreadyExistsClient = \ + PortBindingAlreadyExistsClient + + def reset_state(): global _ADMIN_AUTH global _SESSION @@ -1597,12 +1608,25 @@ def bind_ports_to_host(self, context, instance, host, data = {'binding': binding} try: - binding = client.create_port_binding(port_id, data)['binding'] + step = 'creation' + try: + binding = client.create_port_binding( + port_id, data + )['binding'] + except neutron_client_exc.PortBindingAlreadyExistsClient: + # In case we actually managed to create the binding, + # but something went wrong in between, we can recover + step = 'recovery' + binding = client.show_port_binding( + port_id, host + )['binding'] except neutron_client_exc.NeutronClientException: # Something failed, so log the error and rollback any # successful bindings. - LOG.error('Binding failed for port %s and host %s.', + LOG.error('Binding %s failed for port %s and host %s.', + step, port_id, host, instance=instance, exc_info=True) + for rollback_port_id in bindings_by_port_id: try: client.delete_port_binding(rollback_port_id, host) From 6bf961267459639d197b690df8b449400468b83b Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 16 Nov 2022 13:29:10 +0100 Subject: [PATCH 65/92] vmware: Add get_datastore_ref_by_name helper This function iterates over all datastores in the vCenter trying to find a datastore with the given name. Might be a good candidate for downporting into oslo.vmware. Change-Id: I3fc7f171592c2cd21b765e0eb0218bf87d45a37c (cherry picked from commit 4f0a0ada5ed5e69876110855e32fc930e95ec1b5) --- nova/virt/vmwareapi/vm_util.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 3ca55d941e0..45d4e702f3e 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -1351,6 +1351,27 @@ def get_cluster_ref_by_name(session, cluster_name): return cluster.obj +def get_datastore_ref_by_name(session, datastore_name): + """Return the ManagedObjectReference to the datastore with the given name + + Returns None if no datastore with that name can be found. + """ + try: + results = session._call_method(vim_util, "get_objects", + "Datastore", ["name"]) + with vutil.WithRetrieval(session.vim, results) as objects: + for datastore in objects: + if not getattr(datastore, 'propSet', None): + continue + + if datastore.propSet[0].val != datastore_name: + continue + + return datastore.obj + except Exception as exc: + LOG.warning("Failed to get datastore references %s", exc) + + def get_vmdk_adapter_type(adapter_type): """Return the adapter type to be used in vmdk descriptor. From 3feab43acb094e9cb522257dbb3b9da77af1a00c Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Fri, 25 Nov 2022 09:59:33 +0100 Subject: [PATCH 66/92] Add more password generation options Some password policies require more than one occurence of symbols of one kind, or make restrictions about their occurence effectively requiring them to occure more often. By providing the configuration value 'password_all_group_samples' the administrator can increase the rounds to sample from all groups to adhere to such policies Often, password policies require not only ascii letters (upper/lower-case) and numbers, but also other printable characters in the password as a fourth symbol group. By makeing the symbol-classes the multi-string list 'password_symbol_groups', the administrator can add those and also override the other classes, if desired. Change-Id: I5b995883a41f65296de86f3effa0102ecb12c1fa (cherry picked from commit c80d49ebbee94e6ae2322582d73881908b569b4b) --- nova/conf/base.py | 39 +++++++++++++++++++++++++++++++++ nova/tests/unit/test_utils.py | 41 ++++++++++++++++++++++++++++++----- nova/utils.py | 18 +++++++-------- 3 files changed, 84 insertions(+), 14 deletions(-) diff --git a/nova/conf/base.py b/nova/conf/base.py index dd2730f8fa6..e1750e4dcf6 100644 --- a/nova/conf/base.py +++ b/nova/conf/base.py @@ -17,12 +17,51 @@ from oslo_config import cfg +_DEFAULT_PASSWORD_SYMBOLS = ['23456789', # Removed: 0,1 + 'ABCDEFGHJKLMNPQRSTUVWXYZ', # Removed: I, O + 'abcdefghijkmnopqrstuvwxyz', # Removed: l + ] + base_options = [ cfg.IntOpt( 'password_length', default=12, min=0, help='Length of generated instance admin passwords.'), + cfg.IntOpt( + 'password_all_group_samples', + default=1, + min=0, + help=''' +How often should the symbols be sampled from all groups +to ensure the presence of all of them +* Zero: Purely random, so least predictable, but possibly not confirming to + some password policies +* Any positive number: At least that many symbols will be from each of the + classes. By default: lower-case, upper-case and numbers. + +Interdependencies to other options: + +* If ``password_length`` is smaller than ``password_all_group_samples`` times + three (or more in case more groups are added to ``password_symbol_groups``), + then the password will be cut off after ``password_length``, thereby possibly + reducing the number of symbol classes in the generated password. +'''), + cfg.MultiStrOpt( + 'password_symbol_groups', + default=_DEFAULT_PASSWORD_SYMBOLS, + help=''' +List of symbols to use for passwords. +Default avoids visually confusing characters. (~6 bits per symbol) + +The items in the list represents symbol groups, and from each of those groups +at least ``password_all_group_samples`` symbols are taken randomly. + +Interdependencies to other options: +See ``password_additional_symbols`` for the interaction of the three values +``password_length``, ``password_additional_symbols`` and +``password_symbol_groups`` +'''), cfg.StrOpt( 'instance_usage_audit_period', default='month', diff --git a/nova/tests/unit/test_utils.py b/nova/tests/unit/test_utils.py index b34136b0522..440ca0ed1d4 100644 --- a/nova/tests/unit/test_utils.py +++ b/nova/tests/unit/test_utils.py @@ -16,6 +16,7 @@ import hashlib import os import os.path +import string import tempfile from unittest import mock @@ -31,6 +32,7 @@ from oslo_utils import encodeutils from oslo_utils import fixture as utils_fixture +from nova.conf import base as conf_base from nova import context from nova import exception from nova.objects import base as obj_base @@ -140,11 +142,40 @@ def test_hostname_truncated_no_hyphen(self): def test_generate_password(self): password = utils.generate_password() - self.assertTrue([c for c in password if c in '0123456789']) - self.assertTrue([c for c in password - if c in 'abcdefghijklmnopqrstuvwxyz']) - self.assertTrue([c for c in password - if c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ']) + for group in conf_base._DEFAULT_PASSWORD_SYMBOLS: + if group: + self.assertTrue([c for c in password if c in group]) + + @mock.patch.object(CONF, "password_all_group_samples", 0) + def test_generate_password_zero_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + + @mock.patch.object(CONF, "password_all_group_samples", 2) + def test_generate_password_two_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + for group in conf_base._DEFAULT_PASSWORD_SYMBOLS: + if group: + self.assertGreaterEqual( + len([c for c in password if c in group]), + CONF.password_all_group_samples) + + @mock.patch.object(CONF, "password_all_group_samples", 7) + @mock.patch.object(CONF, "password_length", 13) + def test_generate_password_too_many_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + + @mock.patch.object(CONF, "password_symbol_groups", [string.ascii_lowercase, + string.ascii_uppercase, string.punctuation]) + def test_generate_password_custom_groups(self): + password = utils.generate_password() + self.assertEqual(len(password), CONF.password_length) + self.assertGreaterEqual( + len([c for c in password if c in string.punctuation]), + CONF.password_all_group_samples) + self.assertFalse([c for c in password if c in string.digits]) @mock.patch('nova.privsep.path.chown') def test_temporary_chown(self, mock_chown): diff --git a/nova/utils.py b/nova/utils.py index 877f8efe02e..e95612420b4 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -135,13 +135,6 @@ def generate_random_string(size=8): return ''.join([random.choice(characters) for _x in range(size)]) -# Default symbols to use for passwords. Avoids visually confusing characters. -# ~6 bits per symbol -DEFAULT_PASSWORD_SYMBOLS = ('23456789', # Removed: 0,1 - 'ABCDEFGHJKLMNPQRSTUVWXYZ', # Removed: I, O - 'abcdefghijkmnopqrstuvwxyz') # Removed: l - - def last_completed_audit_period(unit=None, before=None): """This method gives you the most recently *completed* audit period. @@ -232,7 +225,7 @@ def last_completed_audit_period(unit=None, before=None): return (begin, end) -def generate_password(length=None, symbolgroups=DEFAULT_PASSWORD_SYMBOLS): +def generate_password(length=None, symbolgroups=None): """Generate a random password from the supplied symbol groups. At least one symbol from each group will be included. Unpredictable @@ -244,12 +237,19 @@ def generate_password(length=None, symbolgroups=DEFAULT_PASSWORD_SYMBOLS): if length is None: length = CONF.password_length + if symbolgroups is None: + symbolgroups = CONF.password_symbol_groups + r = random.SystemRandom() # NOTE(jerdfelt): Some password policies require at least one character # from each group of symbols, so start off with one random character # from each symbol group - password = [r.choice(s) for s in symbolgroups] + # NOTE(fwiesel): And some policies require even more of them, so + # do it as often as configured in DEFAULT.password_all_group_samples + password = [r.choice(s) + for s in symbolgroups * CONF.password_all_group_samples + if s] # If length < len(symbolgroups), the leading characters will only # be from the first length groups. Try our best to not be predictable # by shuffling and then truncating. From 209683fe6cf53f53becd1f1699d583de3dcd9efd Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 4 Oct 2021 12:40:38 +0200 Subject: [PATCH 67/92] Vmware: Handle deleted image graceful When migrating or resizing an instance, the driver looks for vsphere locations properties stored with the image. We can't do that though, if the image has been deleted, so we fall back gracefully. Change-Id: I55c4d1f49e3c6fc0bb89794ac1b44da99ce009ca (cherry picked from commit 39b4f572b30c349130af0dc087281ebd9d1ae282) (cherry picked from commit 363553473ab35df0b65323ad9b5cc461e9073d8b) --- nova/virt/vmwareapi/images.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/nova/virt/vmwareapi/images.py b/nova/virt/vmwareapi/images.py index e983fe5fada..f8a129d5350 100644 --- a/nova/virt/vmwareapi/images.py +++ b/nova/virt/vmwareapi/images.py @@ -183,14 +183,20 @@ def from_image(cls, context, image_id, image_meta): def get_vsphere_location(context, image_id): """Get image location in vsphere or None.""" # image_id can be None if the instance is booted using a volume. - if image_id: + if not image_id: + return None + + try: metadata = IMAGE_API.get(context, image_id, include_locations=True) - locations = metadata.get('locations') - if locations: - for loc in locations: - loc_url = loc.get('url') - if loc_url and loc_url.startswith('vsphere://'): - return loc_url + except exception.ImageNotFound: + return None + + locations = metadata.get('locations') + if locations: + for loc in locations: + loc_url = loc.get('url') + if loc_url and loc_url.startswith('vsphere://'): + return loc_url return None From 32c2f62e2524e8a00d106de9d9e852c7ec087f2c Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 15 Oct 2021 08:56:28 +0200 Subject: [PATCH 68/92] vmware: Use moref value for logging cluster_ref When we log the cluster moref in getting the list of instances, we log the whole moref and thus create multiple newlines in the log-line. To have all data in a single line, we now only log the moref value. Change-Id: I89d3908abdffe60570cec947e9a74b544488b535 (cherry picked from commit 920d080028ebbefe63d0a12a2793d70badd75443) (cherry picked from commit 0c48be512bd9b69a2a8305678a85a14a7112ff1d) --- nova/tests/unit/virt/vmwareapi/test_vmops.py | 7 ++++--- nova/virt/vmwareapi/vmops.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 16fcb88d7cf..4648e74079d 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -224,7 +224,7 @@ def test_create_folder_if_missing_exception(self, mock_mkdir): def test_get_valid_vms_from_retrieve_result(self): ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock(), - mock.Mock()) + mock.Mock(), cluster=self._cluster.obj) fake_objects = vmwareapi_fake.FakeRetrieveResult() for x in range(0, 3): vm = vmwareapi_fake.VirtualMachine() @@ -237,7 +237,7 @@ def test_get_valid_vms_from_retrieve_result(self): def test_get_valid_vms_from_retrieve_result_with_invalid(self): ops = vmops.VMwareVMOps(self._session, mock.Mock(), mock.Mock(), - mock.Mock()) + mock.Mock(), cluster=self._cluster.obj) fake_objects = vmwareapi_fake.FakeRetrieveResult() valid_vm = vmwareapi_fake.VirtualMachine() valid_vm.set('config.extraConfig["nvp.vm-uuid"]', @@ -367,7 +367,8 @@ def mock_call_method(module, method, *args, **kwargs): def _test_get_datacenter_ref_and_name(self, ds_ref_exists=False): instance_ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds-1') - _vcvmops = vmops.VMwareVMOps(self._session, None, None, None) + _vcvmops = vmops.VMwareVMOps(self._session, None, None, None, + cluster=self._cluster.obj) result = vmwareapi_fake.FakeRetrieveResult() if ds_ref_exists: ds_ref = vmwareapi_fake.ManagedObjectReference(value='ds-1') diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 3aa51a9d266..f97b9a69a37 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -2389,7 +2389,7 @@ def list_instances(self): properties = ['runtime.connectionState', 'config.extraConfig["nvp.vm-uuid"]'] LOG.debug("Getting list of instances from cluster %s", - self._cluster) + vutil.get_moref_value(self._cluster)) vms = [] if self._root_resource_pool: vms = self._session._call_method( From 6441b17eec0c56c670df8d2751080f19f3f8ad40 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Tue, 7 Jun 2022 13:34:46 +0200 Subject: [PATCH 69/92] vmware: Attach root disk first Since we don't explicitly set a disk as boot disk and instead rely on the order the disks have on the VirtualMachine, we need to make sure we attach the root disk first. Change-Id: I3ae6b5f053a3b171ed0a80215fc4204a2bf32481 (cherry picked from commit 7e6dc54a506120babe5a77780d4b9ef1c1e709e6) (cherry picked from commit 2aab52758e15d61a7fb3d904d31d3a22b3cdbe4b) --- nova/virt/vmwareapi/vmops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index f97b9a69a37..6e22dd8515f 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -869,7 +869,8 @@ def spawn(self, context, instance, image_meta, injected_files, # block_device_mapping (i.e. disk_bus) is valid self._is_bdm_valid(block_device_mapping) - for disk in block_device_mapping: + for disk in sorted(block_device_mapping, + key=lambda x: x.get('boot_index') != 0): connection_info = disk['connection_info'] adapter_type = disk.get('disk_bus') or vi.ii.adapter_type From 5030247fcc50875ab2e700f84858e528b1fa8a96 Mon Sep 17 00:00:00 2001 From: mmidolesov Date: Wed, 4 Apr 2018 00:42:48 -0700 Subject: [PATCH 70/92] vmwareapi: disk backing uuid is optional (cherry picked from commit 1269485c4ff06a6baee72311700f5290c396b238) (cherry picked from commit 9e5e899a79f9930e5c6d15b1c56a00d839a65f92) Change-Id: Ie37a83a59ef11dda71e3f88c0fbbb2065e11c97a (cherry picked from commit 846bcef5a8673b51a9593366a617624a8c8c87a5) --- nova/virt/vmwareapi/vm_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 45d4e702f3e..2d46f7c1d72 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -1310,7 +1310,7 @@ def get_vmdk_backed_disk_device(hardware_devices, uuid): if (device.__class__.__name__ == "VirtualDisk" and device.backing.__class__.__name__ == "VirtualDiskFlatVer2BackingInfo" and - device.backing.uuid == uuid): + getattr(device.backing, 'uuid', None) == uuid): return device From aac08707c541c9986dc8bc04ab9ff2cb40c1d634 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 4 Oct 2021 16:17:38 +0200 Subject: [PATCH 71/92] Vmware: Handle missing propSet in list_vms If a vm has none of the requested properties, propSet will not be set. So, we need to skip over that instance Change-Id: Ia633fecb021bffe1557820e36c33ef53cf90db83 (cherry picked from commit 6e8f9c725bd0d5697de0b001a60b94461176c984) (cherry picked from commit be82e9d2f3c5c91cc2c78dd3e972c03dc29d32cc) --- nova/virt/vmwareapi/vmops.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 6e22dd8515f..60eb149d2d9 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -2155,6 +2155,9 @@ def _get_valid_vms_from_retrieve_result(self, retrieve_result): with vutil.WithRetrieval(self._session.vim, retrieve_result) as \ objects: for vm in objects: + prop_set = getattr(vm, 'propSet', None) + if not prop_set: + continue vm_uuid = None conn_state = None for prop in vm.propSet: From cb14339bf5a8ceec6ea383fefbaa8b9f616e02ce Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 13 Jul 2020 14:26:29 +0200 Subject: [PATCH 72/92] vmware: Detect automatically disabled host on restart If we disabled the host automatically, because we couldn't reach the vCenter, we set a flag on the `VCState`. This flag is lost with a compute-node restart, but the service stays disabled. We now check the service's state and compare the disabled_reason on startup to detect if we disabled the service automatically. This means, the driver will automatically enable the service again if it can reach the vCenter. For the first start of a compute-node, the compute-node might not be there. To not prohibit the creation of the compute-node, we ignore the error raised when not finding it. It's fine to ignore the error and not print it, because there's another point in the code doing basically the same and printing out a message if the compute-node doesn't exist, yet. Change-Id: I1e49d5962adfd0bb585543c7a5580d343628f687 Incorporated-Change-Id: I25f77a69a86a876be28d31be2c626a66a5df40b3 (cherry picked from commit cc26bc36957925d561db910ee580b27b5cf6966e) (cherry picked from commit fcb1363d4bc4b82cbd00a3c0549d5bc1735ff075) (cherry picked from commit c680b79fb29cd67002b351966f852ecbd824dbfc) --- nova/tests/unit/virt/vmwareapi/test_driver_api.py | 4 +++- nova/virt/vmwareapi/host.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 9c9f92d1d26..846397ce719 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -162,7 +162,8 @@ def _create_service(self, **kwargs): return objects.Service(**service_ref) @mock.patch.object(driver.VMwareVCDriver, '_register_openstack_extension') - def setUp(self, mock_register): + @mock.patch.object(objects.Service, 'get_by_compute_host') + def setUp(self, mock_svc, mock_register): super(VMwareAPIVMTestCase, self).setUp() ds_util.dc_cache_reset() vm_util.vm_refs_cache_reset() @@ -183,6 +184,7 @@ def setUp(self, mock_register): vmwareapi_fake.reset() self.glance = self.useFixture(nova_fixtures.GlanceFixture(self)) service = self._create_service(host=HOST) + mock_svc.return_value = service self.conn = driver.VMwareVCDriver(None, False) self.assertFalse(service.disabled) diff --git a/nova/virt/vmwareapi/host.py b/nova/virt/vmwareapi/host.py index 8efc59bb61e..b269bb878c8 100644 --- a/nova/virt/vmwareapi/host.py +++ b/nova/virt/vmwareapi/host.py @@ -34,6 +34,8 @@ CONF = nova.conf.CONF LOG = logging.getLogger(__name__) +SERVICE_DISABLED_REASON = 'set by vmwareapi host_state' + def _get_ds_capacity_and_freespace(session, cluster=None, datastore_regex=None): @@ -60,7 +62,14 @@ def __init__(self, session, host_name, cluster, datastore_regex): self._cluster = cluster self._datastore_regex = datastore_regex self._stats = {} - self._auto_service_disabled = False + ctx = context.get_admin_context() + try: + service = objects.Service.get_by_compute_host(ctx, CONF.host) + self._auto_service_disabled = service.disabled \ + and service.disabled_reason == SERVICE_DISABLED_REASON + except exception.ComputeHostNotFound: + # this can happened on newly-added hosts + self._auto_service_disabled = False about_info = self._session._call_method(vim_util, "get_about_info") self._hypervisor_type = about_info.name self._hypervisor_version = versionutils.convert_version_to_int( @@ -120,6 +129,6 @@ def _set_host_enabled(self, enabled): ctx = context.get_admin_context() service = objects.Service.get_by_compute_host(ctx, CONF.host) service.disabled = not enabled - service.disabled_reason = 'set by vmwareapi host_state' + service.disabled_reason = SERVICE_DISABLED_REASON service.save() self._auto_service_disabled = service.disabled From c5856e700457a9bac35dce448f793f6b1ae3ca94 Mon Sep 17 00:00:00 2001 From: Ivaylo Mitev Date: Wed, 30 Oct 2019 19:02:14 +0200 Subject: [PATCH 73/92] vmware: Add history collectors for Task and Event These helper-classes allow iterating over the Task/Event objects belonging to another object, hiding the complexity of the vSphere API splitting those Tasks/Events into pages. Change-Id: Ide281bdd613526561324a9368a67a720f580eb4f Partly-Incorporated-Change-Id: I196bb9ab48867f314d9a5f3b7566384fa72778df Incorporated-Change-Id: I86804af29f4c6aff14fa5495e5344377488ba8fe (cherry picked from commit 368f2a73502a0d97d3336e14bdc7827fa7e9ae2e) (cherry picked from commit 52813faec5535fd84295a699429cbf74286e734d) (cherry picked from commit c5ed5df7060815766b5593322adcc13efff68baf) --- nova/virt/vmwareapi/vm_util.py | 110 ++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 2d46f7c1d72..3f793c68b96 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -21,6 +21,7 @@ import collections import copy import hashlib +import operator import socket import ssl from urllib.parse import urlparse @@ -33,7 +34,6 @@ from oslo_vmware import pbm from oslo_vmware import vim_util as vutil - import nova.conf from nova import exception from nova.i18n import _ @@ -120,6 +120,114 @@ def __init__(self, cpu_limits=None, hw_version=None, self.hw_video_ram = hw_video_ram +class HistoryCollectorItems: + + def __init__(self, session, history_collector, read_page_method, + reverse_page_order=False, max_page_size=10, + page_sort_key_func=None): + self.session = session + self.history_collector = history_collector + self.read_page_method = read_page_method + self.reverse_page_order = reverse_page_order + self.max_page_size = max_page_size + self.page_sort_key_func = page_sort_key_func + + def __iter__(self): + self._latest_page_read = False + self._page_items = None + + if self.reverse_page_order: + self.session._call_method(self.session.vim, + "ResetCollector", + self.history_collector) + else: + self.session._call_method(self.session.vim, + "RewindCollector", + self.history_collector) + + return self + + def __next__(self): + if not self._page_items: + self._load_page() + + if not self._page_items: + raise StopIteration + else: + return self._page_items.pop(0) + + def _load_page(self): + if self.reverse_page_order and not self._latest_page_read: + self._load_latest_page() + + if not self._page_items: + self._page_items = self.session._call_method( + self.session.vim, self.read_page_method, + self.history_collector, maxCount=self.max_page_size) + + if self._page_items and self.page_sort_key_func: + self._page_items.sort(reverse=self.reverse_page_order, + key=self.page_sort_key_func) + + def _load_latest_page(self): + self._latest_page_read = True + latest_page = vutil.get_object_property(self.session.vim, + self.history_collector, + "latestPage") + + self._page_items = vim_util.get_array_items(latest_page) + + def destroy_collector(self): + if self.history_collector: + self.session._call_method(self.session.vim, + "DestroyCollector", + self.history_collector) + + +class TaskHistoryCollectorItems(HistoryCollectorItems): + def __init__(self, session, task_filter_spec, reverse_page_order=False, + max_page_size=10): + task_collector = session._call_method( + session.vim, + "CreateCollectorForTasks", + session.vim.service_content.taskManager, + filter=task_filter_spec) + read_page_method = ("ReadPreviousTasks" if reverse_page_order + else "ReadNextTasks") + page_sort_key_func = operator.attrgetter('queueTime') + super(TaskHistoryCollectorItems, self).__init__(session, + task_collector, + read_page_method, + reverse_page_order, + max_page_size, + page_sort_key_func) + + def __del__(self): + self.destroy_collector() + + +class EventHistoryCollectorItems(HistoryCollectorItems): + def __init__(self, session, event_filter_spec, reverse_page_order=False, + max_page_size=10): + event_collector = session._call_method( + session.vim, + "CreateCollectorForEvents", + session.vim.service_content.eventManager, + filter=event_filter_spec) + read_page_method = ("ReadPreviousEvents" if reverse_page_order + else "ReadNextEvents") + page_sort_key_func = operator.attrgetter('createdTime') + super(EventHistoryCollectorItems, self).__init__(session, + event_collector, + read_page_method, + reverse_page_order, + max_page_size, + page_sort_key_func) + + def __del__(self): + self.destroy_collector() + + def vm_refs_cache_reset(): global _VM_REFS_CACHE _VM_REFS_CACHE = {} From 2d186f20ebe94d71251090b95cf6fad9c6eb194f Mon Sep 17 00:00:00 2001 From: mmidolesov Date: Wed, 28 Mar 2018 07:47:59 -0700 Subject: [PATCH 74/92] Merge clone vm and glance parallel image import (cherry picked from commit 5c0f34233fee9573f5e409581c1f373e1115da2e) (cherry picked from commit f7bcba8b5a7d7c8b33122d92547fe6762003d39f) Change-Id: Ia89d1876dfe1b6e7d872c740126b1a5fcf0470b1 Incorporated-Change-Id: I65cb4339907f5d75430c1eb44ba1fcb364b2856c Incorporated-Change-Id: I5e4bee5f0326185a9b85f361695c941a7d813419 (cherry picked from commit d996c7b572e289a8f656e5f6ecf009ebff8092ad) --- nova/conf/vmware.py | 11 + .../unit/virt/vmwareapi/test_configdrive.py | 1 + .../unit/virt/vmwareapi/test_driver_api.py | 28 ++- nova/tests/unit/virt/vmwareapi/test_images.py | 2 + nova/tests/unit/virt/vmwareapi/test_vmops.py | 29 ++- nova/virt/vmwareapi/images.py | 111 ++++++++-- nova/virt/vmwareapi/vmops.py | 197 +++++++++++++++--- 7 files changed, 319 insertions(+), 60 deletions(-) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index e97a981bf62..ffe0021a854 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -368,6 +368,17 @@ Example: "vmx-13" for vSphere 6.5. Versions can be looked up here: https://kb.vmware.com/s/article/1003746 +"""), + cfg.BoolOpt('full_clone_snapshots', + default=False, + help=""" +Use full clones for creating image snapshots instead of linked clones. +With the right hardware support, it might be faster, especially on the export. +"""), + cfg.BoolOpt('clone_from_snapshot', + default=True, + help=""" +Create a snapshot of the VM before cloning it """), ] diff --git a/nova/tests/unit/virt/vmwareapi/test_configdrive.py b/nova/tests/unit/virt/vmwareapi/test_configdrive.py index 57ce91b730e..0e7112ece3a 100644 --- a/nova/tests/unit/virt/vmwareapi/test_configdrive.py +++ b/nova/tests/unit/virt/vmwareapi/test_configdrive.py @@ -93,6 +93,7 @@ def setUp(self, mock_register, mock_service): 'id': image_ref, 'disk_format': 'vmdk', 'size': int(metadata['size']), + 'owner': '', }) class FakeInstanceMetadata(object): diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 846397ce719..23df60d084a 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -206,6 +206,7 @@ def setUp(self, mock_svc, mock_register): 'id': image_ref, 'disk_format': 'vmdk', 'size': int(metadata['size']), + 'owner': '' }) self.fake_image_uuid = self.image.id self.vnc_host = 'ha-host' @@ -1252,7 +1253,9 @@ def _test_snapshot(self): self._check_vm_info(info, power_state.RUNNING) self.assertIsNone(func_call_matcher.match()) - def test_snapshot(self): + @mock.patch.object(vm_util, 'get_vmdk_info') + def test_snapshot(self, mock_get_vmdk_info): + mock_get_vmdk_info.return_value = self.get_fake_vmdk() self._create_vm() self._test_snapshot() @@ -1268,13 +1271,16 @@ def test_snapshot_non_existent(self): self.context, self.instance, "Test-Snapshot", lambda *args, **kwargs: None) + @mock.patch.object(vm_util, 'get_vmdk_info') @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps._delete_vm_snapshot') @mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps._create_vm_snapshot') @mock.patch('nova.virt.vmwareapi.volumeops.VMwareVolumeOps.' 'attach_volume') def test_snapshot_delete_vm_snapshot(self, mock_attach_volume, mock_create_vm_snapshot, - mock_delete_vm_snapshot): + mock_delete_vm_snapshot, + mock_get_vmdk): + mock_get_vmdk.return_value = self.get_fake_vmdk() self._create_vm() fake_vm = self._get_vm_record() snapshot_ref = vmwareapi_fake.ManagedObjectReference( @@ -1286,12 +1292,26 @@ def test_snapshot_delete_vm_snapshot(self, mock_attach_volume, self._test_snapshot() - mock_create_vm_snapshot.assert_called_once_with(self.instance, - fake_vm.obj) mock_delete_vm_snapshot.assert_called_once_with(self.instance, fake_vm.obj, snapshot_ref) + def get_fake_vmdk(self): + ds_ref = vmwareapi_fake.ManagedObjectReference(value='fake-ref') + ds = ds_obj.Datastore(ds_ref, 'ds1') + device = vmwareapi_fake.DataObject() + backing = vmwareapi_fake.DataObject() + backing.datastore = ds.ref + device.backing = backing + device.key = 'fake-key' + vmdk = vm_util.VmdkInfo('[fake] uuid/root.vmdk', + 'fake-adapter', + 'fake-disk', + 1024, + device) + + return vmdk + def _snapshot_delete_vm_snapshot_exception(self, exception, call_count=1): self._create_vm() fake_vm = vmwareapi_fake.get_first_object_ref("VirtualMachine") diff --git a/nova/tests/unit/virt/vmwareapi/test_images.py b/nova/tests/unit/virt/vmwareapi/test_images.py index 12ecb8d8c39..2a6da86a76f 100644 --- a/nova/tests/unit/virt/vmwareapi/test_images.py +++ b/nova/tests/unit/virt/vmwareapi/test_images.py @@ -226,6 +226,7 @@ def test_from_image_with_image_ref(self): raw_disk_size_in_bytes = raw_disk_size_in_gb * units.Gi mdata = {'size': raw_disk_size_in_bytes, 'disk_format': 'vmdk', + 'owner': '', 'properties': { "vmware_ostype": constants.DEFAULT_OS_TYPE, "vmware_adaptertype": constants.DEFAULT_ADAPTER_TYPE, @@ -262,6 +263,7 @@ def _image_build(self, image_lc_setting, global_lc_setting, mdata = {'size': raw_disk_size_in_btyes, 'disk_format': disk_format, + 'owner': '', 'properties': { "vmware_ostype": os_type, "vmware_adaptertype": adapter_type, diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 4648e74079d..a3d3261798a 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -120,7 +120,8 @@ def setUp(self): mock.Mock, cluster=cluster.obj) self._cluster = cluster - self._image_meta = objects.ImageMeta.from_dict({'id': self._image_id}) + self._image_meta = objects.ImageMeta.from_dict({'id': self._image_id, + 'owner': ''}) subnet_4 = network_model.Subnet(cidr='192.168.0.1/24', dns=[network_model.IP('192.168.0.1')], gateway= @@ -1940,6 +1941,7 @@ def _test_spawn(self, 'id': self._image_id, 'disk_format': 'vmdk', 'size': image_size, + 'owner': '' } image = objects.ImageMeta.from_dict(image) image_info = images.VMwareImage( @@ -2875,9 +2877,15 @@ def test_fetch_image_as_file_exception(self, cookies='Fake-CookieJar') @mock.patch.object(images, 'fetch_image_stream_optimized', - return_value=123) - def test_fetch_image_as_vapp(self, mock_fetch_image): + return_value=(123, '123')) + @mock.patch.object(vmops.VMwareVMOps, '_get_project_folder') + @mock.patch.object(vmops.VMwareVMOps, '_get_image_template_vm_name', + return_value='fake-name') + def test_fetch_image_as_vapp(self, mock_template_vm_name, + mock_get_project_folder, + mock_fetch_image): vi = self._make_vm_config_info() + mock_get_project_folder.return_value = vi.dc_info.vmFolder image_ds_loc = mock.Mock() image_ds_loc.parent.basename = 'fake-name' self._vmops._fetch_image_as_vapp(self._context, vi, image_ds_loc) @@ -2891,12 +2899,23 @@ def test_fetch_image_as_vapp(self, mock_fetch_image): self._vmops._root_resource_pool) self.assertEqual(vi.ii.file_size, 123) - @mock.patch.object(images, 'fetch_image_ova', return_value=123) - def test_fetch_image_as_ova(self, mock_fetch_image): + @mock.patch.object(images, 'fetch_image_ova', return_value=(123, '123')) + @mock.patch.object(vmops.VMwareVMOps, '_get_project_folder') + @mock.patch.object(vmops.VMwareVMOps, '_get_image_template_vm_name', + return_value='fake-name') + def test_fetch_image_as_ova(self, mock_template_vm_name, + mock_get_project_folder, + mock_fetch_image): vi = self._make_vm_config_info() + mock_get_project_folder.return_value = vi.dc_info.vmFolder image_ds_loc = mock.Mock() image_ds_loc.parent.basename = 'fake-name' self._vmops._fetch_image_as_ova(self._context, vi, image_ds_loc) + + mock_template_vm_name.assert_called_once_with( + vi.ii.image_id, vi.datastore.name + ) + mock_fetch_image.assert_called_once_with( self._context, vi.instance, diff --git a/nova/virt/vmwareapi/images.py b/nova/virt/vmwareapi/images.py index f8a129d5350..db8731dd027 100644 --- a/nova/virt/vmwareapi/images.py +++ b/nova/virt/vmwareapi/images.py @@ -17,19 +17,19 @@ Utility functions for Image transfer and manipulation. """ +from lxml import etree import os import tarfile -from lxml import etree from oslo_config import cfg from oslo_log import log as logging from oslo_service import loopingcall from oslo_utils import encodeutils from oslo_utils import strutils from oslo_utils import units +from oslo_vmware import exceptions as vexc from oslo_vmware import rw_handles - from nova import exception from nova.i18n import _ from nova.image import glance @@ -62,7 +62,8 @@ def __init__(self, image_id, file_type=constants.DEFAULT_DISK_FORMAT, linked_clone=None, vsphere_location=None, - vif_model=constants.DEFAULT_VIF_MODEL): + vif_model=constants.DEFAULT_VIF_MODEL, + owner=None): """VMwareImage holds values for use in building VMs. image_id (str): uuid of the image @@ -75,6 +76,7 @@ def __init__(self, image_id, linked_clone (bool): use linked clone, or don't vsphere_location (str): image location in datastore or None vif_model (str): virtual machine network interface + owner (str): uuid of the owning project """ self.image_id = image_id self.file_size = file_size @@ -84,6 +86,7 @@ def __init__(self, image_id, self.disk_type = disk_type self.file_type = file_type self.vsphere_location = vsphere_location + self.owner = owner # NOTE(vui): This should be removed when we restore the # descriptor-based validation. @@ -142,7 +145,8 @@ def from_image(cls, context, image_id, image_meta): 'image_id': image_id, 'linked_clone': linked_clone, 'container_format': container_format, - 'vsphere_location': get_vsphere_location(context, image_id) + 'vsphere_location': get_vsphere_location(context, image_id), + 'owner': image_meta.owner } if image_meta.obj_attr_is_set('size'): @@ -353,23 +357,91 @@ def fetch_image_stream_optimized(context, instance, session, vm_name, read_iter = IMAGE_API.download(context, image_ref) read_handle = rw_handles.ImageReadHandle(read_iter) - write_handle = rw_handles.VmdkWriteHandle(session, - session._host, - session._port, - res_pool_ref, - vm_folder_ref, - vm_import_spec, - file_size) - image_transfer(read_handle, write_handle) - - imported_vm_ref = write_handle.get_imported_vm() + # retry in order to handle conflicts in case of parallel execution + # (multiple agents or previously failed import of the same image + max_attempts = 3 + imported_vm_ref = None + for i in range(max_attempts): + try: + write_handle = rw_handles.VmdkWriteHandle(session, + session._host, + session._port, + res_pool_ref, + vm_folder_ref, + vm_import_spec, + file_size) + image_transfer(read_handle, write_handle) + imported_vm_ref = write_handle.get_imported_vm() + + break + except vexc.DuplicateName: + LOG.debug("Handling name duplication during import of VM %s", + vm_name) + vm_ref = vm_util.get_vm_ref_from_name(session, vm_name) + waited_for_ongoing_import = _wait_for_import_task(session, vm_ref) + if waited_for_ongoing_import: + imported_vm_ref = vm_ref + break + else: + try: + destroy_task = session._call_method(session.vim, + "Destroy_Task", + vm_ref) + session._wait_for_task(destroy_task) + vm_util.vm_ref_cache_delete(vm_name) + except vexc.ManagedObjectNotFoundException: + # another agent destroyed the VM in the meantime + pass + + if not imported_vm_ref: + raise vexc.VMwareDriverException("Could not import image %s within " + "%d attempts." % (vm_name, max_attempts)) LOG.info("Downloaded image file data %(image_ref)s", {'image_ref': instance.image_ref}, instance=instance) vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) session._call_method(session.vim, "UnregisterVM", imported_vm_ref) - LOG.info("The imported VM was unregistered", instance=instance) - return vmdk.capacity_in_bytes + return vmdk.capacity_in_bytes, vmdk.path + + +def _wait_for_import_task(session, vm_ref): + client_factory = session.vim.client.factory + task_filter_spec = client_factory.create('ns0:TaskFilterSpec') + task_filter_spec.entity = client_factory.create( + 'ns0:TaskFilterSpecByEntity') + task_filter_spec.entity.entity = vm_ref + task_filter_spec.entity.recursion = "self" + task_filter_spec.state = ["queued", "running"] + + waited = False + task_collector = None + + try: + task_collector = session._call_method( + session.vim, "CreateCollectorForTasks", + session.vim.service_content.taskManager, filter=task_filter_spec) + + while True: + page_tasks = session._call_method(session.vim, "ReadNextTasks", + task_collector, maxCount=10) + if len(page_tasks) == 0: + break + + for ti in page_tasks: + if ti.descriptionId == "ResourcePool.ImportVAppLRO": + try: + session._wait_for_task(ti.task) + waited = True + except vexc.VimException as e: + LOG.debug("Awaiting previous import on VM %s failed " + "with %s", vm_ref, e) + + return waited + finally: + if task_collector: + session._call_method(session.vim, + "DestroyCollector", task_collector) + return waited def get_vmdk_name_from_ovf(xmlstr): @@ -435,7 +507,12 @@ def fetch_image_ova(context, instance, session, vm_name, ds_name, imported_vm_ref) LOG.info("The imported VM was unregistered", instance=instance) - return vmdk.capacity_in_bytes + try: + return vmdk.capacity_in_bytes, vmdk.path.parent + except AttributeError: + from oslo_vmware.objects.datastore import DatastorePath + return (vmdk.capacity_in_bytes, + DatastorePath.parse(vmdk.path).parent) raise exception.ImageUnacceptable( reason=_("Extracting vmdk from OVA failed."), image_id=image_ref) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 60eb149d2d9..53bdbf24827 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -284,6 +284,11 @@ def _get_folder_name(self, name, id_): # We cannot truncate the 'id' as this is unique across OpenStack. return '%s (%s)' % (name[:40], id_[:36]) + def _get_project_folder(self, dc_info, project_id=None, type_=None): + folder_name = self._get_folder_name('Project', project_id) + folder_path = 'OpenStack/%s/%s' % (folder_name, type_) + return self._create_folders(dc_info.vmFolder, folder_path) + def build_virtual_machine(self, instance, context, image_info, dc_info, datastore, network_info, extra_specs, metadata): @@ -309,10 +314,8 @@ def build_virtual_machine(self, instance, context, image_info, profile_spec=profile_spec, metadata=metadata) - folder_name = self._get_folder_name('Project', - instance.project_id) - folder_path = 'OpenStack/%s/Instances' % folder_name - folder = self._create_folders(dc_info.vmFolder, folder_path) + folder = self._get_project_folder(dc_info, + project_id=instance.project_id, type_='Instances') # Create the VM vm_ref = vm_util.create_vm(self._session, instance, folder, @@ -454,49 +457,66 @@ def _fetch_image_as_file(self, context, vi, image_ds_loc): def _fetch_image_as_vapp(self, context, vi, image_ds_loc): """Download stream optimized image to host as a vApp.""" - # The directory of the imported disk is the unique name - # of the VM use to import it with. - vm_name = image_ds_loc.parent.basename + vm_name = self._get_image_template_vm_name(vi.ii.image_id, + vi.datastore.name) LOG.debug("Downloading stream optimized image %(image_id)s to " - "%(file_path)s on the data store " + "%(vm_name)s on the data store " "%(datastore_name)s as vApp", {'image_id': vi.ii.image_id, - 'file_path': image_ds_loc, + 'vm_name': vm_name, 'datastore_name': vi.datastore.name}, instance=vi.instance) - image_size = images.fetch_image_stream_optimized( + image_size, src_folder_ds_path = images.fetch_image_stream_optimized( context, vi.instance, self._session, vm_name, vi.datastore.name, - vi.dc_info.vmFolder, + self._get_project_folder(vi.dc_info, + project_id=vi.ii.owner, type_='Images'), self._root_resource_pool) # The size of the image is different from the size of the virtual disk. # We want to use the latter. On vSAN this is the only way to get this # size because there is no VMDK descriptor. vi.ii.file_size = image_size + self._cache_vm_image(vi, src_folder_ds_path) + + def _get_image_template_vm_name(self, image_id, datastore_name): + templ_vm_name = '%s (%s)' % (image_id, datastore_name) + return templ_vm_name def _fetch_image_as_ova(self, context, vi, image_ds_loc): """Download root disk of an OVA image as streamOptimized.""" - # The directory of the imported disk is the unique name - # of the VM use to import it with. - vm_name = image_ds_loc.parent.basename - - image_size = images.fetch_image_ova(context, - vi.instance, - self._session, - vm_name, - vi.datastore.name, - vi.dc_info.vmFolder, - self._root_resource_pool) + vm_name = self._get_image_template_vm_name(vi.ii.image_id, + vi.datastore.name) + + image_size, src_folder_ds_path = images.fetch_image_ova( + context, + vi.instance, + self._session, + vm_name, + vi.datastore.name, + self._get_project_folder(vi.dc_info, + project_id=vi.ii.owner, + type_='Images'), + self._root_resource_pool) + + self._move_to_cache(vi.dc_info.ref, + src_folder_ds_path, + vi.cache_image_folder) + + try: + ds_util.mkdir(self._session, vi.cache_image_folder, vi.dc_info.ref) + except vexc.FileAlreadyExistsException: + pass # The size of the image is different from the size of the virtual disk. # We want to use the latter. On vSAN this is the only way to get this # size because there is no VMDK descriptor. vi.ii.file_size = image_size + self._cache_vm_image(vi, src_folder_ds_path) def _prepare_sparse_image(self, vi): tmp_dir_loc = vi.datastore.build_path( @@ -575,6 +595,19 @@ def _cache_flat_image(self, vi, tmp_image_ds_loc): tmp_image_ds_loc.parent, vi.cache_image_folder) + def _cache_vm_image(self, vi, tmp_image_ds_loc): + dst_path = vi.cache_image_folder.join("%s.vmdk" % vi.ii.image_id) + try: + ds_util.mkdir(self._session, + vi.cache_image_folder, vi.dc_info.ref) + except vexc.FileAlreadyExistsException: + pass + try: + ds_util.disk_copy(self._session, vi.dc_info.ref, + tmp_image_ds_loc, dst_path) + except vexc.FileAlreadyExistsException: + pass + def _cache_stream_optimized_image(self, vi, tmp_image_ds_loc): dst_path = vi.cache_image_folder.join("%s.vmdk" % vi.ii.image_id) ds_util.mkdir(self._session, vi.cache_image_folder, vi.dc_info.ref) @@ -628,7 +661,10 @@ def _get_image_callbacks(self, vi): else: image_fetch = self._fetch_image_as_file - if vi.ii.is_iso: + if vi.ii.is_ova or disk_type == constants.DISK_TYPE_STREAM_OPTIMIZED: + image_prepare = lambda vi: (None, None) + image_cache = lambda vi, image_loc: None + elif vi.ii.is_iso: image_prepare = self._prepare_iso_image image_cache = self._cache_iso_image elif disk_type == constants.DISK_TYPE_SPARSE: @@ -665,7 +701,9 @@ def _fetch_image_if_missing(self, context, vi): image_cache(vi, tmp_image_ds_loc) LOG.debug("Cleaning up location %s", str(tmp_dir_loc), instance=vi.instance) - self._delete_datastore_file(str(tmp_dir_loc), vi.dc_info.ref) + if tmp_dir_loc: + self._delete_datastore_file(str(tmp_dir_loc), + vi.dc_info.ref) # The size of the sparse image is different from the size of the # virtual disk. We want to use the latter. @@ -988,12 +1026,12 @@ def _attach_cdrom_to_vm(self, vm_ref, instance, LOG.debug("Reconfigured VM instance to attach cdrom %s", file_path, instance=instance) - def _create_vm_snapshot(self, instance, vm_ref): + def _create_vm_snapshot(self, instance, vm_ref, image_id=None): LOG.debug("Creating Snapshot of the VM instance", instance=instance) snapshot_task = self._session._call_method( self._session.vim, "CreateSnapshot_Task", vm_ref, - name="%s-snapshot" % instance.uuid, + name="%s-snapshot" % (image_id or instance.uuid), description="Taking Snapshot of the VM", memory=False, quiesce=True) @@ -1047,6 +1085,83 @@ def _create_linked_clone_from_snapshot(self, instance, "info") return task_info.result + def _create_vm_clone(self, instance, vm_ref, snapshot_ref, dc_info, + disk_move_type=None, image_id=None, disks=None): + """Clone VM to be deployed to same ds as source VM + """ + image_id = image_id or uuidutils.generate_uuid() + + if disks: + datastore = disks[0].device.backing.datastore + else: + if disk_move_type == "createNewChildDiskBacking": + datastore = None + else: + datastore = ds_util.get_datastore(self._session, self._cluster, + self._datastore_regex) + + vm_name = "%s_%s" % (constants.SNAPSHOT_VM_PREFIX, + image_id) + client_factory = self._session.vim.client.factory + rel_spec = vm_util.relocate_vm_spec( + client_factory, + datastore=datastore, + host=None, + disk_move_type=disk_move_type) + config_spec = client_factory.create('ns0:VirtualMachineConfigSpec') + config_spec.name = vm_name + config_spec.annotation = "Created from %s" % (instance.uuid) + config_spec.numCPUs = 1 + config_spec.numCoresPerSocket = 1 + config_spec.memoryMB = 16 + config_spec.uuid = image_id # Not instanceUuid, + # as we need to import the same image in different datastores + + if disks: + disk_devices = [vmdk_info.device.key for vmdk_info in disks] + hardware_devices = self._session._call_method( + vutil, + "get_object_property", + vm_ref, + "config.hardware.device") + if hardware_devices.__class__.__name__ == "ArrayOfVirtualDevice": + hardware_devices = hardware_devices.VirtualDevice + + device_change = [] + for device in hardware_devices: + if getattr(device, 'macAddress', None) or \ + device.__class__.__name__ == "VirtualDisk" and \ + device.key not in disk_devices: + removal = client_factory.create( + 'ns0:VirtualDeviceConfigSpec') + removal.device = device + removal.operation = 'remove' + device_change.append(removal) + + config_spec.deviceChange = device_change + + clone_spec = vm_util.clone_vm_spec(client_factory, rel_spec, + power_on=False, + snapshot=snapshot_ref, + template=True, + config=config_spec) + + LOG.debug("Cloning VM %s", vm_name, instance=instance) + vm_clone_task = self._session._call_method( + self._session.vim, + "CloneVM_Task", + vm_ref, + folder=self._get_project_folder(dc_info, + project_id=instance.project_id, type_='Images'), + name=vm_name, + spec=clone_spec) + self._session._wait_for_task(vm_clone_task) + task_info = self._session._call_method(vutil, + "get_object_property", + vm_clone_task, + "info") + return task_info.result + def snapshot(self, context, instance, image_id, update_task_state): """Create snapshot from a running VM instance. @@ -1089,17 +1204,30 @@ def _get_vm_and_vmdk_attribs(): # TODO(vui): convert to creating plain vm clone and uploading from it # instead of using live vm snapshot. - snapshot_ref = self._create_vm_snapshot(instance, vm_ref) - update_task_state(task_state=task_states.IMAGE_UPLOADING, - expected_state=task_states.IMAGE_PENDING_UPLOAD) + snapshot_ref = None + snapshot_vm_ref = None try: - # Create a temporary VM (linked clone from snapshot), then export - # the VM's root disk to glance via HttpNfc API - snapshot_vm_ref = self._create_linked_clone_from_snapshot( - instance, vm_ref, snapshot_ref, dc_info) + # If we do linked clones, we need to have a snapshot + if (CONF.vmware.clone_from_snapshot or + not CONF.vmware.full_clone_snapshots): + snapshot_ref = self._create_vm_snapshot( + instance, vm_ref, image_id=image_id) + + if not CONF.vmware.full_clone_snapshots: + disk_move_type = "createNewChildDiskBacking" + else: + disk_move_type = None + + snapshot_vm_ref = self._create_vm_clone( + instance, vm_ref, snapshot_ref, dc_info, + disk_move_type=disk_move_type, image_id=image_id, + disks=[vmdk]) + + update_task_state(task_state=task_states.IMAGE_UPLOADING, + expected_state=task_states.IMAGE_PENDING_UPLOAD) images.upload_image_stream_optimized( context, image_id, instance, self._session, vm=snapshot_vm_ref, vmdk_size=vmdk.capacity_in_bytes) @@ -1114,7 +1242,8 @@ def _get_vm_and_vmdk_attribs(): # Deleting the snapshot after destroying the temporary VM created # based on it allows the instance vm's disks to be consolidated. # TODO(vui) Add handling for when vmdk volume is attached. - self._delete_vm_snapshot(instance, vm_ref, snapshot_ref) + if snapshot_ref: + self._delete_vm_snapshot(instance, vm_ref, snapshot_ref) def reboot(self, instance, network_info, reboot_type="SOFT"): """Reboot a VM instance.""" From f3a6d5e7fc05dfd27bcd9392705a538003abfe16 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 13 Dec 2024 11:50:47 +0100 Subject: [PATCH 75/92] VMware: Image as VM template - Import image as VM template per datastore. - First look on other datastores before fetching the image. In previous versions, we also had code in here that would create a VM from these template-VMs. We never used that feature and thus stopped porting it forward. When we try to fetch an image-template from another datastore, it might happen, that the template has an incompatible hardware version and the vcenter raises VirtualHardwareVersionNotSupported if we try to clone it to our cluster. We handle this case by logging a debug message and continuing with the next image-template we find, as this one is unusable for us. Change-Id: Ia806089b244d331003fe6f7e047b5908088ff367 Incorporated-Change-Id: If9dc9b2a13171252e5f0f0b3a99a51be2f28c6eb Incorporated-Change-Id: I0aa58ad11c499e9423c7ecc7998325b05dd9147e (cherry picked from commit a04c8f79407c8e0f62080898c64a97334c4fff9e) --- nova/conf/vmware.py | 6 + .../unit/virt/vmwareapi/test_configdrive.py | 7 +- .../unit/virt/vmwareapi/test_driver_api.py | 18 ++- nova/tests/unit/virt/vmwareapi/test_images.py | 4 +- nova/tests/unit/virt/vmwareapi/test_vmops.py | 29 +++- nova/virt/vmwareapi/ds_util.py | 20 ++- nova/virt/vmwareapi/images.py | 7 +- nova/virt/vmwareapi/vm_util.py | 74 +++++++--- nova/virt/vmwareapi/vmops.py | 136 ++++++++++++++---- 9 files changed, 233 insertions(+), 68 deletions(-) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index ffe0021a854..3b505a89618 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -379,6 +379,12 @@ default=True, help=""" Create a snapshot of the VM before cloning it +"""), + cfg.BoolOpt('fetch_image_from_other_datastores', + default=True, + help=""" +Before fetching from Glance an image missing on the datastore first look +for it on other datastores and clone it from there if available. """), ] diff --git a/nova/tests/unit/virt/vmwareapi/test_configdrive.py b/nova/tests/unit/virt/vmwareapi/test_configdrive.py index 0e7112ece3a..8d20c59dda8 100644 --- a/nova/tests/unit/virt/vmwareapi/test_configdrive.py +++ b/nova/tests/unit/virt/vmwareapi/test_configdrive.py @@ -127,7 +127,12 @@ def tearDown(self): @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata', return_value='fake_metadata') - def _spawn_vm(self, fake_get_instance_meta, + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) + def _spawn_vm(self, fake_fetch_image_from_other_datastores, + fake_find_image_template_vm, fake_get_instance_meta, mock_update_cluster_placement, injected_files=None, admin_password=None, block_device_info=None): diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 23df60d084a..9676e1e4c52 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -356,9 +356,15 @@ def _create_instance(self, node=None, set_image_ref=True, @mock.patch('nova.compute.utils.is_volume_backed_instance', return_value=False) - def _create_vm(self, fake_is_volume_backed, node=None, num_instances=1, - uuid=None, flavor='m1.large', powered_on=True, - ephemeral=None, bdi=None, flavor_updates=None): + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) + def _create_vm(self, fake_fetch_image_from_other_datastores, + fake_find_image_template_vm, fake_is_volume_backed, + node=None, num_instances=1, uuid=None, flavor='m1.large', + powered_on=True, ephemeral=None, bdi=None, + flavor_updates=None): """Create and spawn the VM.""" if not node: node = self.node_name @@ -1159,7 +1165,13 @@ def _spawn_attach_volume_vmdk(self, mock_info_get_mapping, 'attach_volume') @mock.patch('nova.block_device.volume_in_mapping') @mock.patch('nova.virt.driver.block_device_info_get_mapping') + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) def test_spawn_attach_volume_iscsi(self, + mock_fetch_image_from_other_datastores, + mock_find_image_template_vm, mock_info_get_mapping, mock_block_volume_in_mapping, mock_attach_volume, diff --git a/nova/tests/unit/virt/vmwareapi/test_images.py b/nova/tests/unit/virt/vmwareapi/test_images.py index 2a6da86a76f..96b4e6c9d80 100644 --- a/nova/tests/unit/virt/vmwareapi/test_images.py +++ b/nova/tests/unit/virt/vmwareapi/test_images.py @@ -171,7 +171,7 @@ def fake_extract(name): mock_get_vmdk_info.assert_called_once_with( session, mock.sentinel.vm_ref) session._call_method.assert_called_once_with( - session.vim, "UnregisterVM", mock.sentinel.vm_ref) + session.vim, "MarkAsTemplate", mock.sentinel.vm_ref) @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') @mock.patch('oslo_vmware.rw_handles.VmdkWriteHandle') @@ -217,7 +217,7 @@ def test_fetch_image_stream_optimized(self, mock_image_transfer.assert_called_once_with(mock_read_handle, mock_write_handle) session._call_method.assert_called_once_with( - session.vim, "UnregisterVM", mock.sentinel.vm_ref) + session.vim, "MarkAsTemplate", mock.sentinel.vm_ref) mock_get_vmdk_info.assert_called_once_with( session, mock.sentinel.vm_ref) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index a3d3261798a..c27dc4ca319 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -406,9 +406,14 @@ def test_get_datacenter_ref_and_name_with_no_datastore(self): @mock.patch.object(vm_util, 'reconfigure_vm') @mock.patch.object(vm_util, 'power_on_instance') @mock.patch.object(ds_obj, 'get_datastore_by_ref') - def test_rescue(self, mock_get_ds_by_ref, mock_power_on, mock_reconfigure, - mock_get_boot_spec, mock_find_rescue, - mock_get_vm_ref, mock_disk_copy, + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) + def test_rescue(self, mock_fetch_image_from_other_datastores, + mock_find_image_template_vm, mock_get_ds_by_ref, + mock_power_on, mock_reconfigure, mock_get_boot_spec, + mock_find_rescue, mock_get_vm_ref, mock_disk_copy, mock_power_off, mock_glance): _volumeops = mock.Mock() self._vmops._volumeops = _volumeops @@ -1973,10 +1978,17 @@ def _test_spawn(self, return_value=extra_specs), mock.patch.object(self._vmops, '_get_instance_metadata', return_value='fake-metadata'), - mock.patch.object(ds_util, 'file_size', return_value=0) + mock.patch.object(ds_util, 'file_size', return_value=0), + mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + '_find_image_template_vm', + return_value=None), + mock.patch('nova.virt.vmwareapi.vmops.VMwareVMOps.' + '_fetch_image_from_other_datastores', + return_value=None) ) as (_wait_for_task, _call_method, _generate_uuid, _fetch_image, _get_img_svc, _get_inventory_path, _get_extra_specs, - _get_instance_metadata, file_size): + _get_instance_metadata, file_size, _find_image_template_vm, + _fetch_image_from_other_datastores): self._vmops.spawn(self._context, self._instance, image, injected_files='fake_files', admin_password='password', @@ -1996,6 +2008,7 @@ def _test_spawn(self, extra_specs, constants.DEFAULT_OS_TYPE, profile_spec=None, + vm_name=None, metadata='fake-metadata') mock_create_vm.assert_called_once_with( self._session, @@ -2720,7 +2733,13 @@ def _make_vm_config_info(self, is_iso=False, is_sparse_disk=False, @mock.patch.object(vmops.VMwareVMOps, '_cache_flat_image') @mock.patch.object(vmops.VMwareVMOps, '_delete_datastore_file') @mock.patch.object(vmops.VMwareVMOps, '_update_image_size') + @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', + return_value=None) + @mock.patch.object(vmops.VMwareVMOps, '_fetch_image_from_other_datastores', + return_value=None) def _test_fetch_image_if_missing(self, + mock_fetch_image_from_other_datastores, + mock_find_image_template_vm, mock_update_image_size, mock_delete_datastore_file, mock_cache_flat_image, diff --git a/nova/virt/vmwareapi/ds_util.py b/nova/virt/vmwareapi/ds_util.py index 7ff353845f1..8dfb3532cb1 100644 --- a/nova/virt/vmwareapi/ds_util.py +++ b/nova/virt/vmwareapi/ds_util.py @@ -173,12 +173,22 @@ def _get_allowed_datastores(datastores, datastore_regex): freespace=propdict['summary.freeSpace'])) -def get_available_datastores(session, cluster=None, datastore_regex=None): +def get_available_datastores(session, cluster=None, + datastore_regex=None, dc_ref=None): """Get the datastore list and choose the first local storage.""" - ds = session._call_method(vutil, - "get_object_property", - cluster, - "datastore") + if cluster: + ds = session._call_method(vutil, + "get_object_property", + cluster, + "datastore") + elif dc_ref: + ds = session._call_method(vutil, + "get_object_property", + dc_ref, + "datastore") + else: + return [] + if not ds: return [] datastore_mors = ds.ManagedObjectReference diff --git a/nova/virt/vmwareapi/images.py b/nova/virt/vmwareapi/images.py index db8731dd027..2e9d462b64d 100644 --- a/nova/virt/vmwareapi/images.py +++ b/nova/virt/vmwareapi/images.py @@ -400,7 +400,7 @@ def fetch_image_stream_optimized(context, instance, session, vm_name, LOG.info("Downloaded image file data %(image_ref)s", {'image_ref': instance.image_ref}, instance=instance) vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) - session._call_method(session.vim, "UnregisterVM", imported_vm_ref) + vm_util.mark_vm_as_template(session, instance, imported_vm_ref) return vmdk.capacity_in_bytes, vmdk.path @@ -503,10 +503,7 @@ def fetch_image_ova(context, instance, session, vm_name, ds_name, imported_vm_ref = write_handle.get_imported_vm() vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) - session._call_method(session.vim, "UnregisterVM", - imported_vm_ref) - LOG.info("The imported VM was unregistered", - instance=instance) + vm_util.mark_vm_as_template(session, instance, imported_vm_ref) try: return vmdk.capacity_in_bytes, vmdk.path.parent except AttributeError: diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 3f793c68b96..c718ce887a8 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -292,13 +292,37 @@ def _get_allocation_info(client_factory, limits, allocation_type): return allocation +def append_vif_infos_to_config_spec(client_factory, config_spec, + vif_infos, vif_limits, index=0): + + if not hasattr(config_spec, 'deviceChange') or \ + not config_spec.deviceChange: + config_spec.deviceChange = [] + for offset, vif_info in enumerate(vif_infos): + vif_spec = _create_vif_spec(client_factory, vif_info, vif_limits, + offset) + config_spec.deviceChange.append(vif_spec) + + if not hasattr(config_spec, 'extraConfig') or \ + not config_spec.extraConfig: + config_spec.extraConfig = [] + port_index = index + for vif_info in vif_infos: + if vif_info['iface_id']: + config_spec.extraConfig.append( + _iface_id_option_value(client_factory, + vif_info['iface_id'], + port_index)) + port_index += 1 + + def get_vm_create_spec(client_factory, instance, data_store_name, vif_infos, extra_specs, os_type=constants.DEFAULT_OS_TYPE, - profile_spec=None, metadata=None): + profile_spec=None, metadata=None, vm_name=None): """Builds the VM Create spec.""" config_spec = client_factory.create('ns0:VirtualMachineConfigSpec') - config_spec.name = instance.uuid + config_spec.name = vm_name or instance.uuid config_spec.guestId = os_type # The name is the unique identifier for the VM. config_spec.instanceUuid = instance.uuid @@ -355,11 +379,6 @@ def get_vm_create_spec(client_factory, instance, data_store_name, config_spec.firmware = extra_specs.firmware devices = [] - for i, vif_info in enumerate(vif_infos): - vif_spec = _create_vif_spec(client_factory, vif_info, - extra_specs.vif_limits, i) - devices.append(vif_spec) - serial_port_spec = create_serial_port_spec(client_factory) if serial_port_spec: devices.append(serial_port_spec) @@ -384,14 +403,6 @@ def get_vm_create_spec(client_factory, instance, data_store_name, opt.value = 'true' extra_config.append(opt) - port_index = 0 - for vif_info in vif_infos: - if vif_info['iface_id']: - extra_config.append(_iface_id_option_value(client_factory, - vif_info['iface_id'], - port_index)) - port_index += 1 - if (CONF.vmware.console_delay_seconds and CONF.vmware.console_delay_seconds > 0): opt = client_factory.create('ns0:OptionValue') @@ -407,6 +418,9 @@ def get_vm_create_spec(client_factory, instance, data_store_name, config_spec.extraConfig = extra_config + append_vif_infos_to_config_spec(client_factory, config_spec, + vif_infos, extra_specs.vif_limits) + # Set the VM to be 'managed' by 'OpenStack' managed_by = client_factory.create('ns0:ManagedByInfo') managed_by.extensionKey = constants.EXTENSION_KEY @@ -628,12 +642,10 @@ def get_network_attach_config_spec(client_factory, vif_info, index, vif_limits=None): """Builds the vif attach config spec.""" config_spec = client_factory.create('ns0:VirtualMachineConfigSpec') - vif_spec = _create_vif_spec(client_factory, vif_info, vif_limits) - config_spec.deviceChange = [vif_spec] - if vif_info['iface_id'] is not None: - config_spec.extraConfig = [_iface_id_option_value(client_factory, - vif_info['iface_id'], - index)] + + append_vif_infos_to_config_spec(client_factory, config_spec, + [vif_info], vif_limits, index) + return config_spec @@ -1253,6 +1265,14 @@ def _get_vm_ref_from_vm_uuid(session, instance_uuid): return vm_refs[0] +def find_by_inventory_path(session, inv_path): + return session._call_method( + session.vim, + "FindByInventoryPath", + session.vim.service_content.searchIndex, + inventoryPath=inv_path) + + def _get_vm_ref_from_extraconfig(session, instance_uuid): """Get reference to the VM with the uuid specified.""" vms = session._call_method(vim_util, "get_objects", @@ -1547,6 +1567,18 @@ def destroy_vm(session, instance, vm_ref=None): LOG.exception('Destroy VM failed', instance=instance) +def mark_vm_as_template(session, instance, vm_ref=None): + """Mark a VM instance as template. Assumes VM is powered off.""" + try: + if not vm_ref: + vm_ref = get_vm_ref(session, instance) + LOG.debug("Marking the VM as template", instance=instance) + session._call_method(session.vim, "MarkAsTemplate", vm_ref) + LOG.info("Marked the VM as template", instance=instance) + except Exception: + LOG.exception('Mark VM as template failed', instance=instance) + + def create_virtual_disk(session, dc_ref, adapter_type, disk_type, virtual_disk_path, size_in_kb): # Create a Virtual Disk of the size of the flat vmdk file. This is diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 53bdbf24827..38e9ea56266 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -20,6 +20,7 @@ """ import collections +import copy import os import re import time @@ -284,14 +285,19 @@ def _get_folder_name(self, name, id_): # We cannot truncate the 'id' as this is unique across OpenStack. return '%s (%s)' % (name[:40], id_[:36]) - def _get_project_folder(self, dc_info, project_id=None, type_=None): + def _get_project_folder_path(self, project_id, type_): folder_name = self._get_folder_name('Project', project_id) folder_path = 'OpenStack/%s/%s' % (folder_name, type_) + + return folder_path + + def _get_project_folder(self, dc_info, project_id=None, type_=None): + folder_path = self._get_project_folder_path(project_id, type_) return self._create_folders(dc_info.vmFolder, folder_path) - def build_virtual_machine(self, instance, context, image_info, - dc_info, datastore, network_info, extra_specs, - metadata): + def _get_vm_config_spec(self, instance, image_info, + datastore, network_info, extra_specs, + metadata, vm_name=None): vif_infos = vmwarevif.get_vif_info(self._session, self._cluster, image_info.vif_model, @@ -312,10 +318,21 @@ def build_virtual_machine(self, instance, context, image_info, extra_specs, image_info.os_type, profile_spec=profile_spec, - metadata=metadata) + metadata=metadata, + vm_name=vm_name) + + return config_spec + + def build_virtual_machine(self, instance, context, image_info, + dc_info, datastore, network_info, extra_specs, + metadata, folder_type='Instances', vm_name=None): + config_spec = self._get_vm_config_spec(instance, image_info, datastore, + network_info, extra_specs, + metadata, vm_name=vm_name) folder = self._get_project_folder(dc_info, - project_id=instance.project_id, type_='Instances') + project_id=instance.project_id, + type_=folder_type) # Create the VM vm_ref = vm_util.create_vm(self._session, instance, folder, @@ -454,6 +471,10 @@ def _fetch_image_as_file(self, context, vi, image_ds_loc): image_ds_loc.rel_path, cookies=cookies) + def _get_image_template_vm_name(self, image_id, datastore_name): + templ_vm_name = '%s (%s)' % (image_id, datastore_name) + return templ_vm_name + def _fetch_image_as_vapp(self, context, vi, image_ds_loc): """Download stream optimized image to host as a vApp.""" @@ -475,7 +496,7 @@ def _fetch_image_as_vapp(self, context, vi, image_ds_loc): vm_name, vi.datastore.name, self._get_project_folder(vi.dc_info, - project_id=vi.ii.owner, type_='Images'), + project_id=vi.instance.project_id, type_='Images'), self._root_resource_pool) # The size of the image is different from the size of the virtual disk. # We want to use the latter. On vSAN this is the only way to get this @@ -483,15 +504,11 @@ def _fetch_image_as_vapp(self, context, vi, image_ds_loc): vi.ii.file_size = image_size self._cache_vm_image(vi, src_folder_ds_path) - def _get_image_template_vm_name(self, image_id, datastore_name): - templ_vm_name = '%s (%s)' % (image_id, datastore_name) - return templ_vm_name - def _fetch_image_as_ova(self, context, vi, image_ds_loc): """Download root disk of an OVA image as streamOptimized.""" - vm_name = self._get_image_template_vm_name(vi.ii.image_id, - vi.datastore.name) + vm_name = self._get_image_template_vm_name( + vi.ii.image_id, vi.datastore.name) image_size, src_folder_ds_path = images.fetch_image_ova( context, @@ -500,18 +517,9 @@ def _fetch_image_as_ova(self, context, vi, image_ds_loc): vm_name, vi.datastore.name, self._get_project_folder(vi.dc_info, - project_id=vi.ii.owner, - type_='Images'), + project_id=vi.instance.project_id, type_='Images'), self._root_resource_pool) - self._move_to_cache(vi.dc_info.ref, - src_folder_ds_path, - vi.cache_image_folder) - - try: - ds_util.mkdir(self._session, vi.cache_image_folder, vi.dc_info.ref) - except vexc.FileAlreadyExistsException: - pass # The size of the image is different from the size of the virtual disk. # We want to use the latter. On vSAN this is the only way to get this # size because there is no VMDK descriptor. @@ -681,6 +689,49 @@ def _get_image_callbacks(self, vi): raise exception.InvalidDiskInfo(reason=reason) return image_prepare, image_fetch, image_cache + def _fetch_image_from_other_datastores(self, vi): + dc_all_datastores = ds_util.get_available_datastores( + self._session, dc_ref=vi.dc_info.ref) + dc_other_datastores = [ds for ds in dc_all_datastores if + dict(ds.ref) != dict(vi.datastore.ref)] + + client_factory = self._session.vim.client.factory + tmp_vi = copy.copy(vi) + for ds in dc_other_datastores: + tmp_vi.datastore = ds + other_templ_vm_ref = self._find_image_template_vm(tmp_vi) + if other_templ_vm_ref: + rel_spec = vm_util.relocate_vm_spec( + client_factory, + res_pool=self._root_resource_pool, + disk_move_type="moveAllDiskBackingsAndDisallowSharing", + datastore=vi.datastore.ref) + clone_spec = vm_util.clone_vm_spec(client_factory, + rel_spec, template=True) + templ_vm_clone_task = self._session._call_method( + self._session.vim, + "CloneVM_Task", + other_templ_vm_ref, + folder=self._get_project_folder(vi.dc_info, + project_id=vi.instance.project_id, type_='Images'), + name=self._get_image_template_vm_name( + vi.ii.image_id, vi.datastore.name), + spec=clone_spec) + try: + task_info = \ + self._session._wait_for_task(templ_vm_clone_task) + except vexc.VimFaultException as e: + if 'VirtualHardwareVersionNotSupported' in e.fault_list: + LOG.debug('Could not clone image-template from ' + 'incompatible hardware platform') + else: + LOG.warning('Could not clone image-template from ' + 'other datastore.') + continue + + templ_vm_ref = task_info.result + return templ_vm_ref + def _fetch_image_if_missing(self, context, vi): image_prepare, image_fetch, image_cache = self._get_image_callbacks(vi) LOG.debug("Processing image %s", vi.ii.image_id, instance=vi.instance) @@ -689,9 +740,21 @@ def _fetch_image_if_missing(self, context, vi): lock_file_prefix='nova-vmware-fetch_image'): self.check_cache_folder(vi.datastore.name, vi.datastore.ref) ds_browser = self._get_ds_browser(vi.datastore.ref) - if not ds_util.file_exists(self._session, ds_browser, - vi.cache_image_folder, - vi.cache_image_path.basename): + image_available = ds_util.file_exists( + self._session, ds_browser, + vi.cache_image_folder, + vi.cache_image_path.basename) + + if not image_available: + templ_vm_ref = self._find_image_template_vm(vi) + image_available = (templ_vm_ref is not None) + + if (not image_available and + CONF.vmware.fetch_image_from_other_datastores): + templ_vm_ref = self._fetch_image_from_other_datastores(vi) + image_available = (templ_vm_ref is not None) + + if not image_available: LOG.debug("Preparing fetch location", instance=vi.instance) tmp_dir_loc, tmp_image_ds_loc = image_prepare(vi) LOG.debug("Fetch image to %s", tmp_image_ds_loc, @@ -834,6 +897,27 @@ def update_admin_vm_group_membership(self, instance, remove=False): vm_group_name, vm_ref, remove=remove) + def _build_template_vm_inventory_path(self, vi): + vm_folder_name = self._session._call_method(vutil, + "get_object_property", + vi.dc_info.vmFolder, + "name") + images_folder_path = self._get_project_folder_path( + vi.instance.project_id, 'Images') + templ_vm_name = self._get_image_template_vm_name(vi.ii.image_id, + vi.datastore.name) + templ_vm_inventory_path = '%s/%s/%s/%s' % ( + vi.dc_info.name, vm_folder_name, + images_folder_path, templ_vm_name) + return templ_vm_inventory_path + + def _find_image_template_vm(self, vi): + templ_vm_inventory_path = self._build_template_vm_inventory_path(vi) + templ_vm_ref = vm_util.find_by_inventory_path(self._session, + templ_vm_inventory_path) + + return templ_vm_ref + def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info, block_device_info=None): From 65f813fef7a723d6d2082389c667feb25ba8a6d3 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 26 Aug 2021 14:34:27 +0200 Subject: [PATCH 76/92] Vmwareapi: Fix some linting issues Order of imports Duplicate imports Spelling mistake Indentation Change-Id: I4ff5594b0a628fee9579761248627099b3f251b8 (cherry picked from commit ec4bb85f2c13471a303d0dbc0c299cb27eb98bb9) --- nova/virt/vmwareapi/driver.py | 6 +++--- nova/virt/vmwareapi/vmops.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index af2436ed90a..29fd21512a1 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -118,8 +118,8 @@ def __init__(self, virtapi, scheme="https"): super(VMwareVCDriver, self).__init__(virtapi) if (CONF.vmware.host_ip is None or - CONF.vmware.host_username is None or - CONF.vmware.host_password is None): + CONF.vmware.host_username is None or + CONF.vmware.host_password is None): raise Exception(_("Must specify host_ip, host_username and " "host_password to use vmwareapi.VMwareVCDriver")) @@ -180,7 +180,7 @@ def _check_min_version(self): 'vCenter version %(min_version)s or greater.') % { 'version': vc_version, 'min_version': constants.MIN_VC_VERSION}) - elif v_utils.convert_version_to_int(vc_version) < next_min_ver: + if v_utils.convert_version_to_int(vc_version) < next_min_ver: LOG.warning('Running Nova with a VMware vCenter version less ' 'than %(version)s is deprecated. The required ' 'minimum version of vCenter will be raised to ' diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 38e9ea56266..d8f1fc0485a 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -21,11 +21,11 @@ import collections import copy +import decorator import os import re import time -import decorator from oslo_concurrency import lockutils from oslo_log import log as logging from oslo_serialization import jsonutils From 57b10066f54f3dc03c1a6247e6460f5b420a91bd Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 26 Aug 2021 14:34:27 +0200 Subject: [PATCH 77/92] vmwareapi: Switch to using olso_vmware.vim_util Instead of renaming `vim_util` to `vutil`, we use it directly as `vim_util`, because we have no need to import another `vim_util`. Change-Id: Ic484240f29573cffbbc05742da0231ce17764dca (cherry picked from commit f5f97156e8d07b5212d703ade59f2f4c85ab4154) --- nova/virt/vmwareapi/cluster_util.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index ceefb84b9ca..46c43c23295 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -15,7 +15,7 @@ from oslo_log import log as logging -from oslo_vmware import vim_util as vutil +from oslo_vmware import vim_util from nova import exception from nova.i18n import _ @@ -119,15 +119,15 @@ def fetch_cluster_groups(session, cluster_ref=None, cluster_config=None, if cluster_config is None: cluster_config = session._call_method( - vutil, "get_object_property", cluster_ref, "configurationEx") + vim_util, "get_object_property", cluster_ref, "configurationEx") groups = {} for group in getattr(cluster_config, 'group', []): if group_type == 'vm': - if not vutil.is_vim_instance(group, 'ClusterVmGroup'): + if not vim_util.is_vim_instance(group, 'ClusterVmGroup'): continue elif group_type == 'host': - if not vutil.is_vim_instance(group, 'ClusterHostGroup'): + if not vim_util.is_vim_instance(group, 'ClusterHostGroup'): continue groups[group.name] = group @@ -147,7 +147,7 @@ def fetch_cluster_rules(session, cluster_ref=None, cluster_config=None): if cluster_config is None: cluster_config = session._call_method( - vutil, "get_object_property", cluster_ref, "configurationEx") + vim_util, "get_object_property", cluster_ref, "configurationEx") return {r.name: r for r in getattr(cluster_config, 'rule', [])} @@ -180,7 +180,7 @@ def update_vm_group_membership(session, cluster, vm_group_name, vm_ref, on that group. """ cluster_config = session._call_method( - vutil, "get_object_property", cluster, "configurationEx") + vim_util, "get_object_property", cluster, "configurationEx") client_factory = session.vim.client.factory config_spec = client_factory.create('ns0:ClusterConfigSpecEx') @@ -203,8 +203,9 @@ def update_vm_group_membership(session, cluster, vm_group_name, vm_ref, return filtered_vms = [] found = False + vm_ref_value = vim_util.get_moref_value(vm_ref) for ref in group.vm: - if (vutil.get_moref_value(ref) == vutil.get_moref_value(vm_ref)): + if vim_util.get_moref_value(ref) == vm_ref_value: found = True else: filtered_vms.append(ref) @@ -302,7 +303,7 @@ def add_rule(session, cluster_ref, rule): def get_rule(session, cluster_ref, rule_name): """Get a DRS rule from the cluster by name""" cluster_config = session._call_method( - vutil, "get_object_property", cluster_ref, "configurationEx") + vim_util, "get_object_property", cluster_ref, "configurationEx") return _get_rule(cluster_config, rule_name) @@ -321,7 +322,7 @@ def get_rules_by_prefix(session, cluster_ref, rule_prefix): for. """ cluster_config = session._call_method( - vutil, "get_object_property", cluster_ref, "configurationEx") + vim_util, "get_object_property", cluster_ref, "configurationEx") return [rule for rule in getattr(cluster_config, 'rule', []) if rule.name.startswith(rule_prefix)] @@ -345,7 +346,7 @@ def update_rule(session, cluster_ref, rule): def is_drs_enabled(session, cluster): """Check if DRS is enabled on a given cluster""" - drs_config = session._call_method(vutil, "get_object_property", cluster, + drs_config = session._call_method(vim_util, "get_object_property", cluster, "configuration.drsConfig") if drs_config and hasattr(drs_config, 'enabled'): return drs_config.enabled From 23a95cc247a48e69b9d27bd54f7c17c135a88d12 Mon Sep 17 00:00:00 2001 From: Ivaylo Mitev Date: Tue, 29 Jan 2019 07:55:46 -0800 Subject: [PATCH 78/92] vmwareapi: Handle duplicate name in OVA import We use the same functionality for OVA import as for normal image import, which automatically brings handling of duplicate name. In the process, we also fix that OVA import returned a folder path instead of a VMDK path. Change-Id: Iae731490716cc7592d2e99b7ca112fa56d40e4cc (cherry picked from commit 30e15e0ce6115ab544166be0f63caf0fb9db251d) --- nova/virt/vmwareapi/images.py | 47 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/nova/virt/vmwareapi/images.py b/nova/virt/vmwareapi/images.py index 2e9d462b64d..4bcf1ffca74 100644 --- a/nova/virt/vmwareapi/images.py +++ b/nova/virt/vmwareapi/images.py @@ -357,6 +357,19 @@ def fetch_image_stream_optimized(context, instance, session, vm_name, read_iter = IMAGE_API.download(context, image_ref) read_handle = rw_handles.ImageReadHandle(read_iter) + imported_vm_ref = _import_image(session, read_handle, vm_import_spec, + vm_name, vm_folder_ref, res_pool_ref, + file_size) + + LOG.info("Downloaded image file data %(image_ref)s", + {'image_ref': instance.image_ref}, instance=instance) + vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) + vm_util.mark_vm_as_template(session, instance, imported_vm_ref) + return vmdk.capacity_in_bytes, vmdk.path + + +def _import_image(session, read_handle, vm_import_spec, vm_name, vm_folder_ref, + res_pool_ref, file_size): # retry in order to handle conflicts in case of parallel execution # (multiple agents or previously failed import of the same image max_attempts = 3 @@ -394,14 +407,10 @@ def fetch_image_stream_optimized(context, instance, session, vm_name, pass if not imported_vm_ref: - raise vexc.VMwareDriverException("Could not import image %s within " - "%d attempts." % (vm_name, max_attempts)) - - LOG.info("Downloaded image file data %(image_ref)s", - {'image_ref': instance.image_ref}, instance=instance) - vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) - vm_util.mark_vm_as_template(session, instance, imported_vm_ref) - return vmdk.capacity_in_bytes, vmdk.path + raise vexc.VMwareDriverException("Could not import image" + " %s within %d attempts." + % (vm_name, max_attempts)) + return imported_vm_ref def _wait_for_import_task(session, vm_ref): @@ -489,27 +498,17 @@ def fetch_image_ova(context, instance, session, vm_name, ds_name, elif vmdk_name and tar_info.name.startswith(vmdk_name): # Actual file name is .XXXXXXX extracted = tar.extractfile(tar_info) - write_handle = rw_handles.VmdkWriteHandle( - session, - session._host, - session._port, - res_pool_ref, - vm_folder_ref, - vm_import_spec, - file_size) - image_transfer(extracted, write_handle) + imported_vm_ref = _import_image(session, extracted, + vm_import_spec, vm_name, + vm_folder_ref, res_pool_ref, + file_size) + LOG.info("Downloaded OVA image file %(image_ref)s", {'image_ref': instance.image_ref}, instance=instance) - imported_vm_ref = write_handle.get_imported_vm() vmdk = vm_util.get_vmdk_info(session, imported_vm_ref) vm_util.mark_vm_as_template(session, instance, imported_vm_ref) - try: - return vmdk.capacity_in_bytes, vmdk.path.parent - except AttributeError: - from oslo_vmware.objects.datastore import DatastorePath - return (vmdk.capacity_in_bytes, - DatastorePath.parse(vmdk.path).parent) + return vmdk.capacity_in_bytes, vmdk.path raise exception.ImageUnacceptable( reason=_("Extracting vmdk from OVA failed."), image_id=image_ref) From bfeda53230d95be498a83c00d04c3a59c66a7f2a Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 10 Feb 2022 11:07:42 +0100 Subject: [PATCH 79/92] Vmware: Use Rpc instead of own session The advantage of this approach is, that we now properly encapsulate each compute node instead of 'messing' remotely around in another vcenter. So caching and etc should work as expected. The downside is additional hops: Before: compute -> other-vcenter Now: compute -> messaging -> other-compute -> other-vc So more ways of going wrong. Change-Id: I37be358ff7c3bd9f786e6ce086e91ff2b2fc3861 (cherry picked from commit da4910008f86cca37a67b6f70e4e7bfe8be37697) (cherry picked from commit 7f70e86c5a3a858cde69aa1ba009e1073f6f35c8) --- .../unit/virt/vmwareapi/test_configdrive.py | 3 +- .../unit/virt/vmwareapi/test_driver_api.py | 18 +++-- nova/virt/fake.py | 5 ++ nova/virt/vmwareapi/driver.py | 71 +++++++------------ nova/virt/vmwareapi/rpc.py | 61 ++++++++++++++++ nova/virt/vmwareapi/vmops.py | 20 +++++- 6 files changed, 121 insertions(+), 57 deletions(-) create mode 100644 nova/virt/vmwareapi/rpc.py diff --git a/nova/tests/unit/virt/vmwareapi/test_configdrive.py b/nova/tests/unit/virt/vmwareapi/test_configdrive.py index 8d20c59dda8..c21b28d6fe7 100644 --- a/nova/tests/unit/virt/vmwareapi/test_configdrive.py +++ b/nova/tests/unit/virt/vmwareapi/test_configdrive.py @@ -52,7 +52,8 @@ def setUp(self, mock_register, mock_service): vmwareapi_fake.reset() stubs.set_stubs(self) self.glance = self.useFixture(nova_fixtures.GlanceFixture(self)) - self.conn = driver.VMwareVCDriver(fake.FakeVirtAPI) + virtapi = fake.FakeComputeVirtAPI(mock.MagicMock()) + self.conn = driver.VMwareVCDriver(virtapi) self.network_info = utils.get_test_network_info() self.node_name = self.conn._nodename image_ref = self.glance.auto_disk_config_enabled_image['id'] diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 9676e1e4c52..3d61427145b 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -55,6 +55,7 @@ from nova.tests.unit.virt.vmwareapi import fake as vmwareapi_fake from nova.tests.unit.virt.vmwareapi import stubs from nova.virt import driver as v_driver +from nova.virt import fake from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import driver from nova.virt.vmwareapi import ds_util @@ -186,7 +187,8 @@ def setUp(self, mock_svc, mock_register): service = self._create_service(host=HOST) mock_svc.return_value = service - self.conn = driver.VMwareVCDriver(None, False) + virtapi = fake.FakeComputeVirtAPI(mock.MagicMock()) + self.conn = driver.VMwareVCDriver(virtapi, False) self.assertFalse(service.disabled) self._set_exception_vars() self.node_name = self.conn._nodename @@ -2405,7 +2407,8 @@ def test_datastore_dc_map(self): # currently there are 2 data stores self.assertEqual(2, len(ds_util._DS_DC_MAPPING)) - def _create_live_migrate_data(self): + def _create_live_migrate_data(self, source_compute=None, + dest_compute=None): data = objects.migrate_data.VMwareLiveMigrateData() data.dest_cluster_ref = "cluster-0" @@ -2441,6 +2444,11 @@ def _create_live_migrate_data(self): } } } + migration = objects.Migration( + source_compute=source_compute, + dest_compute=dest_compute, + ) + data.migration = migration return data @@ -2480,8 +2488,7 @@ def test_pre_live_migration(self, mock_place_vm, mock_serialize_object): @mock.patch.object(vim_util, 'deserialize_object') @mock.patch.object(driver.VMwareVCDriver, '_get_volume_mappings', returns=[]) - @mock.patch.object(driver.VMwareVCDriver, '_create_dest_session') - def test_live_migration(self, create_dest_session, get_volume_mappings, + def test_live_migration(self, get_volume_mappings, deserialize_object, get_hardware_devices, relocate_vm): self._create_instance() @@ -2504,13 +2511,12 @@ def test_live_migration(self, create_dest_session, get_volume_mappings, post_method.assert_called() recover_method.assert_not_called() - @mock.patch.object(driver.VMwareVCDriver, '_create_dest_session') @mock.patch.object(driver.VMwareVCDriver, '_get_volume_mappings', returns=[]) @mock.patch.object(vmops.VMwareVMOps, 'live_migration', side_effect=test.TestingException) def test_live_migration_failure_rollback(self, mock_live_migration, - get_volume_mappings, create_dest_session): + get_volume_mappings): self._create_instance() migrate_data = self._create_live_migrate_data() diff --git a/nova/virt/fake.py b/nova/virt/fake.py index c0aa5ba13a3..f4c9cb86e91 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -704,6 +704,11 @@ def update_compute_provider_status(self, context, rp_uuid, enabled): pass +class FakeComputeVirtAPI(FakeVirtAPI): + def __init__(self, compute): + self._compute = compute + + class SmallFakeDriver(FakeDriver): # The api samples expect specific cpu memory and disk sizes. In order to # allow the FakeVirt driver to be used outside of the unit tests, provide diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 29fd21512a1..bfb0d19dffe 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -55,8 +55,8 @@ from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import host +from nova.virt.vmwareapi.rpc import VmwareRpcService from nova.virt.vmwareapi import session -from nova.virt.vmwareapi import vif as vmwarevif from nova.virt.vmwareapi import vim_util as nova_vim_util from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops @@ -168,6 +168,9 @@ def __init__(self, virtapi, scheme="https"): # Register the OpenStack extension self._register_openstack_extension() + virtapi._compute.additional_endpoints.extend([ + VmwareRpcService(self)]) + def _check_min_version(self): min_version = v_utils.convert_version_to_int(constants.MIN_VC_VERSION) next_min_ver = v_utils.convert_version_to_int( @@ -799,11 +802,11 @@ def check_can_live_migrate_destination(self, context, instance, data.cluster_name = self._cluster_name data.dest_cluster_ref = vim_util.get_moref_value(self._cluster_ref) data.datastore_regex = CONF.vmware.datastore_regex - client_factory = self._session.client.factory + client_factory = self._session.vim.client.factory - service = vm_util.create_service_locator(client_factory, url, - self._vcenter_uuid, - vm_util.create_service_locator_name_password( + service = vm_util.create_service_locator(client_factory, + url, self._vcenter_uuid, + vm_util.create_service_locator_name_password(client_factory, username=CONF.vmware.host_username, password=CONF.vmware.host_password)) @@ -822,7 +825,7 @@ def check_can_live_migrate_source(self, context, instance, dest_check_data, block_device_info=None): """Check if it is possible to execute live migration.""" - client_factory = self._session.client.factory + client_factory = self._session.vim.client.factory defaults = dest_check_data.relocate_defaults service_locator = nova_vim_util.deserialize_object(client_factory, defaults["service"], @@ -934,14 +937,13 @@ def live_migration(self, context, instance, dest, post_method(context, instance, dest, block_migration, migrate_data) return - # We require the target-datastore for all volume-attachment - required_volume_attributes = ["datastore_ref"] try: - if migrate_data.is_same_vcenter: - dest_session = self._session - else: + # We require the target-datastore for all volume-attachment + required_volume_attributes = ["datastore_ref"] + + target = self._vmops.api_for_migration(migrate_data.migration) + if not migrate_data.is_same_vcenter: # For the shadow vm fixup after migration - dest_session = self._create_dest_session(migrate_data) required_volume_attributes.append('volume') # Validate that we have all necessary information for the @@ -950,7 +952,16 @@ def live_migration(self, context, instance, dest, # are created prior the live-migration volumes = self._get_checked_volumes(context, instance, required_volume_attributes) - self._set_vif_infos(migrate_data, dest_session) + if 'vifs' not in migrate_data or not migrate_data.vifs: + migrate_data.vif_infos = [] + else: + dest_network_info = [ + vif.get_dest_vif() + for vif in migrate_data.vifs + ] + vif_model = None # Doesn't matter as we won't change the type + migrate_data.vif_infos = target.get_vif_info(context, + vif_model=vif_model, network_info=dest_network_info) self._vmops.live_migration(instance, migrate_data, volumes) LOG.info("Migration operation completed", instance=instance) post_method(context, instance, dest, block_migration, migrate_data) @@ -999,40 +1010,6 @@ def _get_volume_mappings(self, context, instance): volume_infos.append(data) return self._volumeops.map_volumes_to_devices(instance, volume_infos) - def _set_vif_infos(self, migrate_data, session): - if 'vifs' not in migrate_data or not migrate_data.vifs: - migrate_data.vif_infos = [] - return - - cluster_ref = vim_util.get_moref(migrate_data.dest_cluster_ref, - "ClusterComputeResource") - dest_network_info = [ - vif.get_dest_vif() - for vif in migrate_data.vifs - ] - - vif_model = None # We are not reading that value - migrate_data.vif_infos = vmwarevif.get_vif_info(session, - cluster_ref, utils.is_neutron(), vif_model, dest_network_info) - - def _create_dest_session(self, migrate_data): - defaults = migrate_data.relocate_defaults - client_factory = self._session.client.factory - service_locator = nova_vim_util.deserialize_object(client_factory, - defaults["service"], - "ServiceLocator") - dest_url = urllib.parse.urlparse(service_locator.url) - - dest_session = session.VMwareAPISession( - host_ip=dest_url.hostname, - host_port=dest_url.port, - username=service_locator.credential.username, - password=service_locator.credential.password, - # TODO(fwiesel): SSL Settings - ) - - return dest_session - def rollback_live_migration_at_destination(self, context, instance, network_info, block_device_info, diff --git a/nova/virt/vmwareapi/rpc.py b/nova/virt/vmwareapi/rpc.py new file mode 100644 index 00000000000..368b865c578 --- /dev/null +++ b/nova/virt/vmwareapi/rpc.py @@ -0,0 +1,61 @@ +# Copyright (c) 2021 SAP SE +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +RPC server and client for communicating with other Vmareapi drivers directly. +This is the gateway which allows us gathering VMWare related information from +other hosts and perform cross vCenter operations. +""" +import oslo_messaging as messaging + +from nova.compute import rpcapi +from nova import profiler +from nova import rpc + + +_NAMESPACE = 'vmwareapi' +_VERSION = '5.0' + + +@profiler.trace_cls("rpc") +class VmwareRpcApi(object): + def __init__(self, server): + target = messaging.Target(topic=rpcapi.RPC_TOPIC, version=_VERSION, + namespace=_NAMESPACE) + self._router = rpc.ClientRouter(rpc.get_client(target)) + self._server = server + + def _call(self, fname, ctxt, version=None, **kwargs): + version = version or _VERSION + cctxt = self._router.client(ctxt).prepare(server=self._server, + version=version) + return cctxt.call(ctxt, fname, **kwargs) + + def get_vif_info(self, ctxt, vif_model=None, network_info=None): + return self._call('get_vif_info', ctxt, vif_model=vif_model, + network_info=network_info,) + + +@profiler.trace_cls("rpc") +class VmwareRpcService(object): + target = messaging.Target(version=_VERSION, namespace=_NAMESPACE) + + def __init__(self, driver): + self._driver = driver + + def get_vif_info(self, ctxt, vif_model=None, network_info=None): + return self._driver._vmops.get_vif_info(ctxt, vif_model=vif_model, + network_info=network_info, + ) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index d8f1fc0485a..ff73c2f625f 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -61,6 +61,7 @@ from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import imagecache from nova.virt.vmwareapi import images +from nova.virt.vmwareapi.rpc import VmwareRpcApi from nova.virt.vmwareapi import vif as vmwarevif from nova.virt.vmwareapi import vim_util from nova.virt.vmwareapi import vm_util @@ -1778,9 +1779,9 @@ def migrate_disk_and_power_off(self, context, instance, dest, flavor): # Checks if the migration needs a disk resize down. if (not boot_from_volume and ( - flavor.root_gb < instance.flavor.root_gb or - (flavor.root_gb != 0 and - flavor.root_gb < vmdk.capacity_in_bytes / units.Gi))): + flavor.root_gb < instance.flavor.root_gb or + (flavor.root_gb != 0 and + flavor.root_gb < vmdk.capacity_in_bytes / units.Gi))): reason = _("Unable to shrink disk.") raise exception.InstanceFaultRollback( exception.ResizeError(reason=reason)) @@ -1796,6 +1797,19 @@ def migrate_disk_and_power_off(self, context, instance, dest, flavor): step=1, total_steps=RESIZE_TOTAL_STEPS) + def get_vif_info(self, ctxt, vif_model=None, network_info=None): + vif_info = vmwarevif.get_vif_info(self._session, + self._cluster, + vif_model, + network_info) + return vif_info + + def api_for_migration(self, migration): + if migration.dest_compute == migration.source_compute: + return self + + return VmwareRpcApi(migration.dest_compute) + def confirm_migration(self, migration, instance, network_info): """Confirms a resize, destroying the source VM.""" vm_ref = vm_util.get_vm_ref(self._session, instance) From ff76e9b5fbc5e7f170a76f572ecdfc02c3f2c729 Mon Sep 17 00:00:00 2001 From: Andrew Karpow Date: Tue, 18 Dec 2018 18:26:23 +0100 Subject: [PATCH 80/92] [vmware] don't report empty inventory (fixed keyerror) (e.g. when all hosts in maintainence) (cherry picked from commit 4a32b79f72307b23759d32f2dab0d483ae4ed292) (cherry picked from commit fa208aee5de3ba1d2d99b0191ecac5482ae05ff1) (cherry picked from commit 348fa86939c43a6117bad388d7d3bba3b4e1ccb1) --- nova/virt/vmwareapi/driver.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index bfb0d19dffe..d952f8a522a 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -471,23 +471,32 @@ def update_provider_tree(self, provider_tree, nodename, allocations=None): reserved_disk_gb = compute_utils.convert_mb_to_ceil_gb( CONF.reserved_host_disk_mb) result = { - orc.VCPU: { + orc.DISK_GB: { + 'total': total_disk_capacity // units.Gi, + 'reserved': reserved_disk_gb, + 'min_unit': 1, + 'max_unit': max_free_space // units.Gi, + 'step_size': 1, + } + } + if stats['cpu']['max_vcpus_per_host'] > 0: + result.update({orc.VCPU: { 'total': stats['cpu']['vcpus'], 'reserved': CONF.reserved_host_cpus, 'min_unit': 1, 'max_unit': stats['cpu']['max_vcpus_per_host'], 'step_size': 1, 'allocation_ratio': ratios[orc.VCPU], - }, - orc.MEMORY_MB: { + }}) + if stats['mem']['max_mem_mb_per_host'] > 0: + result.update({orc.MEMORY_MB: { 'total': stats['mem']['total'], 'reserved': CONF.reserved_host_memory_mb, 'min_unit': 1, 'max_unit': stats['mem']['max_mem_mb_per_host'], 'step_size': 1, 'allocation_ratio': ratios[orc.MEMORY_MB], - }, - } + }}) # If a sharing DISK_GB provider exists in the provider tree, then our # storage is shared, and we should not report the DISK_GB inventory in From 763e0d735da8fcfa77b4d91d25f0dcf2859b8895 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 27 Sep 2019 14:15:24 +0200 Subject: [PATCH 81/92] [vmware] Add configurable reservations per hostgroup This can be used to have different CPU/memory reservations for hosts depending on the hostgroup they're in. We can also set defaults in percent with this instead of only static values as supported by CONF.reserved_host_memory_mb and CONF.reserved_host_cpus. We also have to change the `os-hypervisors/details` call to adhere to the reservations so both Placement and Nova report the same values. It's important to us, because we get or used to get the quota capacity from there. I opted for changing the resources (for that single call, hence copy.deepcopy) instead of subtracting the reserved values from the totals afterwards, because if an attribute is unset on the `ComputeNode` object, we cannot access it - not even with `getattr`. It just raises a `NotImplementedError` because it cannot fetch the attribute from somewhere (e.g. the DB). But since we need to subtract from the total, we would have to access the attribute. Catching the `NotImplementedError` didn't seem like a good option to me. With subtracting the reserved resources, we have to make sure we do not fall below zero - which could happen for Ironic nodes, where the memory is set to 0. With us disallowing this, one of nova's assumptions failed - that we don't collapse negative values to 0. But since that test only worked with reservations, it couldn't detect that we still allow negative values. We changed the test to test if instance memory can still reduce the free memory to below zero. NOTE: Setting these settings too differently per hostgroup for the same cluster might result in placement API getting confused and scheduling being impossible, as the `max_unit` is still set to the maximum per cluster. In case of our HANA HVs placement would therefore always consider the HANA HVs max unit. That should be no problem as smaller VMs should always fit. (cherry picked from commit 52504365ae43018a4aa70f8a11d7271aea18e62b) (cherry picked from commit 28f4eb88a6170f27ca3d17dfd17c57f3cef3d227) Change-Id: Ie4c9314bd19ec1565f3a380f74054fb1f84a0dcc Incorporated-Change-Id: I191aedb0e3bba7698825771089cf134f320368ec Incorporated-Change-Id: I1c92b193bf867db3a749b09bac57bde500c39e19 (cherry picked from commit 58796bc34def8dcacc34532ee886acef4bebdf1e) --- nova/compute/resource_tracker.py | 20 ++- nova/conf/vmware.py | 25 +++ .../unit/compute/test_resource_tracker.py | 87 ++++++++-- .../unit/virt/vmwareapi/test_driver_api.py | 15 +- .../tests/unit/virt/vmwareapi/test_vm_util.py | 154 ++++++++++++++++-- nova/virt/vmwareapi/driver.py | 12 +- nova/virt/vmwareapi/host.py | 2 + nova/virt/vmwareapi/vm_util.py | 106 +++++++++++- 8 files changed, 388 insertions(+), 33 deletions(-) diff --git a/nova/compute/resource_tracker.py b/nova/compute/resource_tracker.py index 3570d04f3ec..698049fe389 100644 --- a/nova/compute/resource_tracker.py +++ b/nova/compute/resource_tracker.py @@ -847,6 +847,20 @@ def _copy_resources(self, compute_node, resources, initial=False): if conf_alloc_ratio not in (0.0, None): setattr(compute_node, attr, conf_alloc_ratio) + # apply the reservations so that limes's quota can see them + resources = copy.deepcopy(resources) + resources['vcpus'] = resources.get('vcpus', 0) - \ + resources.get('vcpus_reserved', + CONF.reserved_host_cpus) + resources['vcpus'] = max(resources['vcpus'], 0) + resources['memory_mb'] = resources.get('memory_mb', 0) - \ + resources.get('memory_mb_reserved', + CONF.reserved_host_memory_mb) + resources['memory_mb'] = max(resources['memory_mb'], 0) + resources['local_gb'] = resources.get('local_gb', 0) - \ + CONF.reserved_host_disk_mb / 1024 + resources['local_gb'] = max(resources['local_gb'], 0) + # now copy rest to compute_node compute_node.update_from_virt_driver(resources) @@ -1665,9 +1679,9 @@ def _update_usage_from_instances(self, context, instances, nodename): cn = self.compute_nodes[nodename] # set some initial values, reserve room for host/hypervisor: - cn.local_gb_used = CONF.reserved_host_disk_mb / 1024 - cn.memory_mb_used = CONF.reserved_host_memory_mb - cn.vcpus_used = CONF.reserved_host_cpus + cn.local_gb_used = 0 + cn.memory_mb_used = 0 + cn.vcpus_used = 0 cn.free_ram_mb = (cn.memory_mb - cn.memory_mb_used) cn.free_disk_gb = (cn.local_gb - cn.local_gb_used) cn.current_workload = 0 diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index 3b505a89618..eab5225ffa4 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -115,6 +115,31 @@ For big VMs as determined by the `bigvm_mb` setting, this setting is not used. Big VMs always reserve all their memory. +"""), + cfg.StrOpt('hostgroup_reservations_json_file', + help=""" +Specifies the path to a JSON file containing vcpus and memory reservations per +hostgroup. HVs found to be in a hostgroup specified in this dict will report +the amount of vcpus/memory as reserved to the placement API. This enables us to +specify different reservations for HANA-used HVs and for "normal" HVs. The +special key `__default__` holds values applied if there is no specification for +an HV's hostgroup. It defaults to 0. + +There are two ways to specify a hostgroup's values: as static numbers and as +percent. Static numbers take precedence over percent if both are present. +Additionally, values are inherited from the `__default__` group if not present +for a group. To prohibit inheriting a value, explicitly set it to `null`. + +Example configuration could be: + {"__default__": {"memory_percent": 10, "vcpus": 10}, + "hana_hosts": {"memory_mb": 16384, "vcpus": 5}, + "normal_hosts": {"vcpus": null, "vcpus_percent": 2}} + +If there are cluster-wide reservations defined via `reserved_host_cpus` or +`reserved_host_memory_mb`, they are added to the sum of the hostgroup-defined +ones. + +NOTE: Percentage values will be rounded down to the next full vcpu/MB. """), ] diff --git a/nova/tests/unit/compute/test_resource_tracker.py b/nova/tests/unit/compute/test_resource_tracker.py index 3af6eb2be8b..beefce9764f 100644 --- a/nova/tests/unit/compute/test_resource_tracker.py +++ b/nova/tests/unit/compute/test_resource_tracker.py @@ -766,15 +766,15 @@ def test_no_instances_no_migrations_reserved_disk_ram_and_cpu( get_cn_mock.assert_called_once_with(mock.ANY, uuids.cn1) expected_resources = copy.deepcopy(_COMPUTE_NODE_FIXTURES[0]) vals = { - 'free_disk_gb': 5, # 6GB avail - 1 GB reserved - 'local_gb': 6, - 'free_ram_mb': 0, # 512MB avail - 512MB reserved - 'memory_mb_used': 512, # 0MB used + 512MB reserved - 'vcpus_used': 1, - 'local_gb_used': 1, # 0GB used + 1 GB reserved - 'memory_mb': 512, + 'free_disk_gb': 5, # local_gb + 'local_gb': 5, # 6GB avail - 1 GB reserved + 'free_ram_mb': 0, # memory_mb + 'memory_mb_used': 0, # 0MB used + 'vcpus_used': 0, + 'local_gb_used': 0, # 0GB used + 'memory_mb': 0, # 512MB avail - 512MB reserved 'current_workload': 0, - 'vcpus': 4, + 'vcpus': 3, # 4 CPUs - 1 CPU reserved 'running_vms': 0 } _update_compute_node(expected_resources, **vals) @@ -783,6 +783,52 @@ def test_no_instances_no_migrations_reserved_disk_ram_and_cpu( actual_resources)) update_mock.assert_called_once() + @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', + return_value=objects.InstancePCIRequests(requests=[])) + @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', + return_value=objects.PciDeviceList()) + @mock.patch('nova.objects.ComputeNode.get_by_uuid') + @mock.patch('nova.objects.MigrationList.get_in_progress_and_error') + @mock.patch('nova.objects.InstanceList.get_by_host_and_node') + def test_no_instances_no_migrations_reserved_disk_ram_and_cpu_driver( + self, get_mock, migr_mock, get_cn_mock, pci_mock, + instance_pci_mock): + # Setup with config reservations and driver-returned reservations. + # Driver-returned reservations should include the config ones and thus + # overwrite the config ones. + self.flags(reserved_host_disk_mb=1024, + reserved_host_memory_mb=512, + reserved_host_cpus=1) + virt_resources = copy.deepcopy(_VIRT_DRIVER_AVAIL_RESOURCES) + virt_resources.update(vcpus_reserved=2, + memory_mb_reserved=128) + self._setup_rt(virt_resources=virt_resources) + + get_mock.return_value = [] + migr_mock.return_value = [] + get_cn_mock.return_value = _COMPUTE_NODE_FIXTURES[0] + + update_mock = self._update_available_resources() + + get_cn_mock.assert_called_once_with(mock.ANY, uuids.cn1) + expected_resources = copy.deepcopy(_COMPUTE_NODE_FIXTURES[0]) + vals = { + 'free_disk_gb': 5, # local_gb + 'local_gb': 5, # 6GB avail - 1 GB reserved + 'free_ram_mb': 384, # memory_mb + 'memory_mb_used': 0, # 0MB used + 'vcpus_used': 0, + 'local_gb_used': 0, # 0GB used + 'memory_mb': 384, # 512MB avail - 128MB reserved + 'current_workload': 0, + 'vcpus': 2, # 4 vcpus - 2 reserved + 'running_vms': 0 + } + _update_compute_node(expected_resources, **vals) + actual_resources = update_mock.call_args[0][1] + self.assertTrue(obj_base.obj_equal_prims(expected_resources, + actual_resources)) + @mock.patch('nova.compute.utils.is_volume_backed_instance') @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', return_value=objects.InstancePCIRequests(requests=[])) @@ -4029,18 +4075,37 @@ def test_update_usage_from_instances_goes_negative(self): # NOTE(danms): The resource tracker _should_ report negative resources # for things like free_ram_mb if overcommit is being used. This test # ensures that we don't collapse negative values to zero. + # NOTE(jkulik): We've switched how this works at SAP; because we don't + # want limes to see the memory of a compute-node if it's actually + # reserved (as limes didn't care for free/used at the time). Therefore, + # we actually collapse reserved memory to 0 for reserved resources, but + # we don't do it for instance resources. self.flags(reserved_host_memory_mb=2048) self.flags(reserved_host_disk_mb=(11 * 1024)) cn = objects.ComputeNode(memory_mb=1024, local_gb=10) + resources = {'hypervisor_hostname': 'foo', 'memory_mb': 1024, + 'local_gb': 10} + self.rt._copy_resources(cn, resources) self.rt.compute_nodes['foo'] = cn @mock.patch.object(self.rt, '_update_usage_from_instance') - def test(uufi): - self.rt._update_usage_from_instances('ctxt', [], 'foo') + @mock.patch('nova.objects.Service.get_minimum_version', + return_value=22) + def test(version_mock, uufi): + def _update_usage(*args, **kwargs): + # simulate an instance with 512 MB memory and 1 GB disk using + # resources + cn.memory_mb_used += 512 + cn.local_gb_used += 1 + cn.free_ram_mb = cn.memory_mb - cn.memory_mb_used + cn.free_disk_gb = cn.local_gb - cn.local_gb_used + uufi.side_effect = _update_usage + self.rt._update_usage_from_instances('ctxt', [self.instance], + 'foo') test() - self.assertEqual(-1024, cn.free_ram_mb) + self.assertEqual(-512, cn.free_ram_mb) self.assertEqual(-1, cn.free_disk_gb) def test_delete_allocation_for_evacuated_instance(self): diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 3d61427145b..178af4f3838 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -2161,14 +2161,22 @@ def test_datastore_regex_configured_vcstate(self, mock_get_ds_ref): def test_get_available_resource(self): stats = self.conn.get_available_resource(self.node_name) self.assertEqual(32, stats['vcpus']) + self.assertEqual(CONF.reserved_host_cpus, stats['vcpus_reserved']) self.assertEqual(1024, stats['local_gb']) self.assertEqual(1024 - 500, stats['local_gb_used']) self.assertEqual(2048, stats['memory_mb']) self.assertEqual(1000, stats['memory_mb_used']) + self.assertEqual(CONF.reserved_host_memory_mb, + stats['memory_mb_reserved']) self.assertEqual('VMware vCenter Server', stats['hypervisor_type']) self.assertEqual(5005000, stats['hypervisor_version']) self.assertEqual(self.node_name, stats['hypervisor_hostname']) - self.assertIsNone(stats['cpu_info']) + # TODO(xxxx): check cpu-info + # cpu_info = {'features': ['aes', 'avx'], + # 'vendor': 'Intel', + # 'topology': {'threads': 16, 'sockets': 2, 'cores': 8}, + # 'model': 'Intel(R) Xeon(R) CPU E5-4650 v4 @ 2.20GHz'} + # self.assertEqual(jsonutils.loads(stats['cpu_info']), cpu_info) self.assertEqual( [("i686", "vmware", "hvm"), ("x86_64", "vmware", "hvm")], stats['supported_instances']) @@ -2619,8 +2627,9 @@ def test_host_state_service_disabled(self, mock_service): service = self._create_service(disabled=False, host='fake-mini') mock_service.return_value = service - fake_stats = {'cpu': {'vcpus': 4}, - 'mem': {'total': '8194', 'free': '2048'}} + fake_stats = {'cpu': {'vcpus': 4, 'reserved_vcpus': 0}, + 'mem': {'total': '8194', 'free': '2048', + 'reserved_memory_mb': 0}} with test.nested( mock.patch.object(vm_util, 'get_stats_from_cluster', side_effect=[vexc.VimConnectionException('fake'), diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 130e819c9c2..61bf200b099 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -13,8 +13,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +import builtins import collections +import io from unittest import mock from oslo_service import fixture as oslo_svc_fixture @@ -61,7 +62,8 @@ def setUp(self): def _test_get_stats_from_cluster(self, connection_state="connected", maintenance_mode=False, failover_policy=None, - failover_hosts_filtered=0): + failover_hosts_filtered=0, + expected_stats=None): ManagedObjectRefs = [fake.ManagedObjectReference("HostSystem", "host1"), fake.ManagedObjectReference("HostSystem", @@ -107,8 +109,10 @@ def _test_get_stats_from_cluster(self, connection_state="connected", val=quickstats_2)] fake_objects = { - 'host1': fake.ObjectContent("prop_list_host1", prop_list_host_1), - 'host2': fake.ObjectContent("prop_list_host1", prop_list_host_2) + 'host1': + fake.ObjectContent(ManagedObjectRefs[0], prop_list_host_1), + 'host2': + fake.ObjectContent(ManagedObjectRefs[1], prop_list_host_2) } def fake_call_method(*args): @@ -130,12 +134,15 @@ def fake_call_method(*args): else: num_hosts = 1 num_hosts -= failover_hosts_filtered - expected_stats = {'cpu': {'vcpus': num_hosts * 16, - 'max_vcpus_per_host': 16}, - 'mem': {'total': num_hosts * 4096, - 'free': num_hosts * 4096 - - num_hosts * 512, - 'max_mem_mb_per_host': 4096}} + if expected_stats is None: + expected_stats = {'cpu': {'vcpus': num_hosts * 16, + 'max_vcpus_per_host': 16, + 'reserved_vcpus': 0}, + 'mem': {'total': num_hosts * 4096, + 'free': num_hosts * 4096 - + num_hosts * 512, + 'max_mem_mb_per_host': 4096, + 'reserved_memory_mb': 0}} self.assertEqual(expected_stats, result) def test_get_stats_from_cluster_hosts_connected_and_active(self): @@ -177,11 +184,138 @@ def test_get_stats_from_cluster_failover_resources_policy(self): self._test_get_stats_from_cluster(failover_policy=policy, failover_hosts_filtered=0) + @mock.patch('nova.virt.vmwareapi.vm_util._get_host_reservations_map') + def test_get_stats_from_cluster_reservations(self, mock_map): + CONF.set_override('hostgroup_reservations_json_file', '/some/path', + 'vmware') + mock_map.return_value = { + "host1": { + 'memory_mb': 256, + 'vcpus': 4}, + "host2": { + 'memory_mb': 512, + 'vcpus': 2}} + + expected = {'cpu': {'vcpus': 2 * 16, + 'max_vcpus_per_host': 14, # by host2 counts + 'reserved_vcpus': 4 + 2}, # both hosts + 'mem': {'total': 2 * 4096, + 'free': 2 * 4096 - + 2 * 512, + 'max_mem_mb_per_host': 4096 - 256, # host1 + 'reserved_memory_mb': 512 + 256}} # both + self._test_get_stats_from_cluster(expected_stats=expected) + + def test_get_host_reservations_empty(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {} + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 0, 'memory_mb': 0} + self.assertEqual(expected, result) + + mapping = {'host2': {'vcpus': 5}} + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 0, 'memory_mb': 0} + self.assertEqual(expected, result) + + def test_get_host_reservations_with_default(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}} + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 2, 'memory_mb': 96} + self.assertEqual(expected, result) + + def test_get_host_reservations_with_some_default(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}, + 'host1': {'vcpus': 1}} + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 1, 'memory_mb': 96} + self.assertEqual(expected, result) + + mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}, + 'host1': {'memory_mb': 2048}} + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 2, 'memory_mb': 2048} + self.assertEqual(expected, result) + + def test_get_host_reservations_priority(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus_percent': 10, 'memory_percent': 50}, + 'host1': {'vcpus': 1}} + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 1, 'memory_mb': 2048} + self.assertEqual(expected, result) + + mapping['host1']['memory_mb'] = 96 + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 1, 'memory_mb': 96} + self.assertEqual(expected, result) + + def test_get_host_reservations_override(self): + host = fake.ManagedObjectReference('HostSystem', 'host1') + mapping = {'__default__': {'vcpus': 5}, + 'host1': {'vcpus_percent': 10}} + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 5, 'memory_mb': 0} + self.assertEqual(expected, result) + + mapping['host1']['vcpus'] = None + result = vm_util._get_host_reservations(mapping, host, 32, 4096) + expected = {'vcpus': 3, 'memory_mb': 0} + self.assertEqual(expected, result) + def test_get_host_ref_no_hosts_in_cluster(self): self.assertRaises(exception.NoValidHost, vm_util.get_host_ref, fake.FakeObjectRetrievalSession(""), 'fake_cluster') + def test_get_host_reservations_map_no_action(self): + self.assertEqual({}, vm_util._get_host_reservations_map()) + + @mock.patch.object(builtins, 'open') + def test_get_host_reservations_map_default_only(self, mock_open): + CONF.set_override('hostgroup_reservations_json_file', '/some/path', + 'vmware') + reservations = '{"__default__": {"vcpus": 2, "memory_mb": 512}, ' \ + '"foo": {"vcpus": 7}}' + response = io.BytesIO(reservations.encode('utf-8')) + mock_open.return_value = response + expected = {'__default__': {'vcpus': 2, 'memory_mb': 512}} + self.assertEqual(expected, vm_util._get_host_reservations_map()) + + @mock.patch.object(builtins, 'open') + def test_get_host_reservations_map(self, mock_open): + CONF.set_override('hostgroup_reservations_json_file', '/some/path', + 'vmware') + reservations = '{"__default__": {"vcpus": 2, "memory_mb": 512}, ' \ + '"foo": {"vcpus": 7, "vcpus_percent": 10, ' \ + '"memory_mb": 1024}}' + response = io.BytesIO(reservations.encode('utf-8')) + mock_open.return_value = response + + class Group(object): + def __init__(self, name, host=None): + self.name = name + if host is not None: + self.host = host + + host1 = fake.ManagedObjectReference('HostSystem', 'host1') + host2 = fake.ManagedObjectReference('HostSystem', 'host2') + host3 = fake.ManagedObjectReference('HostSystem', 'host3') + + groups = [ + Group('vmgroup'), + Group('hostgroup1', host=[host1]), + Group('foo', host=[host2, host3])] + + expected = {'__default__': {'vcpus': 2, 'memory_mb': 512}, + 'host2': {'vcpus': 7, 'vcpus_percent': 10, + 'memory_mb': 1024}, + 'host3': {'vcpus': 7, 'vcpus_percent': 10, + 'memory_mb': 1024}} + self.assertEqual(expected, vm_util._get_host_reservations_map(groups)) + def test_get_resize_spec(self): vcpus = 2 memory_mb = 2048 diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index d952f8a522a..fdc3213f67b 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -378,6 +378,10 @@ def _get_available_resources(self, host_stats): 'memory_mb_used': host_stats['host_memory_total'] - host_stats['host_memory_free'], 'local_gb_used': host_stats['disk_used'], + 'vcpus_reserved': CONF.reserved_host_cpus + + host_stats['vcpus_reserved'], + 'memory_mb_reserved': CONF.reserved_host_memory_mb + + host_stats['host_memory_reserved'], 'hypervisor_type': host_stats['hypervisor_type'], 'hypervisor_version': host_stats['hypervisor_version'], 'hypervisor_hostname': host_stats['hypervisor_hostname'], @@ -480,18 +484,22 @@ def update_provider_tree(self, provider_tree, nodename, allocations=None): } } if stats['cpu']['max_vcpus_per_host'] > 0: + reserved_vcpus = stats['cpu'].get('reserved_vcpus', 0) + reserved_vcpus += CONF.reserved_host_cpus result.update({orc.VCPU: { 'total': stats['cpu']['vcpus'], - 'reserved': CONF.reserved_host_cpus, + 'reserved': reserved_vcpus, 'min_unit': 1, 'max_unit': stats['cpu']['max_vcpus_per_host'], 'step_size': 1, 'allocation_ratio': ratios[orc.VCPU], }}) if stats['mem']['max_mem_mb_per_host'] > 0: + reserved_memory_mb = stats['mem'].get('reserved_memory_mb', 0) + reserved_memory_mb += CONF.reserved_host_memory_mb result.update({orc.MEMORY_MB: { 'total': stats['mem']['total'], - 'reserved': CONF.reserved_host_memory_mb, + 'reserved': reserved_memory_mb, 'min_unit': 1, 'max_unit': stats['mem']['max_mem_mb_per_host'], 'step_size': 1, diff --git a/nova/virt/vmwareapi/host.py b/nova/virt/vmwareapi/host.py index b269bb878c8..bbc88700c33 100644 --- a/nova/virt/vmwareapi/host.py +++ b/nova/virt/vmwareapi/host.py @@ -103,11 +103,13 @@ def update_status(self): return data data["vcpus"] = stats['cpu']['vcpus'] + data["vcpus_reserved"] = stats['cpu']['reserved_vcpus'] data["disk_total"] = capacity // units.Gi data["disk_available"] = freespace // units.Gi data["disk_used"] = data["disk_total"] - data["disk_available"] data["host_memory_total"] = stats['mem']['total'] data["host_memory_free"] = stats['mem']['free'] + data['host_memory_reserved'] = stats['mem']['reserved_memory_mb'] data["hypervisor_type"] = self._hypervisor_type data["hypervisor_version"] = self._hypervisor_version data["hypervisor_hostname"] = self._host_name diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index c718ce887a8..8159c70e5d5 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -27,6 +27,7 @@ from urllib.parse import urlparse from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_service import loopingcall from oslo_utils import excutils from oslo_utils import units @@ -70,6 +71,8 @@ # unnecessary communication with the backend. _VM_REFS_CACHE = {} +_HOST_RESERVATIONS_DEFAULT_KEY = '__default__' + class Limits(object): @@ -1338,16 +1341,99 @@ def get_vm_state(session, instance): return constants.POWER_STATES[vm_state] +def _get_host_reservations(host_reservations_map, host_moref, host_vcpus, + host_memory_mb): + """Compute the number of vcpus and memory in MB reserved for the host from + the configured reservations or the default. + + For every host, both vcpus and memory can be given as static number or in + percent, with static number taking priority. + """ + reservations = { + 'vcpus': 0, + 'memory_mb': 0 + } + + default_key = _HOST_RESERVATIONS_DEFAULT_KEY + host_reservations = host_reservations_map.get(default_key, {}) + group_reservations = host_reservations_map.get(host_moref.value, {}) + for key in ['vcpus', 'vcpus_percent', 'memory_mb', 'memory_percent']: + if key in group_reservations: + host_reservations[key] = group_reservations[key] + if not host_reservations: + return reservations + + # compute the number of vcpus + if host_reservations.get('vcpus') is not None: + vcpus = max(0, host_reservations['vcpus']) + reservations['vcpus'] = min(host_vcpus, vcpus) + elif host_reservations.get('vcpus_percent') is not None: + percent = max(0, min(100, host_reservations['vcpus_percent'])) + # This will round down. + reservations['vcpus'] = host_vcpus * percent // 100 + + # compute the memory in MB + if host_reservations.get('memory_mb') is not None: + memory_mb = max(0, host_reservations['memory_mb']) + reservations['memory_mb'] = min(host_memory_mb, memory_mb) + elif host_reservations.get('memory_percent') is not None: + percent = max(0, min(100, host_reservations['memory_percent'])) + # This will round down. + reservations['memory_mb'] = host_memory_mb * percent // 100 + + return reservations + + +def _get_host_reservations_map(groups=None): + """return a mapping from hosts to reservations + + Reservations are read from CONF.vmware.hostgroup_reservations_json_file, + which is mapping from hostgroup to reservation. We thus look up the + hostgroups we find in the mapping and create a mapping from each host in + that hostgroup to the reservation. + """ + if groups is None: + groups = [] + + if not CONF.vmware.hostgroup_reservations_json_file: + return {} + + with open(CONF.vmware.hostgroup_reservations_json_file, 'rb') as f: + reservations = jsonutils.load(f) + + hrm = {} + + default = reservations.get(_HOST_RESERVATIONS_DEFAULT_KEY) + if default is not None: + hrm[_HOST_RESERVATIONS_DEFAULT_KEY] = default + + for group in groups: + if not hasattr(group, 'host'): + continue + + if group.name not in reservations: + continue + + for host_moref in group.host: + reservation = reservations[group.name] + hrm[host_moref.value] = reservation + return hrm + + def get_stats_from_cluster(session, cluster): """Get the aggregate resource stats of a cluster.""" vcpus = 0 max_vcpus_per_host = 0 + reserved_vcpus = 0 used_mem_mb = 0 total_mem_mb = 0 max_mem_mb_per_host = 0 + reserved_memory_mb = 0 # Get the Host and Resource Pool Managed Object Refs props = ["host", "resourcePool", "configuration.dasConfig.admissionControlPolicy"] + if CONF.vmware.hostgroup_reservations_json_file: + props.append("configurationEx") prop_dict = session._call_method(vutil, "get_object_properties_dict", cluster, @@ -1359,6 +1445,9 @@ def get_stats_from_cluster(session, cluster): if policy and hasattr(policy, 'failoverHosts'): failover_hosts = set(h.value for h in policy.failoverHosts) + group_ret = getattr(prop_dict.get('configurationEx'), 'group', None) + host_reservations_map = _get_host_reservations_map(group_ret) + host_ret = prop_dict.get('host') if host_ret: host_mors = [m for m in host_ret.ManagedObjectReference @@ -1381,16 +1470,25 @@ def get_stats_from_cluster(session, cluster): # The overcommitment ratio is factored in by the scheduler threads = hardware_summary.numCpuThreads vcpus += threads - max_vcpus_per_host = max(max_vcpus_per_host, threads) used_mem_mb += stats_summary.overallMemoryUsage mem_mb = hardware_summary.memorySize // units.Mi total_mem_mb += mem_mb - max_mem_mb_per_host = max(max_mem_mb_per_host, mem_mb) + reserved = _get_host_reservations( + host_reservations_map, obj.obj, + threads, mem_mb) + reserved_vcpus += reserved['vcpus'] + reserved_memory_mb += reserved['memory_mb'] + max_vcpus_per_host = max(max_vcpus_per_host, + threads - reserved['vcpus']) + max_mem_mb_per_host = max(max_mem_mb_per_host, + mem_mb - reserved['memory_mb']) stats = {'cpu': {'vcpus': vcpus, - 'max_vcpus_per_host': max_vcpus_per_host}, + 'max_vcpus_per_host': max_vcpus_per_host, + 'reserved_vcpus': reserved_vcpus}, 'mem': {'total': total_mem_mb, 'free': total_mem_mb - used_mem_mb, - 'max_mem_mb_per_host': max_mem_mb_per_host}} + 'max_mem_mb_per_host': max_mem_mb_per_host, + 'reserved_memory_mb': reserved_memory_mb}} return stats From 2eb51bb5965d785380a794a6317c2be60206184a Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 28 Oct 2019 15:01:11 +0100 Subject: [PATCH 82/92] [vmware] Optionally spawn VMs on prepared empty host For this to work, the two settings `spawn_on_empty_host_vm_group` and `spawn_somewhere_else_vm_group` need to be defined. The first one specifies a DRS VM-group name, that should have a rule to force the VM to spawn on the prepared host. The second setting specifies a DRS VM-group name where all other VMs should be placed in. This group should have a rule that prohibits all members from spawning/running on the prepared host's hostgroup. (cherry picked from commit 5c80f53b0cc6d1edc798a97e56ee6d51be011bc0) (cherry picked from commit c6611796c44d8f75acbd6e439a842bb7b2f1a2b0) Fix unit tests for spawn VMs on prepared empty host This is regarding the change in _get_server_groups in inside vm_util.py where we are checking if instance needs special spawning. Commit - 5c80f53b (cherry picked from commit b10324cfd0d7406192a9f5e11ae0aaf3ae877bcf) (cherry picked from commit f0f7b064371177a9f67850a33954067462d5cb9d) Change-Id: Iac974e16fa46399562e507c2286daa58a9e75c9d (cherry picked from commit db057275bc06959abf8973c40d84cc40691bafd8) --- nova/conf/vmware.py | 7 +++++++ nova/tests/unit/virt/vmwareapi/test_configdrive.py | 3 ++- nova/tests/unit/virt/vmwareapi/test_driver_api.py | 14 ++++++++++++++ nova/tests/unit/virt/vmwareapi/test_vmops.py | 7 +++++-- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index eab5225ffa4..09e74276a3a 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -140,6 +140,13 @@ ones. NOTE: Percentage values will be rounded down to the next full vcpu/MB. +"""), + cfg.StrOpt('special_spawning_vm_group', + default='bigvm_free_host_antiaffinity_vmgroup', + help=""" +For spawning a VM on a prepared empty host, we need all other VMs to be in a +DRS VM group that prohibits them from running on the prepared host. This +setting configures the name of that VM group. """), ] diff --git a/nova/tests/unit/virt/vmwareapi/test_configdrive.py b/nova/tests/unit/virt/vmwareapi/test_configdrive.py index c21b28d6fe7..fb700f1fc5b 100644 --- a/nova/tests/unit/virt/vmwareapi/test_configdrive.py +++ b/nova/tests/unit/virt/vmwareapi/test_configdrive.py @@ -126,6 +126,7 @@ def tearDown(self): vmwareapi_fake.cleanup() @mock.patch.object(vmops.VMwareVMOps, 'update_cluster_placement') + @mock.patch('nova.utils.vm_needs_special_spawning') @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata', return_value='fake_metadata') @mock.patch.object(vmops.VMwareVMOps, '_find_image_template_vm', @@ -134,7 +135,7 @@ def tearDown(self): return_value=None) def _spawn_vm(self, fake_fetch_image_from_other_datastores, fake_find_image_template_vm, fake_get_instance_meta, - mock_update_cluster_placement, + mock_vm_special_spawning, mock_update_cluster_placement, injected_files=None, admin_password=None, block_device_info=None): diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index 178af4f3838..f6dad771a83 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -21,6 +21,7 @@ import collections import datetime +import fixtures from unittest import mock from eventlet import greenthread @@ -40,6 +41,7 @@ from nova.compute import task_states from nova.compute import vm_states import nova.conf + from nova import context from nova import exception from nova.image import glance @@ -228,6 +230,18 @@ def setUp(self, mock_svc, mock_register): self.pt.new_root(self.cn_rp['name'], self.cn_rp['uuid'], generation=0) self.pt.new_root(self.shared_rp['name'], self.shared_rp['uuid'], generation=0) + """ Monkey patch for vm_util.vm_needs_special_spawning + Currently set to return `True` since most of the methods are calling + vm_util.update_cluster_placement which calls _get_server_groups. This + is done in order to prevent mocking the tests using spawn. + """ + self.useFixture( + fixtures.MonkeyPatch( + 'nova.utils.vm_needs_special_spawning', + self._fake_vm_needs_special_spawning)) + + def _fake_vm_needs_special_spawning(self, *args, **kwargs): + return True def tearDown(self): super(VMwareAPIVMTestCase, self).tearDown() diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index c27dc4ca319..45c869c603d 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -1890,6 +1890,7 @@ def _verify_spawn_method_calls(self, mock_call_method, extras=None): recorded_methods = [c[1][1] for c in mock_call_method.mock_calls] self.assertEqual(expected_methods, recorded_methods) + @mock.patch('nova.utils.vm_needs_special_spawning') @mock.patch('nova.compute.utils.is_volume_backed_instance', return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_create_folders', @@ -1934,10 +1935,10 @@ def _test_spawn(self, mock_update_cluster_placement, mock_create_folders, mock_is_volume_backed, + mock_special_spawning, block_device_info=None, extra_specs=None, config_drive=False): - if extra_specs is None: extra_specs = vm_util.ExtraSpecs() @@ -2383,9 +2384,11 @@ def test_create_swap_with_bdi(self): def test_create_swap_with_no_bdi(self): self._test_create_swap_from_instance(None) + @mock.patch('nova.utils.vm_needs_special_spawning') @mock.patch.object(vmops.VMwareVMOps, '_create_folders', return_value='fake_vm_folder') - def test_build_virtual_machine(self, mock_create_folder): + def test_build_virtual_machine(self, mock_create_folder, + mock_special_spawning): image = images.VMwareImage(image_id=self._image_id) extra_specs = vm_util.ExtraSpecs() From 44fd619ff07b79a32d528d326beef1ba6d99b8b0 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Wed, 22 Jul 2020 11:11:23 +0200 Subject: [PATCH 83/92] vmware: Switch from memoryReservationLockedToMax to memoryAllocation With supporting memoryAllocation updates also on resize, we don't have to lock the memory reservations to max but can use the memoryAllocation instead. This cleans up the code, as we don't have to pass down the parameter through a lot of functions to set memoryReservationLockedToMax. We still have to deconfigure `memoryReservationLockedToMax` on resize, because there might be some old VMs still running with that setting. Change-Id: I91d892c13b31745e190fcee8b97a2f3fb25fc014 (cherry picked from commit 02aa7dc39c966221042bb408aee69e3895e89b18) (cherry picked from commit 1406e3f113de170235614dd0f46de19fe3527175) Incorporated-Change-Id: Ic9aa65ce410b42dbdee907361cb56d8b1d254fdc (cherry picked from commit 85300f11f14507ecec3863a3223b789650f11144) --- .../tests/unit/virt/vmwareapi/test_vm_util.py | 24 +++---------------- nova/tests/unit/virt/vmwareapi/test_vmops.py | 7 +++--- nova/virt/vmwareapi/vm_util.py | 13 ++++------ nova/virt/vmwareapi/vmops.py | 11 +++------ 4 files changed, 14 insertions(+), 41 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 61bf200b099..ff6dcecc552 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -322,8 +322,7 @@ def test_get_resize_spec(self): extra_specs = vm_util.ExtraSpecs() fake_factory = fake.FakeFactory() result = vm_util.get_vm_resize_spec(fake_factory, - vcpus, memory_mb, extra_specs, - memory_reservation_locked=False) + vcpus, memory_mb, extra_specs) expected = fake_factory.create('ns0:VirtualMachineConfigSpec') expected.memoryMB = memory_mb expected.numCPUs = vcpus @@ -355,8 +354,7 @@ def test_get_resize_spec_with_limits(self): memory_limits=memory_limits) fake_factory = fake.FakeFactory() result = vm_util.get_vm_resize_spec(fake_factory, - vcpus, memory_mb, extra_specs, - memory_reservation_locked=True) + vcpus, memory_mb, extra_specs) expected = fake_factory.create('ns0:VirtualMachineConfigSpec') expected.memoryMB = memory_mb expected.numCPUs = vcpus @@ -374,7 +372,7 @@ def test_get_resize_spec_with_limits(self): memoryAllocation.shares.level = 'normal' memoryAllocation.shares.shares = 0 expected.memoryAllocation = memoryAllocation - expected.memoryReservationLockedToMax = True + expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -848,7 +846,6 @@ def test_get_vm_create_spec(self): extra_specs) expected = self._create_vm_config_spec() - expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -867,8 +864,6 @@ def test_get_vm_create_spec_with_serial_port(self): expected = self._create_vm_config_spec() expected.deviceChange = [serial_port_spec] - expected.memoryReservationLockedToMax = False - self.assertEqual(expected, result) def test_get_vm_create_spec_with_allocations(self): @@ -920,7 +915,6 @@ def test_get_vm_create_spec_with_allocations(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) - expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -972,7 +966,6 @@ def test_get_vm_create_spec_with_limit(self): cpu_allocation.shares.level = 'normal' cpu_allocation.shares.shares = 0 expected.cpuAllocation = cpu_allocation - expected.memoryReservationLockedToMax = False expected.numCPUs = 2 self.assertEqual(expected, result) @@ -1010,8 +1003,6 @@ def test_get_vm_create_spec_with_share(self): expected.managedBy.type = 'instance' expected.managedBy.extensionKey = 'org.openstack.compute' - expected.memoryReservationLockedToMax = False - expected.version = None expected.guestId = constants.DEFAULT_OS_TYPE @@ -1064,8 +1055,6 @@ def test_get_vm_create_spec_with_share_custom(self): expected.managedBy.extensionKey = 'org.openstack.compute' expected.managedBy.type = 'instance' - expected.memoryReservationLockedToMax = False - expected.version = None expected.guestId = constants.DEFAULT_OS_TYPE expected.tools = fake_factory.create('ns0:ToolsConfigInfo') @@ -1121,8 +1110,6 @@ def test_get_vm_create_spec_with_metadata(self): expected.managedBy.extensionKey = 'org.openstack.compute' expected.managedBy.type = 'instance' - expected.memoryReservationLockedToMax = False - expected.tools = fake_factory.create('ns0:ToolsConfigInfo') expected.tools.afterPowerOn = True expected.tools.afterResume = True @@ -1165,7 +1152,6 @@ def test_get_vm_create_spec_with_firmware(self): expected.managedBy = fake_factory.create('ns0:ManagedByInfo') expected.managedBy.extensionKey = 'org.openstack.compute' expected.managedBy.type = 'instance' - expected.memoryReservationLockedToMax = False expected.tools = fake_factory.create('ns0:ToolsConfigInfo') expected.tools.afterPowerOn = True @@ -1186,7 +1172,6 @@ def test_get_vm_create_spec_with_default_hw_version(self): extra_specs) expected = self._create_vm_config_spec() - expected.memoryReservationLockedToMax = False expected.version = 'vmx-13' self.assertEqual(expected, result) @@ -1992,7 +1977,6 @@ def test_get_vm_create_spec_with_console_delay(self): expected.tools.beforeGuestReboot = True expected.tools.beforeGuestShutdown = True expected.tools.beforeGuestStandby = True - expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -2036,7 +2020,6 @@ def test_get_vm_create_spec_with_cores_per_socket(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) - expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) @@ -2089,7 +2072,6 @@ def test_get_vm_create_spec_with_memory_allocations(self): extra_config.value = 'true' extra_config.key = 'disk.EnableUUID' expected.extraConfig.append(extra_config) - expected.memoryReservationLockedToMax = False self.assertEqual(expected, result) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 45c869c603d..d7fdb6e84e4 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -879,7 +879,6 @@ def _test_finish_revert_migration(self, fake_list_instances, int(self._instance.vcpus), int(self._instance.memory_mb), extra_specs, - CONF.vmware.reserve_all_memory, metadata=metadata) fake_reconfigure_vm.assert_called_once_with(self._session, 'fake-ref', @@ -1073,7 +1072,7 @@ def test_resize_vm(self, fake_resize_spec, fake_reconfigure, flavor=flavor) fake_resize_spec.assert_called_once_with( self._session.vim.client.factory, 2, 1024, extra_specs, - CONF.vmware.reserve_all_memory, metadata=self._metadata) + metadata=self._metadata) fake_reconfigure.assert_called_once_with(self._session, 'vm-ref', 'fake-spec') @@ -1088,7 +1087,7 @@ def test_resize_vm_bigvm_upsize(self, fake_drs_override, fake_is_big_vm, fake_resize_spec, fake_reconfigure, fake_get_extra_specs, fake_get_metadata): # new is big, new is big, old is not - fake_is_big_vm.side_effect = [True, True, False] + fake_is_big_vm.side_effect = [True, False] extra_specs = vm_util.ExtraSpecs() fake_get_extra_specs.return_value = extra_specs fake_get_metadata.return_value = self._metadata @@ -1118,7 +1117,7 @@ def test_resize_vm_bigvm_downsize(self, fake_drs_override, fake_is_big_vm, fake_resize_spec, fake_reconfigure, fake_get_extra_specs, fake_get_metadata): # new is not big, new is not big, old is big - fake_is_big_vm.side_effect = [False, False, True] + fake_is_big_vm.side_effect = [False, True] extra_specs = vm_util.ExtraSpecs() fake_get_extra_specs.return_value = extra_specs fake_get_metadata.return_value = self._metadata diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 8159c70e5d5..b23dcbbba74 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -39,7 +39,6 @@ from nova import exception from nova.i18n import _ from nova.network import model as network_model -from nova.utils import is_big_vm from nova.virt.vmwareapi import constants from nova.virt.vmwareapi import session from nova.virt.vmwareapi import vim_util @@ -373,11 +372,6 @@ def get_vm_create_spec(client_factory, instance, data_store_name, client_factory, extra_specs.memory_limits, 'ns0:ResourceAllocationInfo') - reservation_lock = CONF.vmware.reserve_all_memory \ - or is_big_vm(int(instance.memory_mb), - instance.flavor) - config_spec.memoryReservationLockedToMax = reservation_lock - if extra_specs.firmware: config_spec.firmware = extra_specs.firmware @@ -489,7 +483,7 @@ def get_vm_boot_spec(client_factory, device): def get_vm_resize_spec(client_factory, vcpus, memory_mb, extra_specs, - memory_reservation_locked, metadata=None): + metadata=None): """Provides updates for a VM spec.""" resize_spec = client_factory.create('ns0:VirtualMachineConfigSpec') resize_spec.numCPUs = vcpus @@ -500,7 +494,10 @@ def get_vm_resize_spec(client_factory, vcpus, memory_mb, extra_specs, resize_spec.memoryAllocation = _get_allocation_info( client_factory, extra_specs.memory_limits, 'ns0:ResourceAllocationInfo') - resize_spec.memoryReservationLockedToMax = memory_reservation_locked + # NOTE(jkulik) We used to set this setting instead of `memoryAllocation`, + # so we need to deconfigure it on VMs created before the patch adding + # `memoryAllocation` support on resize. + resize_spec.memoryReservationLockedToMax = False if metadata: resize_spec.annotation = metadata diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index ff73c2f625f..41d9394c7c1 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -353,6 +353,9 @@ def _get_extra_specs(self, flavor, image_meta=None): if value: setattr(getattr(extra_specs, resource + '_limits'), key, type(value)) + if CONF.vmware.reserve_all_memory \ + or utils.is_big_vm(int(flavor.memory_mb), flavor): + extra_specs.memory_limits.reservation = int(flavor.memory_mb) extra_specs.cpu_limits.validate() extra_specs.memory_limits.validate() extra_specs.disk_io_limits.validate() @@ -1672,16 +1675,12 @@ def _resize_vm(self, context, instance, vm_ref, flavor, image_meta): """Resizes the VM according to the flavor.""" client_factory = self._session.vim.client.factory extra_specs = self._get_extra_specs(flavor, image_meta) - reservation_locked = CONF.vmware.reserve_all_memory \ - or utils.is_big_vm(int(flavor.memory_mb), - flavor) metadata = self._get_instance_metadata(context, instance, flavor=flavor) vm_resize_spec = vm_util.get_vm_resize_spec(client_factory, int(flavor.vcpus), int(flavor.memory_mb), extra_specs, - reservation_locked, metadata=metadata) vm_util.reconfigure_vm(self._session, vm_ref, vm_resize_spec) @@ -1869,16 +1868,12 @@ def finish_revert_migration(self, context, instance, network_info, # Reconfigure the VM properties extra_specs = self._get_extra_specs(instance.flavor, instance.image_meta) - reservation_locked = CONF.vmware.reserve_all_memory \ - or utils.is_big_vm(int(instance.flavor.memory_mb), - instance.flavor) metadata = self._get_instance_metadata(context, instance) vm_resize_spec = vm_util.get_vm_resize_spec( client_factory, int(instance.flavor.vcpus), int(instance.flavor.memory_mb), extra_specs, - reservation_locked, metadata=metadata) vm_util.reconfigure_vm(self._session, vm_ref, vm_resize_spec) From 39d1da97558881ebfb4f17416b6c89d45dfd35f5 Mon Sep 17 00:00:00 2001 From: Jakob Karge Date: Tue, 27 Oct 2020 15:12:41 +0100 Subject: [PATCH 84/92] [memreserv] Reserve memory for certain flavors VMs with reserved memory have better memory allocation performance and -- it's suspected -- less softlock issues too. Coincidentally, many larger VMs also have high performance requirements that puts strict demands on the quick availability of their memory. Implement memory reservation and a configurable maximum number of cluster hosts that could theoretically fail and still let all VMs with memory reservation boot up. Nova provides the flavor extra_spec "quota:memory_reservation" which reserves the given amount of memory. This change does not make use of that feature, but instead introduces a parallel custom resource "CUSTOM_MEMORY_RESERVABLE_MB". The reason is that "quota:memory_reservation" does not allow for limiting the total amount of reservable memory in the same way the resource provider mechanics do. This is a requirement for tolerating the above-mentioned partial host failures, which can occur because a VMware host is a cluster of hypervisors. Add a "vm_reservable_memory_ratio" value to the cluster stats in the VMware driver, and use that to calculate and return the MEMORY_RESERVABLE_MB resource from the VMware driver's get_inventory(). The flavor-based memory reservation has less priority than the setting `reserve_all_memory` and the reservation of all memory for big VMs. (cherry picked from commit 93e22ef1523f76730a65aa6db139ae54f373d16e) Change-Id: I4b1457700428189457b94739d617467248a662df (cherry picked from commit 7f50b0bebef804af05195d2cf3f5780e63a92d49) --- nova/conf/vmware.py | 30 +++++++++++++++ .../unit/virt/vmwareapi/test_driver_api.py | 8 ++++ .../tests/unit/virt/vmwareapi/test_vm_util.py | 18 +++++---- nova/tests/unit/virt/vmwareapi/test_vmops.py | 38 +++++++++++++++++++ nova/utils.py | 5 +++ nova/virt/vmwareapi/driver.py | 10 +++++ nova/virt/vmwareapi/vm_util.py | 21 +++++++++- nova/virt/vmwareapi/vmops.py | 9 +++++ 8 files changed, 130 insertions(+), 9 deletions(-) diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index 09e74276a3a..8a71b79aa67 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -115,6 +115,36 @@ For big VMs as determined by the `bigvm_mb` setting, this setting is not used. Big VMs always reserve all their memory. +"""), + cfg.IntOpt('memory_reservation_cluster_hosts_max_fail', + default=0, + min=0, + help=""" +Allow reserving instance memory while at least n hypervisors of memory remain +unreserved in the cluster. This is a safety margin so a certain number of +cluster hypervisors are allowed to fail. If more memory would be reserved and +all the anticipated HV failures occurred, existing VMs with reserved memory +would no longer be able to start. + +This setting applies to VMs with flavors which have a nonzero extra_spec +"resources:CUSTOM_MEMORY_RESERVABLE_MB" set. + +The default value of 0 also leads to this setting being ignored and falling +back on `memory_reservation_max_ratio_fallback`. +"""), + cfg.FloatOpt('memory_reservation_max_ratio_fallback', + default=1.0, + help=""" +Allow reserving instance memory up to a maximum ratio in the cluster. This is a +safety margin so a certain ratio of cluster hypervisors are allowed to fail. If +more memory than that would be reserved and all the anticipated HV failures +occurred, existing VMs with reserved memory would no longer be able to start. + +This setting applies to VMs with flavors which have a nonzero extra_spec +"resources:CUSTOM_MEMORY_RESERVABLE_MB" set. + +This is a fallback when the `memory_reservation_cluster_hosts_max_fail` config +is set to 0. """), cfg.StrOpt('hostgroup_reservations_json_file', help=""" diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index f6dad771a83..b5d61cdf8e6 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -41,6 +41,7 @@ from nova.compute import task_states from nova.compute import vm_states import nova.conf +import nova.utils from nova import context from nova import exception @@ -2233,6 +2234,13 @@ def test_update_provider_tree(self, mock_get_avail_ds): 'step_size': 1, 'allocation_ratio': 1.0, }, + nova.utils.MEMORY_RESERVABLE_MB_RESOURCE: { + 'total': 2048 - 512, # 512 CONF.reserved_host_memory_mb + 'reserved': 0, + 'min_unit': 1, + 'max_unit': 1024, + 'step_size': 1, + } } inventory = self.pt.data(self.node_name).inventory self.assertEqual(expected, inventory) diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index ff6dcecc552..644874af4ed 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -142,7 +142,8 @@ def fake_call_method(*args): 'free': num_hosts * 4096 - num_hosts * 512, 'max_mem_mb_per_host': 4096, - 'reserved_memory_mb': 0}} + 'reserved_memory_mb': 0, + 'vm_reservable_memory_ratio': 1.0}} self.assertEqual(expected_stats, result) def test_get_stats_from_cluster_hosts_connected_and_active(self): @@ -197,13 +198,14 @@ def test_get_stats_from_cluster_reservations(self, mock_map): 'vcpus': 2}} expected = {'cpu': {'vcpus': 2 * 16, - 'max_vcpus_per_host': 14, # by host2 counts - 'reserved_vcpus': 4 + 2}, # both hosts - 'mem': {'total': 2 * 4096, - 'free': 2 * 4096 - - 2 * 512, - 'max_mem_mb_per_host': 4096 - 256, # host1 - 'reserved_memory_mb': 512 + 256}} # both + 'max_vcpus_per_host': 14, # by host2 counts + 'reserved_vcpus': 4 + 2}, # both hosts + 'mem': {'total': 2 * 4096, + 'free': 2 * 4096 - + 2 * 512, + 'max_mem_mb_per_host': 4096 - 256, # host1 + 'reserved_memory_mb': 512 + 256, # both + 'vm_reservable_memory_ratio': 1.0}} self._test_get_stats_from_cluster(expected_stats=expected) def test_get_host_reservations_empty(self): diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index d7fdb6e84e4..26d7d41eb46 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -1134,6 +1134,44 @@ def test_resize_vm_bigvm_downsize(self, fake_drs_override, fake_is_big_vm, 'vm-ref', operation='remove') + def test_reserve_all_memory_for_memory_reserved_flavor(self): + self.flags(group='vmware', reserve_all_memory=False) + self._instance.flavor.extra_specs.update({ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY: + self._instance.flavor.memory_mb}) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertEqual(specs.memory_limits.reservation, + self._instance.flavor.memory_mb) + + def test_reserve_no_memory_for_memory_unreserved_flavor(self): + self.flags(group='vmware', reserve_all_memory=False) + self._instance.flavor.extra_specs.pop( + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY, None) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertIsNone(specs.memory_limits.reservation) + + def test_global_reserve_all_memory_overrides_flavor_setting(self): + self.flags(group='vmware', reserve_all_memory=True) + self._instance.flavor.extra_specs.update({ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY: + self._instance.flavor.memory_mb // 2}) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertEqual(specs.memory_limits.reservation, + self._instance.flavor.memory_mb) # ie NOT memory_mb/2 + + def test_reserve_max_flavor_memory_for_memory_reserved_flavor(self): + self.flags(group='vmware', reserve_all_memory=False) + self._instance.flavor.extra_specs.update({ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY: + self._instance.flavor.memory_mb + 1000}) + specs = self._vmops._get_extra_specs(self._instance.flavor, + self._image_meta) + self.assertEqual(specs.memory_limits.reservation, + self._instance.flavor.memory_mb) + @mock.patch.object(vmops.VMwareVMOps, '_extend_virtual_disk') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(ds_util, 'disk_move') diff --git a/nova/utils.py b/nova/utils.py index e95612420b4..93647519f2f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -80,6 +80,11 @@ QUOTA_SEPARATE_KEY = 'quota:separate' QUOTA_INSTANCE_ONLY_KEY = 'quota:instance_only' +# Custom resource for reservable memory +MEMORY_RESERVABLE_MB_RESOURCE = 'CUSTOM_MEMORY_RESERVABLE_MB' +MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY = \ + 'resources:' + MEMORY_RESERVABLE_MB_RESOURCE + _FILE_CACHE = {} _SERVICE_TYPES = service_types.ServiceTypes() diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index fdc3213f67b..ccc5604c28c 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -505,6 +505,16 @@ def update_provider_tree(self, provider_tree, nodename, allocations=None): 'step_size': 1, 'allocation_ratio': ratios[orc.MEMORY_MB], }}) + available_memory_mb = stats['mem']['total'] - reserved_memory_mb + result.update({ + utils.MEMORY_RESERVABLE_MB_RESOURCE: { + 'total': available_memory_mb, + 'reserved': int(available_memory_mb * + (1 - stats['mem']['vm_reservable_memory_ratio'])), + 'min_unit': 1, + 'max_unit': stats['mem']['max_mem_mb_per_host'], + 'step_size': 1, + }}) # If a sharing DISK_GB provider exists in the provider tree, then our # storage is shared, and we should not report the DISK_GB inventory in diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index b23dcbbba74..c13c45151d9 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -1479,13 +1479,32 @@ def get_stats_from_cluster(session, cluster): threads - reserved['vcpus']) max_mem_mb_per_host = max(max_mem_mb_per_host, mem_mb - reserved['memory_mb']) + # NOTE (jakobk): For the total amount of hosts it doesn't matter + # whether the host is in MM or unreachable, because the count is + # used to calculate safety margins for resource allocations, and MM + # or otherwise unreachable hosts is precisely what that is supposed + # to guard against. + total_hypervisor_count = len(result.objects) + + # Calculate VM-reservable memory as a ratio of total available + # memory, depending on either the configured tolerance for failed + # hypervisors or a single configurable ratio. + max_fail_hvs = \ + CONF.vmware.memory_reservation_cluster_hosts_max_fail + if max_fail_hvs and total_hypervisor_count: + vm_reservable_memory_ratio = \ + (1 - max_fail_hvs / total_hypervisor_count) + else: + vm_reservable_memory_ratio = \ + CONF.vmware.memory_reservation_max_ratio_fallback stats = {'cpu': {'vcpus': vcpus, 'max_vcpus_per_host': max_vcpus_per_host, 'reserved_vcpus': reserved_vcpus}, 'mem': {'total': total_mem_mb, 'free': total_mem_mb - used_mem_mb, 'max_mem_mb_per_host': max_mem_mb_per_host, - 'reserved_memory_mb': reserved_memory_mb}} + 'reserved_memory_mb': reserved_memory_mb, + 'vm_reservable_memory_ratio': vm_reservable_memory_ratio}} return stats diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 41d9394c7c1..5937f71aa37 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -356,6 +356,15 @@ def _get_extra_specs(self, flavor, image_meta=None): if CONF.vmware.reserve_all_memory \ or utils.is_big_vm(int(flavor.memory_mb), flavor): extra_specs.memory_limits.reservation = int(flavor.memory_mb) + else: + try: + memory_reserved_mb = int(flavor.extra_specs[ + utils.MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY]) + if memory_reserved_mb > 0: + extra_specs.memory_limits.reservation = min( + int(flavor.memory_mb), memory_reserved_mb) + except (ValueError, KeyError): + pass extra_specs.cpu_limits.validate() extra_specs.memory_limits.validate() extra_specs.disk_io_limits.validate() From 7e4641f7897b00884d387feab32618bdbdf1313b Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 8 Nov 2019 11:10:23 +0100 Subject: [PATCH 85/92] vmware: `_list_instances_in_cluster()` returns properties We already have a function listing all the VMs in a cluster (`list_instances()`) but only returning the uuids. With this change, we add an additional `_list_instances_in_cluster()` method called by `list_instances()` so we do not change its interface. The method will also return the retrieved properties of the VMs, if the `additional_properties` parameter is set. This means we don't have to duplicate the code. This feature became necessary for handling the special-spawning cases of big VMs. We don't have a mapping for VM -> host in nova and have to retrieve it. (cherry picked from commit 1fb23b6505bbc690dcbd7154a80f69d47081ca8e) (cherry picked from commit 30daac70701aece9bee1add6de9a072e9e354121) Change-Id: I02a6938c8865c18089bb03a6add80ab7ae7e37b4 (cherry picked from commit 50eb6a06a7a2db126beb27456676741c5b3c7292) --- nova/virt/vmwareapi/vmops.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 5937f71aa37..ea9fc4832e5 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -2380,8 +2380,13 @@ def manage_image_cache(self, context, instances): datastores_info.append((ds, dc_info)) self._imagecache.update(context, instances, datastores_info) - def _get_valid_vms_from_retrieve_result(self, retrieve_result): - """Returns list of valid vms from RetrieveResult object.""" + def _get_valid_vms_from_retrieve_result(self, retrieve_result, + return_properties=False): + """Returns list of valid vms from RetrieveResult object. + + If `return_properties` is True, it will also return the properties of + these VMs, thus returning a tuple (vm_uuid, properties). + """ lst_vm_names = [] with vutil.WithRetrieval(self._session.vim, retrieve_result) as \ objects: @@ -2391,18 +2396,24 @@ def _get_valid_vms_from_retrieve_result(self, retrieve_result): continue vm_uuid = None conn_state = None + props = {} for prop in vm.propSet: if prop.name == "runtime.connectionState": conn_state = prop.val elif prop.name == 'config.extraConfig["nvp.vm-uuid"]': vm_uuid = prop.val.value + props[prop.name] = prop.val # Ignore VM's that do not have nvp.vm-uuid defined if not vm_uuid: continue # Ignoring the orphaned or inaccessible VMs - if conn_state not in ["orphaned", "inaccessible"]: - lst_vm_names.append(vm_uuid) + if conn_state in ["orphaned", "inaccessible"]: + continue + if return_properties: + lst_vm_names.append((vm_uuid, props)) + else: + lst_vm_names.append(vm_uuid) return lst_vm_names def instance_exists(self, instance): @@ -2620,9 +2631,18 @@ def get_datacenter_ref_and_name(self, ds_ref): return ds_util.get_dc_info(self._session, ds_ref) def list_instances(self): + lst_vm_names = self._list_instances_in_cluster() + + LOG.debug("Got total of %s instances", str(len(lst_vm_names))) + + return lst_vm_names + + def _list_instances_in_cluster(self, additional_properties=None): """Lists the VM instances that are registered with vCenter cluster.""" properties = ['runtime.connectionState', 'config.extraConfig["nvp.vm-uuid"]'] + if additional_properties is not None: + properties.extend(additional_properties) LOG.debug("Getting list of instances from cluster %s", vutil.get_moref_value(self._cluster)) vms = [] @@ -2630,9 +2650,10 @@ def list_instances(self): vms = self._session._call_method( vim_util, 'get_inner_objects', self._root_resource_pool, 'vm', 'VirtualMachine', properties) - lst_vm_names = self._get_valid_vms_from_retrieve_result(vms) + return_properties = additional_properties is not None + lst_vm_names = self._get_valid_vms_from_retrieve_result(vms, + return_properties=return_properties) - LOG.debug("Got total of %s instances", str(len(lst_vm_names))) return lst_vm_names def get_vnc_console(self, instance): From b299651b2c7aa6a9c842135c49db113f117b7425 Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Mon, 22 Nov 2021 12:56:26 +0100 Subject: [PATCH 86/92] vmware: Add helper to fetch DRS overrides This helper functions retrieves all DRS VM overrides of a cluster and provides them as a dict. We plan to use this in the special_spawning code. Change-Id: I091878d88b8545cb094b0f534f4fa57221c33719 (cherry picked from commit 06cdb9efae1655fecfa8b76dee4339df0fd1348e) (cherry picked from commit 6f59c694507db8906e061dfb37695e2f361741d5) --- nova/virt/vmwareapi/cluster_util.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/nova/virt/vmwareapi/cluster_util.py b/nova/virt/vmwareapi/cluster_util.py index 46c43c23295..c925604d9cc 100644 --- a/nova/virt/vmwareapi/cluster_util.py +++ b/nova/virt/vmwareapi/cluster_util.py @@ -389,3 +389,26 @@ def update_cluster_drs_vm_override(session, cluster, vm_ref, operation='add', config_spec.drsVmConfigSpec = [drs_vm_spec] reconfigure_cluster(session, cluster, config_spec) + + +def fetch_cluster_drs_vm_overrides(session, cluster_ref=None, + cluster_config=None): + """Fetch all enabled DRS overrides of the cluster + + The cluster can be identified by a cluster_ref or by an explicit + cluster_config. If identified by cluster_ref, we fetch the cluster_config. + + Returns a dict (key, behavior) where key is the VM moref value and behavior + a `DrsBehaviour` as string. + """ + if (cluster_config, cluster_ref) == (None, None): + msg = 'Either cluster_config or cluster_ref must be given.' + raise exception.ValidationError(msg) + + if cluster_config is None: + cluster_config = session._call_method( + vim_util, "get_object_property", cluster_ref, "configurationEx") + + overrides = getattr(cluster_config, 'drsVmConfig', []) + return {vim_util.get_moref_value(o.key): o.behavior for o in overrides + if o.enabled} From d7f5be38531c7181e1cb123c75b0bdf442a8282d Mon Sep 17 00:00:00 2001 From: Johannes Kulik Date: Fri, 8 Nov 2019 11:16:46 +0100 Subject: [PATCH 87/92] Support spawning big VMs We need to be able to spawn big VM and thus have to prepare a host for it i.e. free it from other VMs. This cannot be mapped in nova currently, as nova has no view on the individual hosts in a VMware cluster. Therefore, we implement some function in the VMware driver to actually free up the host and also release it into the cluster again. Placement is used for keeping a view over all clusters, as we need one host free per HV size, per Shard, per AZ. We have to make sure that we run the functionality only once. Placing it into the conductor will - at least in regions with cells - run it multiple times. So we could add some leader-election into the process or create our own service that we only spawn once per region. We opted for the latter, as it should be simpler. We have a threshold in place up to which memory usage in a cluster we try to spawn big VMs. Since trying to free a host in a too full cluster would lead to memory overcommit on the remaining hosts, we by default only keep or create a free host until the cluster has by default 80 % memory usage. If a RP is marked as only accepting hana_* flavors by the `CUSTOM_HANA_EXCLUSIVE_HOST` trait, we do not need to adhere to this threshold. These clusters have explicit failover hosts to keep enough resources available. Additionally, we can also prohibit freeing up a host in a cluster or giving back a free host to the cluster by adding the `CUSTOM_BIGVM_DISABLED` trait to its resource provider. Since spawning on disabled compute nodes doesn't work and Nova adds the `COMPUTE_STATUS_DISABLED` trait to the RP, when a compute node gets disabled, we can and should ignore those nodes when looking for a new big VM cluster. The placement-client has an internal cache for resource-providers, which we need to fill to be able to use a couple of functions. Since we're using private functions, the client doesn't call `ensure_resource_provider()` everywhere and thus might not have the resource-provider in the cache and thus refuses to work with it. If we change something by a direct request to Placement's API, we also have to update/refresh the cache. We clean up a provider if we get an "error" returned as state. This can happen, if the host we decided on earlier e.g. got disconnected or became failover host. We then need to find another host - maybe in another cluster - to be able to spawn VMs for that hypervisor size. We also have to remove the host from the hostgroup, so it can properly join the cluster later on. We check already freed up hosts in every run to ensure that they are still suitable. In calling `free_host()` again, the `nova-compute` service should return FREE_HOST_STATE_DONE if the host is still suitable. We also clean providers that match our naming scheme but for which we do not find a vmware provider anymore, because they can lead to `KeyError`s late in the code. These could be created with changing settings or manually allowing big VMs on some cluster. inside nova-compute =================== If the DRS policy is not set to "fullyAutomated", i.e. if DRS will not take action on existing, running VMs, we first check if there's already a host free (e.g. because an operator freed one manually). If we don't find one, we have no chance to free up a host without DRS and thus we error out to tell nova-bigvm to search for another cluster. We ignore failover hosts and hosts in disconnected state or maintenance mode. This is the same as `vm_util.get_stats_from_cluster()` does for reporting capacity cluster capacity. Reasoning: We can't spawn a VM on any of those hosts. These filters are not only applied when searching for a new host to free, but also for checking on the host already destined to be freed up. If its state changes, we return an error-state, which should let nova-bigvm restart the search for that host's hypervisor-size. Instead of adding up the configured memory of the VMs on a host, we add up the used memory of the VMs. The reasoning behind this is, that it would be faster/easier to free up the host if the VMs are not using all their RAM, because there's less to copy around and hopefully less memory activity. We remove the hostgroup, because having an empty host list is not possible. At the same time, we also have to delete the matching DRS rule, because DRS does not update its internal state for a rule, if the rule has become an invalid configuration. This effectively means, that the rule stays in place if we remove the hostgroup but not the rule using it. Previously, we had a check in place that would return an error if no VMs were found on the cluster. This prohibited using newly-built clusters, that did not have a VM, so we do not want such a check anymore. We had a case, where the vCenter discovered some random VM, which didn't even have a `config` attribute when retrieved from the API. We now ignore these VMs instead of letting the `KeyError` bubble up, because they are pretty much unusable and should not matter for our decision-making process. Additionally, with vSphere 7 there are `vCLS` VMs providing (parts of) the DRS service, which we need to ignore. The reason is that DRS doesn't move those VMs, even if they violate a DRS rule. Therefore, we would never see a host getting freed up, if it contained a `vCLS` VM. They take up 100 MiB of reserved RAM and thus should fit next to a big VM. Therefore, we ignore them. With large VMs being set to partiallyAutomated, compute nodes stayed in "waiting" state more and more often and never came out of it. Since we still need to deploy big VMs to those compute nodes and they are usually just stopped from being free by a large VM that cannot move, we often mark those compute nodes free by hand. Since this doesn't scale, we want to automate this behavior by letting the nova-compute take the same decision. Therefore, if there are only partiallyAutomated VMs left on a host, we still report that host as freed up. This is done by ignoring all `partiallyAutomated` VMs during the check of running VMs on the host. When we spawn or resize a VM on the prepared empty host in the cluster, we have to mark the resource as reserved on the special resource provider. This allows the code handling the emptying of hosts to recognize a used host and start the clean-up operations necessary after successful spawning/resizing. Before marking the host as used, we have to refresh the RP inventory, because the RP generation in our cache can be out of sync. For resizes, we have to apply it for both big->big as well as small->big resizes. scheduler ========= Ignore hana/BigVM flavors by default in the ShardFilter, allowing such flavors to be deployed on any shard. This behaviour can be disabled globally by setting [filter_scheduler] sharding_ignore_hana to False It can also be disabled per project, by adding the tag `use_individual_shard_tags_hana` to the project. We can tell the scheduler how to determine if a flavor is hana and/or BigVM by setting the [filter_scheduler] hana_detection_strategy configuration to either `hana_exclusive_host` or `memory_mb`. This has to be set to `memory_mb` for region where there are no hana exclusive BBs and nova-bigvm is still used to free up hosts. Co-Authored-By: Marius Leustean Co-Authored-By: Jakob Karge Change-Id: I2e6e7ecf5348ac7d32e23349c7196b65a73b4cf8 Incorporated-Change-Id: I65c2be9781b60e31d9a6c6dd84a02c7fe4aa1e72 Incorporated-Change-Id: I6e994281ae2e62096c2f5241dd493fe1f9c9d515 Incorporated-Change-Id: Idbdfe82b4057844401e710fb9d87141478bb3353 Incorporated-Change-Id: I5ceb4d9338f6d94f49cc2deff25eefb19df2030f Incorporated-Change-Id: Ie033332436674f4fe792f4aa3f83f33b12a6d9ed Incorporated-Change-Id: Ic9d707c59a4ea405f3a982dbe269cdfea0d03aa5 Incorporated-Change-Id: I2329db43adc00e04b07d16d312361ae5e669d298 Incorporated-Change-Id: I36813ed3d95fd8572c6b75544ebb2fc1936f6bdb Incorporated-Change-Id: I737f312db0e156fa971a189d47efd227c666b178 Incorporated-Change-Id: I43d3d060f7c51841a8561a29d3189e37e4f87fb1 Incorporated-Change-Id: I2883c5c713ee657006c80d61ebc59a086ec22411 Incorporated-Change-Id: Ibd576211ac33f38ef0e1b9016381a955916d7c1c Incorporated-Change-Id: I1b18a7b7db0452a10f0adc499be2df26d923f936 Incorporated-Change-Id: Ie96a41bd9151e869fabd18d69777f38db85d0ca6 Incorporated-Change-Id: Iaa1d18eb0a3e78bf1e361c8e8d1040aa07344448 Incorporated-Change-Id: Ic9b86aebbae7796fc17db3bb69f6f9ee9fffb9c9 Incorporated-Change-Id: I1eec50a5b1b6206e5b3eab8d5f9fa891fecb4b25 Incorporated-Change-Id: Ic57a71bc4e69c57833396690fc3fb5453aa122b3 Incorporated-Change-Id: Iac3b8c183f633bf5ddc59acf340b477bd1eb88cc Incorporated-Change-Id: I47feb6a34c0e210f0ebb0edd4479550750e605d7 Incorporated-Change-Id: I22dbdcc9f135bbfc9ef05e13c801e88a78e64236 Incorporated-Change-Id: I250f203b3bb24e084ec1b499a923f7f66e638102 Incorporated-Change-Id: If3986df022273f20e109816f2752ce0254db4f10 Incorporated-Change-Id: I5a4c6c5a1894d1f6f5cff6e3475670c27bb97f28 Incorporated-Change-Id: I2905b470e91770d5951260e69a81d1910cb15972 Incorporated-Change-Id: I57cad032fc9002cb4b5013e4868215a60ebe412a Incorporated-Change-Id: I6801fce6cf95c37096a045799d30cbde843c509a Incorporated-Change-Id: I840e5d671c3df24cd01952b522288982ddfde80e Incorporated-Change-Id: I71f0324395e0135d8facdbf45c74c9682093104d Incorporated-Change-Id: I43fe1c4dc40caf5781457a0d4de83c6cb53ebc8f Incorporated-Change-Id: Iabeae6e01f9615be7c122d1e3fd719a1e53762d9 Incorporated-Change-Id: Ieb4ad968657ef8d2bf19e08ec954da5acc51372d Incorporated-Change-Id: I71e856382453343d8f593b330e9268d3b36b4dfa (cherry picked from commit 21502fed8f63d3654705975e68f3b4e8c2ea44fd) --- nova/bigvm/__init__.py | 0 nova/bigvm/manager.py | 729 ++++++++++++++++++ nova/cmd/bigvm.py | 44 ++ nova/conf/base.py | 26 + nova/conf/scheduler.py | 27 + nova/conf/vmware.py | 9 + nova/objects/compute_node.py | 3 +- nova/objects/fields.py | 3 +- nova/scheduler/filters/shard_filter.py | 29 +- nova/service.py | 1 + .../objects/test_notification.py | 2 +- .../scheduler/filters/test_shard_filter.py | 69 ++ nova/tests/unit/virt/vmwareapi/test_vmops.py | 28 +- nova/utils.py | 2 + nova/virt/vmwareapi/constants.py | 1 + nova/virt/vmwareapi/driver.py | 2 + nova/virt/vmwareapi/special_spawning.py | 360 +++++++++ nova/virt/vmwareapi/vmops.py | 43 +- setup.cfg | 1 + 19 files changed, 1361 insertions(+), 18 deletions(-) create mode 100644 nova/bigvm/__init__.py create mode 100644 nova/bigvm/manager.py create mode 100644 nova/cmd/bigvm.py create mode 100644 nova/virt/vmwareapi/special_spawning.py diff --git a/nova/bigvm/__init__.py b/nova/bigvm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/nova/bigvm/manager.py b/nova/bigvm/manager.py new file mode 100644 index 00000000000..8a332f118f3 --- /dev/null +++ b/nova/bigvm/manager.py @@ -0,0 +1,729 @@ +# Copyright 2019 SAP SE +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +BigVM service +""" +import itertools + +import os_resource_classes as orc +import os_traits +from oslo_log import log as logging +from oslo_messaging import exceptions as oslo_exceptions +from oslo_service import periodic_task + +import nova.conf +from nova import context as nova_context +from nova import exception +from nova import manager +from nova.objects.aggregate import AggregateList +from nova.objects.cell_mapping import CellMappingList +from nova.objects.compute_node import ComputeNodeList +from nova.objects.host_mapping import HostMappingList +from nova.scheduler.client.report import get_placement_request_id +from nova.scheduler.client.report import NESTED_PROVIDER_API_VERSION +from nova.scheduler.client.report import SchedulerReportClient +from nova.scheduler.utils import ResourceRequest +from nova.utils import BIGVM_EXCLUSIVE_TRAIT +from nova.virt.vmwareapi import special_spawning + +LOG = logging.getLogger(__name__) + +CONF = nova.conf.CONF + +MEMORY_MB = orc.MEMORY_MB +BIGVM_RESOURCE = special_spawning.BIGVM_RESOURCE +BIGVM_DISABLED_TRAIT = 'CUSTOM_BIGVM_DISABLED' +VMWARE_HV_TYPE = 'VMware vCenter Server' +SHARD_PREFIX = 'vc-' +HV_SIZE_BUCKET_THRESHOLD_PERCENT = 10 + + +class BigVmManager(manager.Manager): + """Takes care of the needs of big VMs""" + + def __init__(self, *args, **kwargs): + self.placement_client = SchedulerReportClient() + self.special_spawn_rpc = special_spawning.SpecialVmSpawningInterface() + + super(BigVmManager, self).__init__(service_name='bigvm', + *args, **kwargs) + + @periodic_task.periodic_task(spacing=CONF. + prepare_empty_host_for_spawning_interval, + run_immediately=True) + def _prepare_empty_host_for_spawning(self, context): + """Handle freeing up hosts per hv_size per VC for spawning + + The general workflow is: + 1) Find all hypervisor sizes (hv_size) existing in a vCenter (VC) + within an availability zone (az). + 2) Choose one host per hv_size per VC and create a child resource + provider (rp) for it with a well-known name. This marks that + host as responsible for the hv_size in that VC. The child rp has no + resources, yet. This also triggers the vmware-driver to free up a + host in the cluster. + 3) Either directly or in the next iteration every host without + resources is checked for its status by calling the vmware + driver. If we're done freeing up the host, we add the + BIGVM_RESOURCE to the child rp to make it consumable. + 4) In every iteration, we check for reserved BIGVM_RESOURCEs on a + child rp and if the host is still free. Reserving a resource happens + after consumption, so we know the host is not longer free and have + to clean up the child rp and the vmware part. The host might be used + again if a migration happened in the background. We need to clean up + the child rp in this case and redo the scheduling. + + We only want to fill up clusters to a certain point, configurable via + bigvm_cluster_max_usage_percent. resource-providers having more RAM + usage than this, will not be used for a hv_size - if they are not + exclusively used for HANA flavors.. Additionally, we check in every + iteration, if we have to give up a freed-up host, because the cluster + reached the limit. + """ + client = self.placement_client + + # make sure our custom trait exists + client._ensure_traits(context, [BIGVM_DISABLED_TRAIT, + BIGVM_EXCLUSIVE_TRAIT]) + + vcenters, bigvm_providers, vmware_providers = \ + self._get_providers(context) + + self._check_and_clean_providers(context, client, bigvm_providers, + vmware_providers) + + missing_hv_sizes_per_vc = self._get_missing_hv_sizes(context, + vcenters, bigvm_providers, vmware_providers) + + if not any(missing_hv_sizes_per_vc.values()): + LOG.info('Free host for spawning defined for every ' + 'vCenter and hypervisor-size.') + return + + def _flatten(list_of_lists): + return itertools.chain.from_iterable(list_of_lists) + + # retrieve allocation candidates for all hv sizes. we later have to + # filter them by VC, because our placement doesn't know about VCs. + candidates = {} + for hv_size in set(_flatten(missing_hv_sizes_per_vc.values())): + resources = ResourceRequest() + resources._add_resource(MEMORY_MB, hv_size) + res = client.get_allocation_candidates(context, resources) + if res is None: + continue + alloc_reqs, provider_summaries, allocation_request_version = res + + # filter out providers, that don't match the full host, e.g. don't + # allow 3 TB on a 6 TB host, as we need a fully free host + provider_summaries = {p: d for p, d in provider_summaries.items() + if vmware_providers.get(p, {}).get('hv_size') == hv_size} + + if not provider_summaries: + LOG.warning('Could not find enough resources to free up a ' + 'host for hypervisor size %(hv_size)d.', + {'hv_size': hv_size}) + continue + + # filter out providers that are too full already + filtered_provider_summaries = {} + for p, d in provider_summaries.items(): + # Hosts exclusively used for hana_* flavors cannot be too full + if BIGVM_EXCLUSIVE_TRAIT in d['traits']: + filtered_provider_summaries[p] = d + continue + + used = vmware_providers.get(p, {})\ + .get('memory_mb_used_percent', 100) + if used > CONF.bigvm_cluster_max_usage_percent: + continue + filtered_provider_summaries[p] = d + + if not filtered_provider_summaries: + LOG.warning('Could not find a resource-provider to free up a ' + 'host for hypervisor size %(hv_size)d, because ' + 'all clusters are used more than %(max_used)d.', + {'hv_size': hv_size, + 'max_used': CONF.bigvm_cluster_max_usage_percent}) + continue + + # filter out providers that are disabled in general or for bigVMs + # specifically + provider_summaries = filtered_provider_summaries + filtered_provider_summaries = {} + for p, d in provider_summaries.items(): + if os_traits.COMPUTE_STATUS_DISABLED in d['traits'] \ + or BIGVM_DISABLED_TRAIT in d['traits']: + continue + filtered_provider_summaries[p] = d + + if not filtered_provider_summaries: + LOG.warning('Could not find a resource-provider to free up a ' + 'host for hypervisor size %(hv_size)d, because ' + 'all providers with enough space are disabled.', + {'hv_size': hv_size}) + continue + + candidates[hv_size] = (alloc_reqs, filtered_provider_summaries) + + for vc in vcenters: + for hv_size in missing_hv_sizes_per_vc[vc]: + if hv_size not in candidates: + LOG.warning('Could not find a resource-provider to free ' + 'up a host for hypervisor size %(hv_size)d in ' + '%(vc)s.', + {'hv_size': hv_size, 'vc': vc}) + continue + alloc_reqs, provider_summaries = candidates[hv_size] + + # filter providers by VC, as placement returned all matching + # providers + providers = {p: d for p, d in provider_summaries.items() + if vmware_providers.get(p, {}).get('vc') == vc} + + # select the one with the least usage + def _free_memory(p): + memory = providers[p]['resources'][MEMORY_MB] + return memory['capacity'] - memory['used'] + + provider_uuids = sorted((p for p in providers), + key=_free_memory, reverse=True) + + try: + for rp_uuid in provider_uuids: + host = vmware_providers[rp_uuid]['host'] + cm = vmware_providers[rp_uuid]['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + if self._free_host_for_provider(cctxt, rp_uuid, + host): + break + except oslo_exceptions.MessagingTimeout as e: + # we don't know if the timeout happened after we started + # freeing a host already or because we couldn't reach the + # nova-compute node. Therefore, we move on to the next HV + # size for that VC and hope the timeout resolves for the + # next run. + LOG.exception(e) + LOG.warning('Skipping HV size %(hv_size)s in VC %(vc)s ' + 'because of error', + {'hv_size': hv_size, 'vc': vc}) + + def _get_providers(self, context): + """Return our special and the basic vmware resource-providers + + This returns a list of vcenters and two dicts, where the + resource-provider uuid is the key. The value contains a dict with the + important information for each resource-provider, like host, az, vc and + cell_mapping + either the hypervisor size (vmware provider) or the + resource-provider dict (special provider). + """ + client = self.placement_client + + vmware_hvs = {} + for cm in CellMappingList.get_all(context): + with nova_context.target_cell(context, cm) as cctxt: + vmware_hvs.update({cn.uuid: cn.host for cn in + ComputeNodeList.get_by_hypervisor_type(cctxt, + VMWARE_HV_TYPE) + if not cn.deleted}) + + host_azs = {} + host_vcs = {} + for agg in AggregateList.get_all(context): + if not agg.availability_zone: + continue + + if agg.name == agg.availability_zone: + for host in agg.hosts: + host_azs[host] = agg.name + elif agg.name.startswith(SHARD_PREFIX): + for host in agg.hosts: + host_vcs[host] = agg.name + + vcenters = set(host_vcs.values()) + + host_mappings = {hm.host: hm.cell_mapping + for hm in HostMappingList.get_all(context)} + + # find all resource-providers that we added and also a list of vmware + # resource-providers + bigvm_providers = {} + vmware_providers = {} + resp = client.get('/resource_providers', + version=NESTED_PROVIDER_API_VERSION) + for rp in resp.json()['resource_providers']: + if rp['name'].startswith(CONF.bigvm_deployment_rp_name_prefix): + # We use the _root_ RP and not the parent because that will + # always map to the ComputeNode, even if we decide to nest RPs + # further. + host_rp_uuid = rp['root_provider_uuid'] + host = vmware_hvs[host_rp_uuid] + cell_mapping = host_mappings[host] + bigvm_providers[rp['uuid']] = {'rp': rp, + 'host': host, + 'az': host_azs[host], + 'vc': host_vcs[host], + 'cell_mapping': cell_mapping, + 'host_rp_uuid': host_rp_uuid} + elif rp['uuid'] not in vmware_hvs: # ignore baremetal + continue + else: + # retrieve the MEMORY_MB resource + url = '/resource_providers/{}/inventories/{}'.format( + rp['uuid'], MEMORY_MB) + resp = client.get(url) + if resp.status_code != 200: + LOG.error('Could not retrieve inventory for RP %(rp)s.', + {'rp': rp['uuid']}) + continue + memory_mb_inventory = resp.json() + + # retrieve the usage + url = '/resource_providers/{}/usages' + resp = client.get(url.format(rp['uuid'])) + if resp.status_code != 200: + LOG.error('Could not retrieve usages for RP %(rp)s.', + {'rp': rp['uuid']}) + continue + usages = resp.json()['usages'] + + hv_size = memory_mb_inventory['max_unit'] + memory_mb_total = (memory_mb_inventory['total'] - + memory_mb_inventory['reserved']) + memory_mb_used_percent = (usages[MEMORY_MB] / float( + memory_mb_total) * 100) + + host = vmware_hvs[rp['uuid']] + # ignore hypervisors we would never use anyways + if hv_size < CONF.bigvm_mb: + LOG.debug('Ignoring %(host)s (%(hv_size)s < %(bigvm_mb)s)', + {'host': host, 'hv_size': hv_size, + 'bigvm_mb': CONF.bigvm_mb}) + continue + + cell_mapping = host_mappings[host] + if host not in host_azs or host not in host_vcs: + # seen this happening during buildup + LOG.debug('Ignoring %(host)s as it is not assigned to an ' + 'AZ or VC.', + {'host': host}) + continue + + # retrieve traits so we can find disabled and hana exclusive + # hosts + traits = client.get_provider_traits(context, rp['uuid']) + vmware_providers[rp['uuid']] = { + 'hv_size': hv_size, + 'host': host, + 'az': host_azs[host], + 'vc': host_vcs[host], + 'cell_mapping': cell_mapping, + 'traits': traits, + 'memory_mb_used_percent': memory_mb_used_percent} + + # make sure the placement cache is filled + client.get_provider_tree_and_ensure_root(context, rp['uuid'], + rp['name']) + + # retrieve all bigvm provider's inventories + for rp_uuid, rp in bigvm_providers.items(): + inventory = client._get_inventory(context, rp_uuid) + rp['inventory'] = inventory['inventories'] if inventory else {} + + # make sure grouping by hv_size works properly later on, even if there + # are marginal differences in the reported hv_size. we need to have all + # vmware_providers to have the same hv_size if they're in the same + # "bucket", e.g. they're supposed to be 3 TB HVs. To make sure the + # placement query for allocation-candidates works, we use the smallest + # size and assign it to all in the same bucket. + hv_size_bucket = None + for rp_uuid, rp in sorted(vmware_providers.items(), + key=lambda x: x[1]['hv_size']): + if hv_size_bucket is None: + # first one is always a new bucket + hv_size_bucket = rp['hv_size'] + continue + + threshold = HV_SIZE_BUCKET_THRESHOLD_PERCENT * hv_size_bucket / 100 + if rp['hv_size'] - hv_size_bucket > threshold: + # set key if the difference to the last key is over the + # threshold + hv_size_bucket = rp['hv_size'] + + rp['hv_size'] = hv_size_bucket + + return (vcenters, bigvm_providers, vmware_providers) + + def _check_and_clean_providers(self, context, client, bigvm_providers, + vmware_providers): + + # check for reserved resources which indicate that the free host was + # consumed + providers_to_delete = {rp_uuid: rp + for rp_uuid, rp in bigvm_providers.items() + if rp['inventory'].get(BIGVM_RESOURCE, {}) + .get('reserved')} + + # check if we don't have a valid vmware provider for it (anymore) and + # thus cannot be a valid provider ourselves + providers_to_delete.update({ + rp_uuid: rp for rp_uuid, rp in bigvm_providers.items() + if rp_uuid not in providers_to_delete and + rp['host_rp_uuid'] not in vmware_providers}) + + # check for resource-providers having more than + # bigvm_cluster_max_usage_percent usage + for rp_uuid, rp in bigvm_providers.items(): + if rp_uuid in providers_to_delete: + # no need to check if we already remove it anyways + continue + + host_rp = vmware_providers[rp['host_rp_uuid']] + # Hosts exclusively used for hana_* flavors cannot be too full + if BIGVM_EXCLUSIVE_TRAIT in host_rp['traits']: + continue + + used_percent = host_rp['memory_mb_used_percent'] + if used_percent > CONF.bigvm_cluster_max_usage_percent: + providers_to_delete[rp_uuid] = rp + LOG.info('Resource-provider %(host_rp_uuid)s with free host ' + 'is overused. Marking %(rp_uuid)s for deletion.', + {'host_rp_uuid': rp['host_rp_uuid'], + 'rp_uuid': rp_uuid}) + + # check if a provider got used in the background without our knowledge + for rp_uuid, rp in bigvm_providers.items(): + if rp_uuid in providers_to_delete: + # no need to check if we already remove it anyways + continue + + # if we have no resources on the resource-provider, we don't expect + # it to be free, yet + if not rp['inventory'].get(BIGVM_RESOURCE, {}): + continue + + # ask the compute-node if the host is still free. anything other + # than FREE_HOST_STATE_DONE means we've got an unexpected state and + # should re-schedule that size + cm = rp['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + state = self.special_spawn_rpc.free_host(cctxt, rp['host']) + if state != special_spawning.FREE_HOST_STATE_DONE: + LOG.info('Checking on already freed up host %(host)s ' + 'returned with state %(state)s. Marking ' + '%(rp_uuid)s for deletion.', + {'host': rp['host'], + 'state': state, + 'rp_uuid': rp_uuid}) + providers_to_delete[rp_uuid] = rp + + # check if a provider was disabled by now + for rp_uuid, rp in bigvm_providers.items(): + if rp_uuid in providers_to_delete: + # no need to check if we already remove it anyways + continue + + host_rp = vmware_providers[rp['host_rp_uuid']] + if os_traits.COMPUTE_STATUS_DISABLED in host_rp['traits'] \ + or BIGVM_DISABLED_TRAIT in host_rp['traits']: + providers_to_delete[rp_uuid] = rp + LOG.info('Resource-provider %(host_rp_uuid)s got disabled in' + ' general or specifically for bigVMs. Marking' + ' %(rp_uuid)s for deletion.', + {'host_rp_uuid': rp['host_rp_uuid'], + 'rp_uuid': rp_uuid}) + + for rp_uuid, rp in providers_to_delete.items(): + self._clean_up_consumed_provider(context, rp_uuid, rp) + + # clean up our list of resource-providers from consumed or overused + # hosts + for rp_uuid in providers_to_delete: + del bigvm_providers[rp_uuid] + + def _get_allocations_for_consumer(self, context, consumer_uuid): + """Same as SchedulerReportClient.get_allocations_for_consumer() but + includes user_id and project_id in the returned values, by doing the + request with a newer version. + """ + client = self.placement_client + url = '/allocations/%s' % consumer_uuid + resp = client.get(url, global_request_id=context.global_id, + version=1.17) + if not resp: + return {} + else: + return resp.json() + + def _remove_provider_from_consumer_allocations(self, context, + consumer_uuid, rp_uuid): + """This is basically the same as + SchedulerClient.remove_provider_from_instance_allocation, but without + the resize-on-same-host detection - it simply removes the provider from + the allocations of the consumer. + """ + client = self.placement_client + + # get the allocation details, because we need user_id and + # project_id to call the delete function + current_allocs = \ + self._get_allocations_for_consumer(context, consumer_uuid) + + LOG.debug('Found the following allocations for consumer ' + '%(consumer_uuid)s: %(allocations)s', + {'consumer_uuid': consumer_uuid, + 'allocations': current_allocs}) + + new_allocs = [ + { + 'resource_provider': { + 'uuid': alloc_rp_uuid, + }, + 'resources': alloc['resources'], + } + for alloc_rp_uuid, alloc in current_allocs['allocations'].items() + if alloc_rp_uuid != rp_uuid + ] + payload = {'allocations': new_allocs, + 'project_id': current_allocs['project_id'], + 'user_id': current_allocs['user_id']} + LOG.debug("Sending updated allocation %s for instance %s after " + "removing resources for %s.", + new_allocs, consumer_uuid, rp_uuid) + url = '/allocations/%s' % consumer_uuid + r = client.put(url, payload, version='1.10', + global_request_id=context.global_id) + if r.status_code != 204: + LOG.warning("Failed to save allocation for %s. Got HTTP %s: %s", + consumer_uuid, r.status_code, r.text) + return r.status_code == 204 + + def _clean_up_consumed_provider(self, context, rp_uuid, rp): + """Clean up after a resource-provider was consumed + + We need to remove all allocations, the resource-provider itself and + also the hostgroup from the vCenter. + """ + client = self.placement_client + + # find the consumer + allocations = client.get_allocations_for_resource_provider( + context, rp_uuid).allocations + # we might have already deleted them and got killed or the VM got + # deleted in the mean time + failures = 0 + for consumer_uuid, resources in allocations.items(): + # delete the allocations + # we can't use + # SchedulerClient.remove_provider_from_instance_allocation here, as + # this would detect a resize and try to remove the resources, but + # keep an allocation. We need the specific allocation for our + # resource-provider removed. + if self._remove_provider_from_consumer_allocations(context, + consumer_uuid, rp_uuid): + LOG.info('Removed bigvm allocations for %(consumer_uuid)s ' + 'from RP', {'consumer_uuid': consumer_uuid}) + else: + LOG.error('Could not remove bigvm allocations for ' + '%(consumer_uuid)s corresponding RP.', + {'consumer_uuid': consumer_uuid}) + failures += 1 + + if failures: + # skip removing the resource-provider because we couldn't + # remove all allocations. we'll retry on the next run + LOG.warning('Skippping removal of resource-provider ' + '%(rp_uuid)s as we could not remove some ' + 'allocations.', + {'rp_uuid': rp_uuid}) + return + + # remove the hostgroup from the host + cm = rp['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + if not self.special_spawn_rpc.remove_host_from_hostgroup(cctxt, + rp['host']): + LOG.warning('Skipping removal of resource-provider ' + '%(rp_uuid)s as we could not remove the hostgroup ' + 'from the vCenter.', + {'rp_uuid': rp_uuid}) + return + + # delete the resource-provider + client._delete_provider(rp_uuid) + LOG.info('Removed resource-provider %(rp_uuid)s.', + {'rp_uuid': rp_uuid}) + + def _get_missing_hv_sizes(self, context, vcenters, + bigvm_providers, vmware_providers): + """Search and return hypervisor sizes having no freed-up host + + Returns a dict containing a set of hv sizes missing a freed-up host for + each vCenter. + """ + found_hv_sizes_per_vc = {vc: set() for vc in vcenters} + + for rp_uuid, rp in bigvm_providers.items(): + host_rp_uuid = rp['host_rp_uuid'] + hv_size = vmware_providers[host_rp_uuid]['hv_size'] + found_hv_sizes_per_vc[rp['vc']].add(hv_size) + + # if there are no resources in that resource-provider, it means, + # that we started freeing up a host. We have to check the process + # state and add the resources once it's done. + if not rp['inventory'].get(BIGVM_RESOURCE): + cm = rp['cell_mapping'] + with nova_context.target_cell(context, cm) as cctxt: + state = self.special_spawn_rpc.free_host(cctxt, rp['host']) + + if state == special_spawning.FREE_HOST_STATE_DONE: + self._add_resources_to_provider(context, rp_uuid, rp) + elif state == special_spawning.FREE_HOST_STATE_ERROR: + LOG.warning('Freeing a host for spawning failed on ' + '%(host)s.', + {'host': rp['host']}) + # do some cleanup, so another compute-node is used + found_hv_sizes_per_vc[rp['vc']].remove(hv_size) + self._clean_up_consumed_provider(context, rp_uuid, rp) + else: + LOG.info('Waiting for host on %(host)s to free up.', + {'host': rp['host']}) + + hv_sizes_per_vc = { + vc: set(rp['hv_size'] for rp in vmware_providers.values() + if rp['vc'] == vc) + for vc in vcenters} + + missing_hv_sizes_per_vc = { + vc: hv_sizes_per_vc[vc] - found_hv_sizes_per_vc[vc] + for vc in vcenters} + + return missing_hv_sizes_per_vc + + def _add_resources_to_provider(self, context, rp_uuid, rp): + """Add our custom resources to the provider so they can be consumed. + + This should be called once the host is freed up in the cluster. + """ + client = self.placement_client + inv_data = {BIGVM_RESOURCE: { + # we use 2 here so we can reserve 1 later. in queens we can't + # reserve $total + 'max_unit': 2, 'min_unit': 2, 'total': 2}} + + client._ensure_resource_provider( + context, rp_uuid, rp['rp']['name'], + parent_provider_uuid=rp['host_rp_uuid']) + try: + client.set_inventory_for_provider(context, rp_uuid, inv_data) + except Exception as err: + LOG.error('Adding inventory to the resource-provider for ' + 'spawning on %(host)s failed: %(err)s', + {'host': rp['host'], 'err': err}) + else: + LOG.info('Added inventory to the resource-provider for spawning ' + 'on %(host)s.', + {'host': rp['host']}) + + def _free_host_for_provider(self, context, rp_uuid, host): + """Takes care of creating a child resource provider in placement to + "claim" a resource-provider/host for freeing up a host. Then calls the + driver to actually free up the host in the cluster. + """ + client = self.placement_client + needs_cleanup = True + new_rp_uuid = None + try: + # TODO(jkulik) try to reserve the necessary memory for freeing a + # full hypervisor + + # create a child resource-provider + new_rp_name = '{}-{}'.format(CONF.bigvm_deployment_rp_name_prefix, + host) + # this is basically copied from placement client, but we don't want + # to set the uuid manually which it doesn't support + url = "/resource_providers" + payload = { + 'name': new_rp_name, + 'parent_provider_uuid': rp_uuid + } + resp = client.post(url, payload, + version=NESTED_PROVIDER_API_VERSION, + global_request_id=context.global_id) + placement_req_id = get_placement_request_id(resp) + if resp.status_code == 201: + new_rp_uuid = resp.headers['Location'].split('/')[-1] + msg = ("[%(placement_req_id)s] Created resource provider " + "record via placement API for host %(host)s for " + "special spawning: %(rp_uuid)s") + args = { + 'host': host, + 'placement_req_id': placement_req_id, + 'rp_uuid': new_rp_uuid + } + LOG.info(msg, args) + else: + msg = ("[%(placement_req_id)s] Failed to create resource " + "provider record in placement API for %(host)s for " + "special spawning. Got %(status_code)d: %(err_text)s.") + args = { + 'host': host, + 'status_code': resp.status_code, + 'err_text': resp.text, + 'placement_req_id': placement_req_id, + } + LOG.error(msg, args) + raise exception.ResourceProviderCreationFailed( + name=new_rp_name) + # make sure the placement cache is filled + client.get_provider_tree_and_ensure_root(context, new_rp_uuid, + new_rp_name) + + # Remove the following once elektra doesn't look at aggregates + # anymore to show spawnable bigvm flavors: + + # ensure the parent resource-provider has its uuid as aggregate set + # in addition to its previous aggregates + agg_info = client._get_provider_aggregates(context, rp_uuid) + if rp_uuid not in agg_info.aggregates: + agg_info.aggregates.add(rp_uuid) + client.set_aggregates_for_provider( + context, rp_uuid, agg_info.aggregates, + generation=agg_info.generation) + # add the newly-created resource-provider to the parent uuid's + # aggregate + client.set_aggregates_for_provider(context, new_rp_uuid, [rp_uuid]) + # Remove until here after elektra change. + + # find a host and let DRS free it up + state = self.special_spawn_rpc.free_host(context, host) + + if state == special_spawning.FREE_HOST_STATE_DONE: + # there were free resources available immediately + needs_cleanup = False + new_rp = {'host': host, + 'rp': {'name': new_rp_name}, + 'host_rp_uuid': rp_uuid} + self._add_resources_to_provider(context, new_rp_uuid, new_rp) + elif state == special_spawning.FREE_HOST_STATE_STARTED: + # it started working on it. we have to check back later + # if it's done + needs_cleanup = False + finally: + # clean up placement, if something went wrong + if needs_cleanup and new_rp_uuid is not None: + client._delete_provider(new_rp_uuid) + + return not needs_cleanup diff --git a/nova/cmd/bigvm.py b/nova/cmd/bigvm.py new file mode 100644 index 00000000000..0d39516d4d8 --- /dev/null +++ b/nova/cmd/bigvm.py @@ -0,0 +1,44 @@ +# Copyright 2019 SAP SE +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Starter script for BigVM manager.""" + +import sys + +from oslo_log import log as logging +from oslo_reports import guru_meditation_report as gmr +from oslo_reports import opts as gmr_opts + +import nova.conf +from nova import config +from nova import objects +from nova import service +from nova import version + +CONF = nova.conf.CONF + + +def main(): + config.parse_args(sys.argv) + logging.setup(CONF, "nova") + objects.register_all() + gmr_opts.set_defaults(CONF) + objects.Service.enable_min_version_cache() + + gmr.TextGuruMeditation.setup_autorun(version, conf=CONF) + + server = service.Service.create(binary='nova-bigvm') + service.serve(server) + service.wait() diff --git a/nova/conf/base.py b/nova/conf/base.py index e1750e4dcf6..2951becf846 100644 --- a/nova/conf/base.py +++ b/nova/conf/base.py @@ -117,6 +117,32 @@ For a couple of operations, e.g. scheduling decisions and special settings when spawning, we have to identify a big VM and handle them differently. Every VM having more or equal to this setting's amount of RAM is a big VM. +"""), + cfg.StrOpt( + 'bigvm_deployment_rp_name_prefix', + default='bigvm-deployment', + help=""" +This is the prefix used when creating resource-providers in placement for +handling spawning of VMs with special requirements like big VMs. The suffix of +the name will contain the nova-compute host. Prefix and suffix are joined by a +"-". +"""), + cfg.IntOpt( + 'prepare_empty_host_for_spawning_interval', + default=-1, + help=""" +Time in seconds between runs of the periodic task that frees up a host for +spawning VMs with special needs like big VMs. + +This is disabled by default, because it only makes sense for some setups. +"""), + cfg.IntOpt( + 'bigvm_cluster_max_usage_percent', + default=80, + help=""" +Clusters/resource-provider with this much usage are not used for freeing up a +host for spawning (a big VM). Clusters found to reach that amount, that already +have a host freed, get their free host removed. """), ] diff --git a/nova/conf/scheduler.py b/nova/conf/scheduler.py index b053c0366eb..1b377277f73 100644 --- a/nova/conf/scheduler.py +++ b/nova/conf/scheduler.py @@ -1065,6 +1065,33 @@ Possible values: * Any positive integer +"""), + cfg.BoolOpt("sharding_ignore_hana", + default=True, + help=""" +Ignore hana and/or BigVM flavors in the ShardFilter. +Enabling this option will allow such flavors to be placed on any shard, +regardless the shards that were enabled for the project. + +While this is set to True, individual projects may still have the +``use_individual_shard_tags_hana`` tag set, which will disable it for +that project. +"""), + cfg.StrOpt("hana_detection_strategy", + default="hana_exclusive_host", + help=""" +How to determine if a flavor is a Hana/BigVM to be ignored by the ShardFilter. + +Possible values are: + +* ``hana_exclusive_host`` + it will look for the trait:CUSTOM_HANA_EXCLUSIVE_HOST=required +* ``memory_mb`` + it will compare flavor.memory_mb >= CONF.bigvm_mb + +Related options: + +* ``[DEFAULT] bigvm_mb`` """), ] diff --git a/nova/conf/vmware.py b/nova/conf/vmware.py index 8a71b79aa67..c1169d5d336 100644 --- a/nova/conf/vmware.py +++ b/nova/conf/vmware.py @@ -388,6 +388,15 @@ Possible values: * Any string or empty to keep VMware default +"""), + cfg.StrOpt('bigvm_deployment_free_host_hostgroup_name', + default='', + help=""" +Name of the hostgroup used to free up a host for special spawning. + +All VMs in the cluster have to be in a VM group that has a rule specifying a +must-not-run-on-hostgroup rule for this hostgroup. Putting a host in this +hostgroup will then result in the host getting freed up by DRS. """), ] diff --git a/nova/objects/compute_node.py b/nova/objects/compute_node.py index dfc1b2ae284..789d1a10204 100644 --- a/nova/objects/compute_node.py +++ b/nova/objects/compute_node.py @@ -505,7 +505,8 @@ def get_all_by_uuids(cls, context, compute_uuids): @db.select_db_reader_mode def _db_compute_node_get_by_hv_type(context, hv_type): db_computes = context.session.query(models.ComputeNode).filter( - models.ComputeNode.hypervisor_type == hv_type).all() + models.ComputeNode.hypervisor_type == hv_type, + models.ComputeNode.deleted == 0).all() return db_computes @classmethod diff --git a/nova/objects/fields.py b/nova/objects/fields.py index 2a031074687..bc0bbaaae2f 100644 --- a/nova/objects/fields.py +++ b/nova/objects/fields.py @@ -904,9 +904,10 @@ class NotificationSource(BaseNovaEnum): # bumped to 3.0 CONSOLE = 'nova-console' METADATA = 'nova-metadata' + BIGVM = 'nova-bigvm' ALL = (API, COMPUTE, CONDUCTOR, SCHEDULER, - NETWORK, CONSOLEAUTH, CELLS, CONSOLE, METADATA) + NETWORK, CONSOLEAUTH, CELLS, CONSOLE, METADATA, BIGVM) @staticmethod def get_source_by_binary(binary): diff --git a/nova/scheduler/filters/shard_filter.py b/nova/scheduler/filters/shard_filter.py index bc9b5ff7f40..d6c4e73a719 100644 --- a/nova/scheduler/filters/shard_filter.py +++ b/nova/scheduler/filters/shard_filter.py @@ -19,6 +19,7 @@ from nova.scheduler.mixins import ProjectTagMixin from nova.scheduler import utils from nova import utils as nova_utils +from nova.utils import BIGVM_EXCLUSIVE_TRAIT LOG = logging.getLogger(__name__) @@ -35,8 +36,9 @@ class ShardFilter(filters.BaseHostFilter, ProjectTagMixin): """ _ALL_SHARDS = "sharding_enabled" + _HANA_USE_SHARDING = "use_individual_shard_tags_hana" _SHARD_PREFIX = 'vc-' - _PROJECT_TAG_TAGS = [_ALL_SHARDS] + _PROJECT_TAG_TAGS = [_ALL_SHARDS, _HANA_USE_SHARDING] _PROJECT_TAG_PREFIX = _SHARD_PREFIX def _get_shards(self, project_id): @@ -45,6 +47,14 @@ def _get_shards(self, project_id): # _get_shards() so it's clear what we return return self._get_tags(project_id) + def filter_all(self, filter_obj_list, spec_obj): + if self._ignore_hana_flavor(spec_obj): + LOG.debug("Hana/BigVM flavor requested. Ignoring sharding.") + return filter_obj_list + + return super(ShardFilter, self).filter_all( + filter_obj_list, spec_obj) + def host_passes(self, host_state, spec_obj): # Only VMware if utils.is_non_vmware_spec(spec_obj): @@ -94,3 +104,20 @@ def host_passes(self, host_state, spec_obj): 'host_shard': host_shard_names, 'project_shards': shards}) return False + + def _ignore_hana_flavor(self, spec_obj): + if not CONF.filter_scheduler.sharding_ignore_hana: + return False + + proj_tags = self._get_tags(spec_obj.project_id) + if self._HANA_USE_SHARDING in proj_tags: + return False + + strategy = CONF.filter_scheduler.hana_detection_strategy + extra_specs = spec_obj.flavor.extra_specs + + if strategy == "hana_exclusive_host": + trait = f"trait:{BIGVM_EXCLUSIVE_TRAIT}" + return extra_specs.get(trait) == "required" + elif strategy == "memory_mb": + return spec_obj.flavor.memory_mb >= CONF.bigvm_mb diff --git a/nova/service.py b/nova/service.py index 3107c113583..0e1aa55d7dc 100644 --- a/nova/service.py +++ b/nova/service.py @@ -60,6 +60,7 @@ 'nova-compute': 'nova.compute.manager.ComputeManager', 'nova-conductor': 'nova.conductor.manager.ConductorManager', 'nova-scheduler': 'nova.scheduler.manager.SchedulerManager', + 'nova-bigvm': 'nova.bigvm.manager.BigVmManager', } diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py index d45099ed9a7..8b6adca5837 100644 --- a/nova/tests/unit/notifications/objects/test_notification.py +++ b/nova/tests/unit/notifications/objects/test_notification.py @@ -426,7 +426,7 @@ def test_payload_is_not_generated_if_notification_format_is_unversioned( 'MetricPayload': '1.0-6a932f141c9ae2f50fc9d79c548338aa', 'MetricsNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'MetricsPayload': '1.0-a087851790cd7e76883bdc64a146917a', - 'NotificationPublisher': '2.2-b6ad48126247e10b46b6b0240e52e614', + 'NotificationPublisher': '2.2-ff8ef16673817ca7a3ea69c689e260c6', 'RequestSpecPayload': '1.1-9530d710bf7eaa101a93f6745fbe7aea', 'SchedulerRetriesPayload': '1.0-6e7e6204e638c0a070412f0e765c320c', 'SelectDestinationsNotification': '1.0-a73147b93b520ff0061865849d3dfa56', diff --git a/nova/tests/unit/scheduler/filters/test_shard_filter.py b/nova/tests/unit/scheduler/filters/test_shard_filter.py index 8c6edcce34e..41c7cd85808 100644 --- a/nova/tests/unit/scheduler/filters/test_shard_filter.py +++ b/nova/tests/unit/scheduler/filters/test_shard_filter.py @@ -16,11 +16,14 @@ from unittest import mock +import nova.conf from nova import objects from nova.scheduler.filters import shard_filter from nova import test from nova.tests.unit.scheduler import fakes +CONF = nova.conf.CONF + class TestShardFilter(test.NoDBTestCase): @@ -234,3 +237,69 @@ def test_non_vmware_spec(self, mock_is_non_vmware_spec): self.assertTrue(self.filt_cls.host_passes(host, spec_obj)) mock_is_non_vmware_spec.assert_called_once_with(spec_obj) + + def test_ignores_hana_by_trait(self): + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'required'} + self._shard_filter_ignoring_hana(True, extra_specs) + + def test_ignores_hana_by_memory(self): + CONF.set_override('bigvm_mb', 8192) + CONF.set_override('hana_detection_strategy', 'memory_mb', + 'filter_scheduler') + extra_specs = {} + self._shard_filter_ignoring_hana(True, + extra_specs=extra_specs, + memory_mb=8192) + + def test_sharding_ignore_hana_config(self): + CONF.set_override('sharding_ignore_hana', False, 'filter_scheduler') + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'required'} + self._shard_filter_ignoring_hana(False, + extra_specs=extra_specs) + + def test_use_individual_shard_tags_hana_tag(self): + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'required'} + tag = 'use_individual_shard_tags_hana' + self._shard_filter_ignoring_hana(False, + extra_specs=extra_specs, + project_tag=tag) + + def test_allow_hana_by_trait_forbidden(self): + extra_specs = {'trait:CUSTOM_HANA_EXCLUSIVE_HOST': 'forbidden'} + self._shard_filter_ignoring_hana(False, extra_specs) + + def test_allow_hana_by_memory(self): + CONF.set_override('bigvm_mb', 8192) + CONF.set_override('hana_detection_strategy', 'memory_mb', + 'filter_scheduler') + extra_specs = {} + self._shard_filter_ignoring_hana(False, + extra_specs=extra_specs, + memory_mb=8191) + + def _shard_filter_ignoring_hana(self, expect_to_ignore, extra_specs=None, + project_tag=None, memory_mb=4096): + aggs = [objects.Aggregate(id=1, name='some-az-a', hosts=['host1', + 'host2']), + objects.Aggregate(id=1, name='vc-a-1', hosts=['host1'])] + + host_allow = fakes.FakeHostState( + 'host1', 'compute', {'aggregates': aggs}) + host_forbid = fakes.FakeHostState( + 'host2', 'compute', {'aggregates': aggs[:1]}) + + spec_obj = objects.RequestSpec( + context=mock.sentinel.ctx, project_id='foo', + flavor=objects.Flavor(extra_specs=extra_specs, + memory_mb=memory_mb)) + + self.filt_cls._PROJECT_TAG_CACHE['foo'] = ['vc-a-0', 'vc-a-1', + 'vc-b-0'] + if project_tag: + self.filt_cls._PROJECT_TAG_CACHE['foo'].append(project_tag) + result = list(self.filt_cls.filter_all( + [host_allow, host_forbid], spec_obj)) + if expect_to_ignore: + self.assertEqual(result, [host_allow, host_forbid]) + else: + self.assertEqual(result, [host_allow]) diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index 26d7d41eb46..0f3e58bb38d 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -1078,21 +1078,20 @@ def test_resize_vm(self, fake_resize_spec, fake_reconfigure, @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') + @mock.patch.object(vmops.VMwareVMOps, '_clean_up_after_special_spawning') @mock.patch.object(vm_util, 'reconfigure_vm') @mock.patch.object(vm_util, 'get_vm_resize_spec', return_value='fake-spec') - @mock.patch.object(utils, 'is_big_vm') @mock.patch.object(cluster_util, 'update_cluster_drs_vm_override') - def test_resize_vm_bigvm_upsize(self, fake_drs_override, fake_is_big_vm, - fake_resize_spec, fake_reconfigure, + def test_resize_vm_bigvm_upsize(self, fake_drs_override, fake_resize_spec, + fake_reconfigure, + fake_cleanup_after_special_spawning, fake_get_extra_specs, fake_get_metadata): - # new is big, new is big, old is not - fake_is_big_vm.side_effect = [True, False] extra_specs = vm_util.ExtraSpecs() fake_get_extra_specs.return_value = extra_specs fake_get_metadata.return_value = self._metadata - flavor = objects.Flavor(name='m1.small', - memory_mb=1024, + flavor = objects.Flavor(name='bigvm-test', + memory_mb=CONF.bigvm_mb, vcpus=2, extra_specs={}) instance = self._instance.obj_clone() @@ -1105,19 +1104,20 @@ def test_resize_vm_bigvm_upsize(self, fake_drs_override, fake_is_big_vm, 'vm-ref', operation='add', behavior=behavior) + expected = (self._context, int(flavor.memory_mb), flavor) + fake_cleanup_after_special_spawning.assert_called_once_with(*expected) @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') + @mock.patch.object(vmops.VMwareVMOps, '_clean_up_after_special_spawning') @mock.patch.object(vm_util, 'reconfigure_vm') @mock.patch.object(vm_util, 'get_vm_resize_spec', return_value='fake-spec') - @mock.patch.object(utils, 'is_big_vm') @mock.patch.object(cluster_util, 'update_cluster_drs_vm_override') - def test_resize_vm_bigvm_downsize(self, fake_drs_override, fake_is_big_vm, + def test_resize_vm_bigvm_downsize(self, fake_drs_override, fake_resize_spec, fake_reconfigure, + fake_cleanup_after_special_spawning, fake_get_extra_specs, fake_get_metadata): - # new is not big, new is not big, old is big - fake_is_big_vm.side_effect = [False, True] extra_specs = vm_util.ExtraSpecs() fake_get_extra_specs.return_value = extra_specs fake_get_metadata.return_value = self._metadata @@ -1127,12 +1127,15 @@ def test_resize_vm_bigvm_downsize(self, fake_drs_override, fake_is_big_vm, extra_specs={}) instance = self._instance.obj_clone() instance.old_flavor = instance.flavor.obj_clone() + instance.old_flavor.memory_mb = CONF.bigvm_mb self._vmops._resize_vm(self._context, instance, 'vm-ref', flavor, None) fake_drs_override.assert_called_once_with(self._session, self._cluster.obj, 'vm-ref', operation='remove') + expected = (self._context, int(flavor.memory_mb), flavor) + fake_cleanup_after_special_spawning.assert_called_once_with(*expected) def test_reserve_all_memory_for_memory_reserved_flavor(self): self.flags(group='vmware', reserve_all_memory=False) @@ -1927,7 +1930,8 @@ def _verify_spawn_method_calls(self, mock_call_method, extras=None): recorded_methods = [c[1][1] for c in mock_call_method.mock_calls] self.assertEqual(expected_methods, recorded_methods) - @mock.patch('nova.utils.vm_needs_special_spawning') + @mock.patch('nova.utils.vm_needs_special_spawning', + return_value=False) @mock.patch('nova.compute.utils.is_volume_backed_instance', return_value=False) @mock.patch.object(vmops.VMwareVMOps, '_create_folders', diff --git a/nova/utils.py b/nova/utils.py index 93647519f2f..6c8de175fdc 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -85,6 +85,8 @@ MEMORY_RESERVABLE_MB_RESOURCE_SPEC_KEY = \ 'resources:' + MEMORY_RESERVABLE_MB_RESOURCE +BIGVM_EXCLUSIVE_TRAIT = 'CUSTOM_HANA_EXCLUSIVE_HOST' + _FILE_CACHE = {} _SERVICE_TYPES = service_types.ServiceTypes() diff --git a/nova/virt/vmwareapi/constants.py b/nova/virt/vmwareapi/constants.py index 5bd5ba96dbd..f75ac167ce8 100644 --- a/nova/virt/vmwareapi/constants.py +++ b/nova/virt/vmwareapi/constants.py @@ -53,6 +53,7 @@ DEFAULT_DISK_FORMAT = DISK_FORMAT_VMDK DEFAULT_CONTAINER_FORMAT = CONTAINER_FORMAT_BARE +DRS_BEHAVIOR_FULLY_AUTOMATED = 'fullyAutomated' DRS_BEHAVIOR_PARTIALLY_AUTOMATED = 'partiallyAutomated' IMAGE_VM_PREFIX = "OSTACK_IMG" diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index ccc5604c28c..25b95acd680 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -57,6 +57,7 @@ from nova.virt.vmwareapi import host from nova.virt.vmwareapi.rpc import VmwareRpcService from nova.virt.vmwareapi import session +from nova.virt.vmwareapi import special_spawning from nova.virt.vmwareapi import vim_util as nova_vim_util from nova.virt.vmwareapi import vm_util from nova.virt.vmwareapi import vmops @@ -169,6 +170,7 @@ def __init__(self, virtapi, scheme="https"): self._register_openstack_extension() virtapi._compute.additional_endpoints.extend([ + special_spawning._SpecialVmSpawningServer(self), VmwareRpcService(self)]) def _check_min_version(self): diff --git a/nova/virt/vmwareapi/special_spawning.py b/nova/virt/vmwareapi/special_spawning.py new file mode 100644 index 00000000000..fff6c469f8c --- /dev/null +++ b/nova/virt/vmwareapi/special_spawning.py @@ -0,0 +1,360 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# Copyright (c) 2012 VMware, Inc. +# Copyright (c) 2011 Citrix Systems, Inc. +# Copyright 2011 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from operator import itemgetter + +from oslo_log import log as logging +import oslo_messaging as messaging +from oslo_vmware import vim_util as vutil + +from nova.compute import rpcapi +import nova.conf +from nova import exception +from nova import profiler +from nova import rpc +from nova.virt.vmwareapi import cluster_util +from nova.virt.vmwareapi import constants +from nova.virt.vmwareapi import vim_util +from nova.virt.vmwareapi.vm_util import propset_dict + +LOG = logging.getLogger(__name__) +CONF = nova.conf.CONF +FREE_HOST_STATE_DONE = 'done' +FREE_HOST_STATE_ERROR = 'error' +FREE_HOST_STATE_STARTED = 'started' + +BIGVM_RESOURCE = 'CUSTOM_BIGVM' + + +@profiler.trace_cls("rpc") +class SpecialVmSpawningInterface(object): + """RPC client foundr calling _SpecialVmSpawningServer""" + + def __init__(self): + super(SpecialVmSpawningInterface, self).__init__() + target = messaging.Target(topic=rpcapi.RPC_TOPIC, version='5.0') + self.router = rpc.ClientRouter(rpc.get_client(target)) + + def remove_host_from_hostgroup(self, ctxt, compute_host_name): + version = '5.0' + cctxt = self.router.client(ctxt).prepare( + server=compute_host_name, version=version) + return cctxt.call(ctxt, 'remove_host_from_hostgroup') + + def free_host(self, ctxt, compute_host_name): + version = '5.0' + cctxt = self.router.client(ctxt).prepare( + server=compute_host_name, version=version) + return cctxt.call(ctxt, 'free_host') + + +class _SpecialVmSpawningServer(object): + """Additional RPC interface for handling special spawning needs.""" + + target = messaging.Target(version='5.0') + + def __init__(self, driver): + self._driver = driver + self._session = driver._session + self._cluster = driver._cluster_ref + self._vmops = driver._vmops + + def _get_group(self, cluster_config=None): + """Return the hostgroup or None if not found.""" + if cluster_config is None: + cluster_config = self._session._call_method( + vutil, "get_object_property", self._cluster, "configurationEx") + if not cluster_config: + # that should never happen. we should not proceed with whatever + # called us + msg = 'Cluster {} must have an attribute "configurationEx".' + raise exception.ValidationError(msg.format(self._cluster)) + + group_ret = getattr(cluster_config, 'group', None) + if not group_ret: + return None + + hg_name = CONF.vmware.bigvm_deployment_free_host_hostgroup_name + if not hg_name: + raise exception.ValidationError('Function for special spawning ' + 'were called, but the setting ' + '"bigvm_deployment_free_host_hostgroup_name" is unconfigured.') + + for group in group_ret: + # we are only interested in one special group + if group.name == hg_name: + return group + + def _get_hosts_in_cluster(self, cluster_ref): + """Return a list of HostSystem morefs belonging to the cluster""" + result = self._session._call_method( + vim_util, 'get_inner_objects', cluster_ref, 'host', 'HostSystem') + with vutil.WithRetrieval(self._session.vim, result) as objects: + return [obj.obj for obj in objects] + + def _get_vms_on_host(self, host_ref): + """Return a list of VMs uuids with their memory size and state""" + vm_data = [] + vm_ret = self._session._call_method(vutil, + "get_object_property", + host_ref, + "vm") + # if there are no VMs on the host, we don't need to look further + if not vm_ret: + return vm_data + + vm_mors = vm_ret.ManagedObjectReference + result = self._session._call_method(vutil, + "get_properties_for_a_collection_of_objects", + "VirtualMachine", vm_mors, + ["config.instanceUuid", "runtime.powerState", + "config.hardware.memoryMB", "config.managedBy"]) + with vutil.WithRetrieval(self._session.vim, result) as objects: + for obj in objects: + vm_props = propset_dict(obj.propSet) + # sometimes, the vCenter finds a file it thinks is a VM and it + # doesn't even have a config attribute ... instead of crashing + # with a KeyError, we assume this VM is not running and totally + # doesn't matter as nova also will not be able to handle it + if 'config.instanceUuid' not in vm_props: + continue + + vm_data.append(( + vm_props['config.instanceUuid'], + vm_props['config.hardware.memoryMB'], + vm_props['runtime.powerState'], + vm_props.get('config.managedBy'), + vutil.get_moref_value(obj.obj))) + return vm_data + + def remove_host_from_hostgroup(self, context): + """Search for the host in the special spawning hostgroup and remove + that group, because emptying it seems not to work. + """ + group = self._get_group() + # no group -> nothing to do + if group is None: + return True + + # if there are no hosts in the group, we're done + if not getattr(group, 'host', None): + return True + + client_factory = self._session.vim.client.factory + + group_spec = cluster_util.create_group_spec(client_factory, group, + 'remove') + # if there are rules using this hostgroup, we also have to remove that + # one, because that rule will not update inside of DRS as it's invalid + # if we remove the group + rule_specs = [] + rules = cluster_util.fetch_cluster_rules(self._session, self._cluster) + for rule in rules: + attrs = ('antiAffineHostGroupName', 'affineHostGroupName') + if all(getattr(rule, attr, None) != group.name for attr in attrs): + continue + rule_spec = \ + cluster_util.create_rule_spec(client_factory, rule, 'remove') + rule_specs.append(rule_spec) + + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + config_spec.groupSpec = [group_spec] + config_spec.rulesSpec = rule_specs + cluster_util.reconfigure_cluster(self._session, self._cluster, + config_spec) + + return True + + def free_host(self, context): + """Find a host that doesn't have a bigvm and put it into the special + hostgroup. If that's already the case, return whether there are running + VMs left on the host, i.e. the process is finished. + """ + cluster_config = self._session._call_method( + vutil, "get_object_property", self._cluster, "configurationEx") + + # get the group + group = self._get_group(cluster_config) + + failover_hosts = [] + policy = cluster_config.dasConfig.admissionControlPolicy + if policy and hasattr(policy, 'failoverHosts'): + failover_hosts = set(vutil.get_moref_value(h) + for h in policy.failoverHosts) + + if group is None or not getattr(group, 'host', None): + # find a host to free + + # retrieve all hosts of the cluster + host_objs = {vutil.get_moref_value(h): h + for h in self._get_hosts_in_cluster(self._cluster)} + vms_per_host = {h: [] for h in host_objs} + + # get all the vms in a cluster, because we need to find a host + # without big VMs. + props = ['config.hardware.memoryMB', 'runtime.host', + 'runtime.powerState', + 'summary.quickStats.hostMemoryUsage'] + cluster_vms = self._vmops._list_instances_in_cluster(props) + + for vm_uuid, vm_props in cluster_vms: + props = (vm_props.get('config.hardware.memoryMB', 0), + vm_props.get('runtime.powerState', 'poweredOff'), + vm_props.get('summary.quickStats.hostMemoryUsage', 0)) + # every host_obj is different, even though the value, which + # really matters, is the same + host_obj = vm_props.get('runtime.host') + if not host_obj: + continue + + host = vutil.get_moref_value(host_obj) + vms_per_host.setdefault(host, []). \ + append(props) + + # filter for hosts without big VMs + vms_per_host = {h: vms for h, vms in vms_per_host.items() + if all(mem < CONF.bigvm_mb + for mem, state, used_mem in vms)} + + if not vms_per_host: + LOG.warning('No suitable host found for freeing a host for ' + 'spawning (bigvm filter).') + return FREE_HOST_STATE_ERROR + + # filter hosts which are failover hosts + vms_per_host = {h: vms for h, vms in vms_per_host.items() + if h not in failover_hosts} + + if not vms_per_host: + LOG.warning('No suitable host found for freeing a host for ' + 'spawning (failover host filter).') + return FREE_HOST_STATE_ERROR + + # filter hosts which are in a wrong state + result = self._session._call_method(vim_util, + "get_properties_for_a_collection_of_objects", + "HostSystem", + [host_objs[h] for h in vms_per_host], + ['summary.runtime']) + host_states = {} + with vutil.WithRetrieval(self._session.vim, result) as objects: + for obj in objects: + host_props = propset_dict(obj.propSet) + runtime_summary = host_props['summary.runtime'] + moref_value = vutil.get_moref_value(obj.obj) + host_states[moref_value] = ( + runtime_summary.inMaintenanceMode is False and + runtime_summary.connectionState == "connected") + + vms_per_host = {h: vms for h, vms in vms_per_host.items() + if host_states[h]} + + if not vms_per_host: + LOG.warning('No suitable host found for freeing a host for ' + 'spawning (host state filter).') + return FREE_HOST_STATE_ERROR + + mem_per_host = {h: sum(used_mem for mem, state, used_mem in vms) + for h, vms in vms_per_host.items()} + + # take the one with least memory used + host, _ = sorted(mem_per_host.items(), key=itemgetter(1))[0] + host_ref = host_objs[host] + + client_factory = self._session.vim.client.factory + config_spec = client_factory.create('ns0:ClusterConfigSpecEx') + + # we need to either create the group from scratch or at least add a + # host to it + operation = 'add' if group is None else 'edit' + group = cluster_util.create_host_group(client_factory, + CONF.vmware.bigvm_deployment_free_host_hostgroup_name, + [host_ref], group) + group_spec = cluster_util.create_group_spec(client_factory, + group, operation) + config_spec.groupSpec = [group_spec] + + # create the appropriate rule for VMs to leave the host + rule_name = '{}_anti-affinity'.format( + CONF.vmware.bigvm_deployment_free_host_hostgroup_name) + rule = cluster_util._get_rule(cluster_config, rule_name) + rule_spec = cluster_util._create_cluster_group_rules_spec( + client_factory, rule_name, + CONF.vmware.special_spawning_vm_group, + CONF.vmware.bigvm_deployment_free_host_hostgroup_name, + 'anti-affinity', rule) + config_spec.rulesSpec = [rule_spec] + + cluster_util.reconfigure_cluster(self._session, self._cluster, + config_spec) + else: + if len(group.host) > 1: + LOG.warning('Found more than 1 host in spawning hostgroup.') + host_ref = group.host[0] + + # check if the host is still suitable + if vutil.get_moref_value(host_ref) in failover_hosts: + LOG.warning('Host destined for spawning became a failover ' + 'host.') + return FREE_HOST_STATE_ERROR + + runtime_summary = self._session._call_method( + vutil, "get_object_property", host_ref, 'summary.runtime') + if (runtime_summary.inMaintenanceMode is True or + runtime_summary.connectionState != "connected"): + LOG.warning('Host destined for spawning was set to ' + 'maintenance or became disconnected.') + return FREE_HOST_STATE_ERROR + + # filter the VMs on the host, so we don't look at the non-movable + # DRS-created and -owned VMs + vcls_identifier = (constants.VCLS_EXTENSION_KEY, + constants.VCLS_EXTENSION_TYPE_AGENT) + vms_on_host = [ + (u, state, ref) + for u, h, state, m, ref in self._get_vms_on_host(host_ref) + if not m or (m.extensionKey, m.type) != vcls_identifier] + + # ignore partiallyAutomated VMs. They should not be big VMs as we chose + # a host appropriately, so they should be large VMs, which we tolerate + # next to a big VM as they get moved by the nanny + drs_overrides = cluster_util.fetch_cluster_drs_vm_overrides( + self._session, cluster_config=cluster_config) + wanted_drs_override_behaviors = ( + constants.DRS_BEHAVIOR_PARTIALLY_AUTOMATED,) + vms_on_host = [ + (u, state) for u, state, ref in vms_on_host + if drs_overrides.get(ref) not in wanted_drs_override_behaviors] + + # check if there are running VMs on that host + vms_on_host = [u for u, state in vms_on_host + if state != 'poweredOff'] + if vms_on_host: + # check if DRS is enabled, so freeing up can work + if cluster_config.drsConfig.defaultVmBehavior != \ + constants.DRS_BEHAVIOR_FULLY_AUTOMATED: + LOG.error('DRS set to %(actual)s, expected %(expected)s.', + {'actual': cluster_config.drsConfig.defaultVmBehavior, + 'expected': constants.DRS_BEHAVIOR_FULLY_AUTOMATED}) + return FREE_HOST_STATE_ERROR + + LOG.debug('Freeing up %(host)s for spawning in progress.', + {'host': vutil.get_moref_value(host_ref)}) + return FREE_HOST_STATE_STARTED + + LOG.info('Done freeing up %(host)s for spawning.', + {'host': vutil.get_moref_value(host_ref)}) + return FREE_HOST_STATE_DONE diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index ea9fc4832e5..57d9854335a 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -62,6 +62,7 @@ from nova.virt.vmwareapi import imagecache from nova.virt.vmwareapi import images from nova.virt.vmwareapi.rpc import VmwareRpcApi +from nova.virt.vmwareapi import special_spawning from nova.virt.vmwareapi import vif as vmwarevif from nova.virt.vmwareapi import vim_util from nova.virt.vmwareapi import vm_util @@ -1049,6 +1050,41 @@ def spawn(self, context, instance, image_meta, injected_files, vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) + self._clean_up_after_special_spawning(context, instance.memory_mb, + instance.flavor) + + def _clean_up_after_special_spawning(self, context, instance_memory_mb, + instance_flavor): + if utils.vm_needs_special_spawning(int(instance_memory_mb), + instance_flavor): + # we're using a child resource provider, so we don't have to change + # the drivers' report-code to keep the CUSTOM_BIGVM resource, but + # instead can independently add it or remove it on a cluster + placement_client = self._virtapi._compute.reportclient + cn = self._virtapi._compute._get_compute_info(context, CONF.host) + parent_rp_uuid = cn.uuid + parent_tree = placement_client.get_provider_tree_and_ensure_root( + context, parent_rp_uuid) + rp_name = '{}-{}'.format(CONF.bigvm_deployment_rp_name_prefix, + cn.host) + try: + rp = parent_tree.data(rp_name) + except ValueError: + LOG.warning('Could not find resource-provider %(rp)s for ' + 'reserving resources after (re)starting a big VM.', + {'rp': rp_name}) + else: + # we need to update the inventory of the bigvm provider in our + # cache, because the generation might be too old after the + # allocations of our currently spawning big vm + placement_client._refresh_and_get_inventory(context, rp.uuid) + # reserve the bigvm resource. this prohibits any further + # deployment needing a free host on that compute-node. + inv_data = rp.inventory + inv_data[special_spawning.BIGVM_RESOURCE]['reserved'] = 1 + placement_client.set_inventory_for_provider(context, rp.uuid, + inv_data) + def _is_bdm_valid(self, block_device_mapping): """Checks if the block device mapping is valid.""" valid_bus = (constants.DEFAULT_ADAPTER_TYPE, @@ -1694,8 +1730,8 @@ def _resize_vm(self, context, instance, vm_ref, flavor, image_meta): vm_util.reconfigure_vm(self._session, vm_ref, vm_resize_spec) old_flavor = instance.old_flavor - new_is_big = utils.is_big_vm(int(old_flavor.memory_mb), old_flavor) - old_is_big = utils.is_big_vm(int(flavor.memory_mb), flavor) + old_is_big = utils.is_big_vm(int(old_flavor.memory_mb), old_flavor) + new_is_big = utils.is_big_vm(int(flavor.memory_mb), flavor) if not old_is_big and new_is_big: # Make sure we don't automatically move around "big" VMs @@ -1721,6 +1757,9 @@ def _resize_vm(self, context, instance, vm_ref, flavor, image_meta): LOG.exception('Could not remove DRS override.', instance=instance) + self._clean_up_after_special_spawning(context, flavor.memory_mb, + flavor) + def _resize_disk(self, instance, vm_ref, vmdk, flavor): extra_specs = self._get_extra_specs(instance.flavor, instance.image_meta) diff --git a/setup.cfg b/setup.cfg index 1b878af0912..8ca42f584c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,6 +77,7 @@ console_scripts = nova-api = nova.cmd.api:main nova-api-metadata = nova.cmd.api_metadata:main nova-api-os-compute = nova.cmd.api_os_compute:main + nova-bigvm = nova.cmd.bigvm:main nova-compute = nova.cmd.compute:main nova-conductor = nova.cmd.conductor:main nova-manage = nova.cmd.manage:main From 955bb115d685dca0de4be19b0204c55ce15f5632 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Wed, 11 Aug 2021 14:25:52 +0200 Subject: [PATCH 88/92] Vmware: Use WithRetrieval to get all results In various places, own version of iterating over the results are implemented, sometimes even faulty. The following functions were only getting up to vmware.maximum_objects objects (100 by default) vm_util.get_all_cluster_mors, vm_util.get_stats_from_cluster Previously, the results were fetched in batches of up to vmware.maximum_objects items. Using WithRetrieval yields an iterator to the results, which pages transparently to the next request. Receiver of the output of the results where changed to consume an iterator, where easily possible. Replace the quadratic algorithm in `ds_util._filter_datastores_matching_storage_policy` with one of O(n log(n)) runtime (cherry picked from commit fcabb04fbef894d46d1da5b535a2a7b053b1c248) Change-Id: I2ac02a0cf4569398455dd63cfa4e2380a71b8ab9 (cherry picked from commit 84ee36d7f2a3ab4a408b3d33bacf0b1f2c91e368) --- .../tests/unit/virt/vmwareapi/test_ds_util.py | 4 ++-- nova/virt/vmwareapi/ds_util.py | 14 +++++------- nova/virt/vmwareapi/vm_util.py | 22 +++++++++---------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_ds_util.py b/nova/tests/unit/virt/vmwareapi/test_ds_util.py index 1716027afb0..66fea4dbd67 100644 --- a/nova/tests/unit/virt/vmwareapi/test_ds_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_ds_util.py @@ -289,9 +289,9 @@ def next_datastore(*args, **kwargs): return None with mock.patch.object(self.session, '_call_method', - side_effect=fake_call_method), \ + side_effect=fake_call_method), \ mock.patch('oslo_vmware.vim_util.continue_retrieval', - side_effect=next_datastore), \ + side_effect=next_datastore), \ mock.patch('oslo_vmware.vim_util.cancel_retrieval'): yield diff --git a/nova/virt/vmwareapi/ds_util.py b/nova/virt/vmwareapi/ds_util.py index 8dfb3532cb1..391fb535a91 100644 --- a/nova/virt/vmwareapi/ds_util.py +++ b/nova/virt/vmwareapi/ds_util.py @@ -51,7 +51,7 @@ def _select_datastore(session, datastores, best_match, datastore_regex=None, """Find the most preferable datastore in a given RetrieveResult object. :param session: vmwareapi session - :param datastores: an iterator to the objects of RetrieveResult + :param datastores: an iterator to the results of the datastore list :param best_match: the current best match for datastore :param datastore_regex: an optional regular expression to match names :param storage_policy: storage policy for the datastore @@ -62,12 +62,10 @@ def _select_datastore(session, datastores, best_match, datastore_regex=None, if storage_policy: matching_ds = _filter_datastores_matching_storage_policy( session, datastores, storage_policy) - if not matching_ds: - return best_match else: matching_ds = datastores - # matching_ds is actually a RetrieveResult object from vSphere API call + # data_stores is actually a RetrieveResult object from vSphere API call for obj_content in matching_ds: # the propset attribute "need not be set" by returning API if not hasattr(obj_content, 'propSet'): @@ -141,7 +139,6 @@ def get_datastore(session, cluster, datastore_regex=None, datastore_regex, storage_policy, allowed_ds_types) - if best_match: return best_match @@ -149,12 +146,11 @@ def get_datastore(session, cluster, datastore_regex=None, raise exception.DatastoreNotFound( _("Storage policy %s did not match any datastores") % storage_policy) - elif datastore_regex: + if datastore_regex: raise exception.DatastoreNotFound( _("Datastore regex %s did not match any datastores") % datastore_regex.pattern) - else: - raise exception.DatastoreNotFound() + raise exception.DatastoreNotFound() def _get_allowed_datastores(datastores, datastore_regex): @@ -439,7 +435,7 @@ def _filter_datastores_matching_storage_policy(session, datastores, storage_policy): """Get datastores matching the given storage policy. - :param datastores: an iterator over objects of a RetrieveResult + :param datastores: an iterator over datastore results :param storage_policy: the storage policy name :return: an iterator to datastores conforming to the given storage policy """ diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index c13c45151d9..0a80a767ab8 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -1427,8 +1427,8 @@ def get_stats_from_cluster(session, cluster): max_mem_mb_per_host = 0 reserved_memory_mb = 0 # Get the Host and Resource Pool Managed Object Refs - props = ["host", "resourcePool", - "configuration.dasConfig.admissionControlPolicy"] + admission_policy_key = "configuration.dasConfig.admissionControlPolicy" + props = ["host", "resourcePool", admission_policy_key] if CONF.vmware.hostgroup_reservations_json_file: props.append("configurationEx") prop_dict = session._call_method(vutil, @@ -1437,8 +1437,7 @@ def get_stats_from_cluster(session, cluster): props) if prop_dict: failover_hosts = [] - key = 'configuration.dasConfig.admissionControlPolicy' - policy = prop_dict.get(key) + policy = prop_dict.get(admission_policy_key) if policy and hasattr(policy, 'failoverHosts'): failover_hosts = set(h.value for h in policy.failoverHosts) @@ -1454,11 +1453,18 @@ def get_stats_from_cluster(session, cluster): "HostSystem", host_mors, ["summary.hardware", "summary.runtime", "summary.quickStats"]) + total_hypervisor_count = 0 + # NOTE (jakobk): For the total amount of hosts it doesn't matter + # whether the host is in MM or unreachable, because the count is + # used to calculate safety margins for resource allocations, and MM + # or otherwise unreachable hosts is precisely what that is supposed + # to guard against. with vutil.WithRetrieval(session.vim, result) as objects: for obj in objects: + total_hypervisor_count += 1 host_props = propset_dict(obj.propSet) runtime_summary = host_props['summary.runtime'] - if (runtime_summary.inMaintenanceMode is not False or + if (runtime_summary.inMaintenanceMode or runtime_summary.connectionState != "connected"): continue hardware_summary = host_props['summary.hardware'] @@ -1479,12 +1485,6 @@ def get_stats_from_cluster(session, cluster): threads - reserved['vcpus']) max_mem_mb_per_host = max(max_mem_mb_per_host, mem_mb - reserved['memory_mb']) - # NOTE (jakobk): For the total amount of hosts it doesn't matter - # whether the host is in MM or unreachable, because the count is - # used to calculate safety margins for resource allocations, and MM - # or otherwise unreachable hosts is precisely what that is supposed - # to guard against. - total_hypervisor_count = len(result.objects) # Calculate VM-reservable memory as a ratio of total available # memory, depending on either the configured tolerance for failed From 51f6b59edf1c038fc9b9fd193e50d9362ebb4cea Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 12 Aug 2021 11:49:40 +0200 Subject: [PATCH 89/92] Vmwareapi: Split out getting hosts from get_stats_from_cluster This extracts getting the hosts and reservations from a cluster in get_stats_from_cluster into its own function get_hosts_and_reservations_for_cluster. For cross-vcenter vmotion, we need to specify hosts, and we want the same ones as we use elsewhere. (cherry picked from commit 05cd96bbb101487d32620711fe8d361b1acc13e5) Change-Id: I4f738e76553c3afefc735fb4b05f1a496c3c028f (cherry picked from commit fb3d5dbca35518c1bfa71fd77568d1c397501ee4) --- nova/virt/vmwareapi/vm_util.py | 154 ++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 0a80a767ab8..8c64b3d09e8 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -1426,6 +1426,73 @@ def get_stats_from_cluster(session, cluster): total_mem_mb = 0 max_mem_mb_per_host = 0 reserved_memory_mb = 0 + + host_mors, host_reservations_map = \ + get_hosts_and_reservations_for_cluster(session, cluster) + + if host_mors: + result = session._call_method(vim_util, + "get_properties_for_a_collection_of_objects", + "HostSystem", host_mors, + ["summary.hardware", "summary.runtime", + "summary.quickStats"]) + total_hypervisor_count = 0 + # NOTE (jakobk): For the total amount of hosts it doesn't matter + # whether the host is in MM or unreachable, because the count is + # used to calculate safety margins for resource allocations, and MM + # or otherwise unreachable hosts is precisely what that is supposed + # to guard against. + with vutil.WithRetrieval(session.vim, result) as objects: + for obj in objects: + total_hypervisor_count += 1 + host_props = propset_dict(obj.propSet) + runtime_summary = host_props['summary.runtime'] + if (runtime_summary.inMaintenanceMode or + runtime_summary.connectionState != "connected"): + continue + hardware_summary = host_props['summary.hardware'] + stats_summary = host_props['summary.quickStats'] + # Total vcpus is the sum of all pCPUs of individual hosts + # The overcommitment ratio is factored in by the scheduler + threads = hardware_summary.numCpuThreads + vcpus += threads + used_mem_mb += stats_summary.overallMemoryUsage + mem_mb = hardware_summary.memorySize // units.Mi + total_mem_mb += mem_mb + reserved = _get_host_reservations( + host_reservations_map, obj.obj, + threads, mem_mb) + reserved_vcpus += reserved['vcpus'] + reserved_memory_mb += reserved['memory_mb'] + max_vcpus_per_host = max(max_vcpus_per_host, + threads - reserved['vcpus']) + max_mem_mb_per_host = max(max_mem_mb_per_host, + mem_mb - reserved['memory_mb']) + + # Calculate VM-reservable memory as a ratio of total available + # memory, depending on either the configured tolerance for failed + # hypervisors or a single configurable ratio. + max_fail_hvs = \ + CONF.vmware.memory_reservation_cluster_hosts_max_fail + if max_fail_hvs and total_hypervisor_count: + vm_reservable_memory_ratio = \ + (1 - max_fail_hvs / total_hypervisor_count) + else: + vm_reservable_memory_ratio = \ + CONF.vmware.memory_reservation_max_ratio_fallback + + stats = {'cpu': {'vcpus': vcpus, + 'max_vcpus_per_host': max_vcpus_per_host, + 'reserved_vcpus': reserved_vcpus}, + 'mem': {'total': total_mem_mb, + 'free': total_mem_mb - used_mem_mb, + 'max_mem_mb_per_host': max_mem_mb_per_host, + 'reserved_memory_mb': reserved_memory_mb, + 'vm_reservable_memory_ratio': vm_reservable_memory_ratio}} + return stats + + +def get_hosts_and_reservations_for_cluster(session, cluster): # Get the Host and Resource Pool Managed Object Refs admission_policy_key = "configuration.dasConfig.admissionControlPolicy" props = ["host", "resourcePool", admission_policy_key] @@ -1435,77 +1502,24 @@ def get_stats_from_cluster(session, cluster): "get_object_properties_dict", cluster, props) - if prop_dict: - failover_hosts = [] - policy = prop_dict.get(admission_policy_key) - if policy and hasattr(policy, 'failoverHosts'): - failover_hosts = set(h.value for h in policy.failoverHosts) - - group_ret = getattr(prop_dict.get('configurationEx'), 'group', None) - host_reservations_map = _get_host_reservations_map(group_ret) - - host_ret = prop_dict.get('host') - if host_ret: - host_mors = [m for m in host_ret.ManagedObjectReference + if not prop_dict: + return None, None + + failover_hosts = [] + policy = prop_dict.get(admission_policy_key) + if policy and hasattr(policy, 'failoverHosts'): + failover_hosts = set(h.value for h in policy.failoverHosts) + + group_ret = getattr(prop_dict.get('configurationEx'), 'group', None) + + host_ret = prop_dict.get('host') + + if not host_ret: + return None, None + + host_mors = [m for m in host_ret.ManagedObjectReference if m.value not in failover_hosts] - result = session._call_method(vim_util, - "get_properties_for_a_collection_of_objects", - "HostSystem", host_mors, - ["summary.hardware", "summary.runtime", - "summary.quickStats"]) - total_hypervisor_count = 0 - # NOTE (jakobk): For the total amount of hosts it doesn't matter - # whether the host is in MM or unreachable, because the count is - # used to calculate safety margins for resource allocations, and MM - # or otherwise unreachable hosts is precisely what that is supposed - # to guard against. - with vutil.WithRetrieval(session.vim, result) as objects: - for obj in objects: - total_hypervisor_count += 1 - host_props = propset_dict(obj.propSet) - runtime_summary = host_props['summary.runtime'] - if (runtime_summary.inMaintenanceMode or - runtime_summary.connectionState != "connected"): - continue - hardware_summary = host_props['summary.hardware'] - stats_summary = host_props['summary.quickStats'] - # Total vcpus is the sum of all pCPUs of individual hosts - # The overcommitment ratio is factored in by the scheduler - threads = hardware_summary.numCpuThreads - vcpus += threads - used_mem_mb += stats_summary.overallMemoryUsage - mem_mb = hardware_summary.memorySize // units.Mi - total_mem_mb += mem_mb - reserved = _get_host_reservations( - host_reservations_map, obj.obj, - threads, mem_mb) - reserved_vcpus += reserved['vcpus'] - reserved_memory_mb += reserved['memory_mb'] - max_vcpus_per_host = max(max_vcpus_per_host, - threads - reserved['vcpus']) - max_mem_mb_per_host = max(max_mem_mb_per_host, - mem_mb - reserved['memory_mb']) - - # Calculate VM-reservable memory as a ratio of total available - # memory, depending on either the configured tolerance for failed - # hypervisors or a single configurable ratio. - max_fail_hvs = \ - CONF.vmware.memory_reservation_cluster_hosts_max_fail - if max_fail_hvs and total_hypervisor_count: - vm_reservable_memory_ratio = \ - (1 - max_fail_hvs / total_hypervisor_count) - else: - vm_reservable_memory_ratio = \ - CONF.vmware.memory_reservation_max_ratio_fallback - stats = {'cpu': {'vcpus': vcpus, - 'max_vcpus_per_host': max_vcpus_per_host, - 'reserved_vcpus': reserved_vcpus}, - 'mem': {'total': total_mem_mb, - 'free': total_mem_mb - used_mem_mb, - 'max_mem_mb_per_host': max_mem_mb_per_host, - 'reserved_memory_mb': reserved_memory_mb, - 'vm_reservable_memory_ratio': vm_reservable_memory_ratio}} - return stats + return host_mors, _get_host_reservations_map(group_ret) def get_host_ref(session, cluster=None): From 81fddbcd69f6a5df10e4a6d0e55468cebfb83c08 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Thu, 26 Aug 2021 13:41:40 +0200 Subject: [PATCH 90/92] VmWare: Refactored resource and inventory collection Joined the collection of the cpu-info with the host-stats in order to avoid calling twice the property collector in different places to get information over all the host. Split the code into getting the data and aggregating it over the cluster. Partly to split the logic into more easily consumable parts, but it is also a preparation for exposing the individual ESXi hosts as hypervisors. Host in certain states (such as disconnected ones) do not report `quickStats` about memory usage, so we have to assume values here, which is better than failing updating the stats for all hosts. Since they are not available, both the usage as well as the free memory will not be added to the total for the cluster Disconnected hosts also don't provide other properties, so we handle that gracefully by defaulting them to 0. Change-Id: I383854bb0e956519e3bdc42121b59d43ca54743d Incorporated-Change-Id: I05b3a158a058034d9a50a6948cdf302769f2d0eb Incorporated-Change-Id: I1c53477e891b5b95859ca267fcad8cd1bff260ef (cherry picked from commit f6279b5800bfb0c5f247a4780400e9e2a41afe82) --- nova/tests/unit/virt/vmwareapi/fake.py | 135 +++++++--- .../unit/virt/vmwareapi/test_driver_api.py | 51 +++- .../tests/unit/virt/vmwareapi/test_vm_util.py | 170 +++++++----- nova/virt/vmwareapi/driver.py | 153 +++++------ nova/virt/vmwareapi/host.py | 69 +++-- nova/virt/vmwareapi/vm_util.py | 252 ++++++++++++------ 6 files changed, 533 insertions(+), 297 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/fake.py b/nova/tests/unit/virt/vmwareapi/fake.py index fd717f56d74..5685d8ca6f8 100644 --- a/nova/tests/unit/virt/vmwareapi/fake.py +++ b/nova/tests/unit/virt/vmwareapi/fake.py @@ -742,10 +742,90 @@ def __init__(self): class HostSystem(ManagedObject): """Host System class.""" + NUM_CPU_CORES = 8 + NUM_CPU_PKGS = 2 + NUM_CPU_THREADS = 16 - def __init__(self, name="ha-host", connected=True, ds_ref=None, + @staticmethod + def create_summary_hardware(): + hardware = DataObject() + hardware.numCpuCores = HostSystem.NUM_CPU_CORES + hardware.numCpuPkgs = HostSystem.NUM_CPU_PKGS + hardware.numCpuThreads = HostSystem.NUM_CPU_THREADS + hardware.vendor = "Intel" + hardware.cpuModel = "Intel(R) Xeon(R)" + hardware.uuid = "host-uuid" + hardware.memorySize = units.Gi + return hardware + + @staticmethod + def create_summary(connected, maintenance_mode): + summary = DataObject() + summary.hardware = HostSystem.create_summary_hardware() + summary.runtime = HostSystem.create_summary_runtime(connected, + maintenance_mode) + quickstats = DataObject() + quickstats.overallMemoryUsage = 500 + summary.quickStats = quickstats + + product = DataObject() + product.name = "VMware ESXi" + # Avoid deprecation warning + product.version = constants.NEXT_MIN_VC_VERSION + config = DataObject() + config.product = product + summary.config = config + + return summary + + @staticmethod + def create_summary_runtime(connected, maintenance_mode): + runtime = DataObject() + if connected: + runtime.connectionState = "connected" + else: + runtime.connectionState = "disconnected" + + runtime.inMaintenanceMode = maintenance_mode + return runtime + + @staticmethod + def create_cpu_pkgs(): + cpuPkg = DataObject('HostCpuPackage') + cpuPkg.description = 'Intel(R) Xeon(R) CPU E5-4650 v4 @ 2.20GHz' + cpuPkg.vendor = 'intel' + cpu_pkgs = _create_array_of_type('HostCpuPackage') + cpu_pkgs.HostCpuPackage = HostSystem.NUM_CPU_PKGS * [cpuPkg] + return cpu_pkgs + + @staticmethod + def create_hardware_cpu_info(): + cpuInfo = DataObject('CpuInfo') + cpuInfo.numCpuCores = HostSystem.NUM_CPU_CORES + cpuInfo.numCpuPackages = HostSystem.NUM_CPU_PKGS + cpuInfo.numCpuThreads = HostSystem.NUM_CPU_THREADS + return cpuInfo + + @staticmethod + def create_config_feature_capability(): + aes = DataObject('HostFeatureCapability') + aes.featureName = 'cpuid.AES' + aes.value = '1' + avx = DataObject('HostFeatureCapability') + avx.featureName = 'cpuid.AVX' + avx.value = '1' + amd = DataObject('HostFeatureCapability') + amd.featureName = 'cpuid.AMD' + amd.value = '0' + caps = _create_array_of_type('HostFeatureCapability') + caps.HostFeatureCapability = [aes, avx, amd] + return caps + + def __init__(self, name=None, connected=True, ds_ref=None, maintenance_mode=False): super(HostSystem, self).__init__("host") + if not name: + name = "ha-{}".format(self.mo_id) self.set("name", name) if _no_objects_of_type("HostNetworkSystem"): create_host_network_system() @@ -766,50 +846,31 @@ def __init__(self, name="ha-host", connected=True, ds_ref=None, datastores.ManagedObjectReference = [ds_ref] self.set("datastore", datastores) - summary = DataObject() - hardware = DataObject() - hardware.numCpuCores = 8 - hardware.numCpuPkgs = 2 - hardware.numCpuThreads = 16 - hardware.vendor = "Intel" - hardware.cpuModel = "Intel(R) Xeon(R)" - hardware.uuid = "host-uuid" - hardware.memorySize = units.Gi - summary.hardware = hardware - - runtime = DataObject() - if connected: - runtime.connectionState = "connected" - else: - runtime.connectionState = "disconnected" - - runtime.inMaintenanceMode = maintenance_mode - - summary.runtime = runtime - - quickstats = DataObject() - quickstats.overallMemoryUsage = 500 - summary.quickStats = quickstats - - product = DataObject() - product.name = "VMware ESXi" - # Avoid deprecation warning - product.version = constants.NEXT_MIN_VC_VERSION - config = DataObject() - config.product = product - summary.config = config + caps = self.create_config_feature_capability() + self.set('config.featureCapability', caps) pnic_do = DataObject() pnic_do.device = "vmnic0" net_info_pnic = DataObject() net_info_pnic.PhysicalNic = [pnic_do] + self.set("config.network.pnic", net_info_pnic) + + cpu_pkgs = self.create_cpu_pkgs() + self.set('hardware.cpuPkg', cpu_pkgs) + + cpu_info = self.create_hardware_cpu_info() + self.set('hardware.cpuInfo', cpu_info) + + summary = self.create_summary(connected, maintenance_mode) self.set("summary", summary) + self.set("summary.config", summary.config) + self.set("summary.config.product", summary.config.product) + self.set("summary.hardware", summary.hardware) + self.set("summary.runtime", summary.runtime) + self.set("summary.quickStats", summary.quickStats) + self.set("capability.maxHostSupportedVcpus", 600) - self.set("summary.hardware", hardware) - self.set("summary.runtime", runtime) - self.set("summary.quickStats", quickstats) - self.set("config.network.pnic", net_info_pnic) self.set("connected", connected) if _no_objects_of_type("Network"): diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index b5d61cdf8e6..7d76e5c2981 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -214,7 +214,6 @@ def setUp(self, mock_svc, mock_register): 'owner': '' }) self.fake_image_uuid = self.image.id - self.vnc_host = 'ha-host' # create compute node resource provider self.cn_rp = dict( @@ -1744,7 +1743,8 @@ def _test_get_vnc_console(self): opt_val = OptionValue(key='', value=5906) fake_vm.set(vm_util.VNC_CONFIG_KEY, opt_val) vnc_console = self.conn.get_vnc_console(self.context, self.instance) - self.assertEqual(self.vnc_host, vnc_console.host) + host = vmwareapi_fake.get_first_object('HostSystem') + self.assertEqual(host.name, vnc_console.host) self.assertEqual(5906, vnc_console.port) def test_get_vnc_console(self): @@ -2169,6 +2169,7 @@ def test_datastore_regex_configured(self): @mock.patch('nova.virt.vmwareapi.ds_util.get_available_datastores') def test_datastore_regex_configured_vcstate(self, mock_get_ds_ref): vcstate = self.conn._vc_state + vcstate._stats = {} # Clear cache to force new collection self.conn.get_available_resource(self.node_name) mock_get_ds_ref.assert_called_with( vcstate._session, vcstate._cluster, vcstate._datastore_regex) @@ -2198,6 +2199,7 @@ def test_get_available_resource(self): @mock.patch('nova.virt.vmwareapi.ds_util.get_available_datastores') def test_update_provider_tree(self, mock_get_avail_ds): + self.assertEqual(CONF.reserved_host_memory_mb, 512) ds1 = ds_obj.Datastore(ref='fake-ref', name='datastore1', capacity=10 * units.Gi, freespace=3 * units.Gi) ds2 = ds_obj.Datastore(ref='fake-ref', name='datastore2', @@ -2209,6 +2211,8 @@ def test_update_provider_tree(self, mock_get_avail_ds): self.assertFalse(self.pt.has_inventory(self.node_name)) # now update it self.conn.update_provider_tree(self.pt, self.node_name) + vcstate = self.conn._vc_state + vcstate._stats = {} # Clear cache to force new collection expected = { orc.VCPU: { 'total': 32, @@ -2216,7 +2220,7 @@ def test_update_provider_tree(self, mock_get_avail_ds): 'min_unit': 1, 'max_unit': 16, 'step_size': 1, - 'allocation_ratio': 4.0, + 'allocation_ratio': CONF.initial_cpu_allocation_ratio, }, orc.MEMORY_MB: { 'total': 2048, @@ -2224,15 +2228,15 @@ def test_update_provider_tree(self, mock_get_avail_ds): 'min_unit': 1, 'max_unit': 1024, 'step_size': 1, - 'allocation_ratio': 1.0, + 'allocation_ratio': CONF.initial_ram_allocation_ratio, }, - orc.DISK_GB: { - 'total': 95, + orc.DISK_GB: { # Fix this + 'total': 1024.0, 'reserved': 0, 'min_unit': 1, - 'max_unit': 25, + 'max_unit': 500.0, 'step_size': 1, - 'allocation_ratio': 1.0, + 'allocation_ratio': CONF.initial_disk_allocation_ratio, }, nova.utils.MEMORY_RESERVABLE_MB_RESOURCE: { 'total': 2048 - 512, # 512 CONF.reserved_host_memory_mb @@ -2644,16 +2648,39 @@ def test_warning_deprecated_version(self, mock_version, mock_warning): break self.assertTrue(version_arg_found) + def _mock_get_stats_from_cluster_per_host(self, + memory_mb_reserved=0): + return { + 'host1': { + 'available': True, + 'vcpus': 16, + 'vcpus_used': 0, + 'vcpus_reserved': 2, + 'memory_mb': 1024, + 'memory_mb_used': 0, + 'memory_mb_reserved': memory_mb_reserved, + 'cpu_info': {}, + }, + 'host2': { + 'available': True, + 'vcpus': 16, + 'vcpus_used': 0, + 'vcpus_reserved': 0, + 'memory_mb': 1024, + 'memory_mb_used': 0, + 'memory_mb_reserved': 0, + 'cpu_info': {}, + }, + } + @mock.patch.object(objects.Service, 'get_by_compute_host') def test_host_state_service_disabled(self, mock_service): service = self._create_service(disabled=False, host='fake-mini') mock_service.return_value = service - fake_stats = {'cpu': {'vcpus': 4, 'reserved_vcpus': 0}, - 'mem': {'total': '8194', 'free': '2048', - 'reserved_memory_mb': 0}} + fake_stats = self._mock_get_stats_from_cluster_per_host() with test.nested( - mock.patch.object(vm_util, 'get_stats_from_cluster', + mock.patch.object(vm_util, 'get_stats_from_cluster_per_host', side_effect=[vexc.VimConnectionException('fake'), fake_stats, fake_stats]), mock.patch.object(service, 'save')) as (mock_stats, diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 644874af4ed..8ced16a0857 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -59,6 +59,15 @@ def setUp(self): 'uuid': uuidutils.generate_uuid(), 'vcpus': 2, 'memory_mb': 2048}) + @staticmethod + def _expected_cpu_info(): + return { + 'features': ['aes', 'avx'], + 'model': 'Intel(R) Xeon(R) CPU E5-4650 v4 @ 2.20GHz', + 'topology': {'cores': 8, 'sockets': 2, 'threads': 16}, + 'vendor': 'Intel', + } + def _test_get_stats_from_cluster(self, connection_state="connected", maintenance_mode=False, failover_policy=None, @@ -76,13 +85,13 @@ def _test_get_stats_from_cluster(self, connection_state="connected", k = 'configuration.dasConfig.admissionControlPolicy' prop_dict[k] = failover_policy - hardware = fake.DataObject() - hardware.numCpuCores = 8 - hardware.numCpuThreads = 16 - hardware.vendor = "Intel" - hardware.cpuModel = "Intel(R) Xeon(R)" + hardware = fake.HostSystem.create_summary_hardware() hardware.memorySize = 4 * units.Gi + cpu_pkgs = fake.HostSystem.create_cpu_pkgs() + cpu_info = fake.HostSystem.create_hardware_cpu_info() + cap = fake.HostSystem.create_config_feature_capability() + runtime_host_1 = fake.DataObject() runtime_host_1.connectionState = "connected" runtime_host_1.inMaintenanceMode = False @@ -97,12 +106,22 @@ def _test_get_stats_from_cluster(self, connection_state="connected", runtime_host_2.connectionState = connection_state runtime_host_2.inMaintenanceMode = maintenance_mode - prop_list_host_1 = [fake.Prop(name="summary.hardware", val=hardware), + prop_list_host_1 = [fake.Prop(name="name", val="host-1.test"), + fake.Prop(name="summary.hardware", val=hardware), + fake.Prop(name="hardware.cpuPkg", val=cpu_pkgs), + fake.Prop(name="hardware.cpuInfo", val=cpu_info), + fake.Prop(name="config.featureCapability", + val=cap), fake.Prop(name="summary.runtime", val=runtime_host_1), fake.Prop(name="summary.quickStats", val=quickstats_1)] - prop_list_host_2 = [fake.Prop(name="summary.hardware", val=hardware), + prop_list_host_2 = [fake.Prop(name="name", val="host-2.test"), + fake.Prop(name="summary.hardware", val=hardware), + fake.Prop(name="hardware.cpuPkg", val=cpu_pkgs), + fake.Prop(name="hardware.cpuInfo", val=cpu_info), + fake.Prop(name="config.featureCapability", + val=cap), fake.Prop(name="summary.runtime", val=runtime_host_2), fake.Prop(name="summary.quickStats", @@ -118,13 +137,12 @@ def _test_get_stats_from_cluster(self, connection_state="connected", def fake_call_method(*args): if "get_object_properties_dict" in args: return prop_dict - elif "get_properties_for_a_collection_of_objects" in args: + if "get_properties_for_a_collection_of_objects" in args: objects = fake.FakeRetrieveResult() for moref in args[3]: objects.add_object(fake_objects[moref.value]) return objects - else: - raise Exception('unexpected method call') + raise Exception('unexpected method call') session = fake.FakeSession() with mock.patch.object(session, '_call_method', fake_call_method): @@ -135,15 +153,18 @@ def fake_call_method(*args): num_hosts = 1 num_hosts -= failover_hosts_filtered if expected_stats is None: - expected_stats = {'cpu': {'vcpus': num_hosts * 16, - 'max_vcpus_per_host': 16, - 'reserved_vcpus': 0}, - 'mem': {'total': num_hosts * 4096, - 'free': num_hosts * 4096 - - num_hosts * 512, - 'max_mem_mb_per_host': 4096, - 'reserved_memory_mb': 0, - 'vm_reservable_memory_ratio': 1.0}} + expected_stats = { + 'cpu_info': self._expected_cpu_info(), + 'vcpus': num_hosts * 16, + 'vcpus_used': 0, + 'vcpus_reserved': 0, + 'max_vcpus_per_host': 16, + 'memory_mb': num_hosts * 4096, + 'memory_mb_used': num_hosts * 512, + 'memory_mb_reserved': 0, + 'max_mem_mb_per_host': 4096, + 'vm_reservable_memory_ratio': 1.0 + } self.assertEqual(expected_stats, result) def test_get_stats_from_cluster_hosts_connected_and_active(self): @@ -197,75 +218,104 @@ def test_get_stats_from_cluster_reservations(self, mock_map): 'memory_mb': 512, 'vcpus': 2}} - expected = {'cpu': {'vcpus': 2 * 16, - 'max_vcpus_per_host': 14, # by host2 counts - 'reserved_vcpus': 4 + 2}, # both hosts - 'mem': {'total': 2 * 4096, - 'free': 2 * 4096 - - 2 * 512, - 'max_mem_mb_per_host': 4096 - 256, # host1 - 'reserved_memory_mb': 512 + 256, # both - 'vm_reservable_memory_ratio': 1.0}} + expected = { + 'cpu_info': self._expected_cpu_info(), + 'vcpus': 2 * 16, + 'vcpus_used': 0, + 'vcpus_reserved': 4 + 2, # both hosts + 'max_vcpus_per_host': 16 - 2, # by host2 counts + 'memory_mb': 2 * 4096, + 'memory_mb_used': 2 * 512, + 'memory_mb_reserved': 512 + 256, # both + 'max_mem_mb_per_host': 4096 - 256, # host1 + 'vm_reservable_memory_ratio': 1.0 + } self._test_get_stats_from_cluster(expected_stats=expected) - def test_get_host_reservations_empty(self): + def assertSubset(self, expected, full): + self.assertDictEqual(expected, { + key: full[key] for key in expected + }) + + def _create_stats(self, vcpus=32, memory_mb=4096): + return { + "vcpus": vcpus, + "memory_mb": memory_mb, + } + + def _create_reserved(self, vcpus_reserved=0, memory_mb_reserved=0): + return { + "vcpus_reserved": vcpus_reserved, + "memory_mb_reserved": memory_mb_reserved, + } + + def test_set_host_reservations_empty(self): host = fake.ManagedObjectReference('HostSystem', 'host1') mapping = {} - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 0, 'memory_mb': 0} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(0, 0) + self.assertSubset(expected, result) mapping = {'host2': {'vcpus': 5}} - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 0, 'memory_mb': 0} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + self.assertSubset(expected, result) - def test_get_host_reservations_with_default(self): + def test_set_host_reservations_with_default(self): host = fake.ManagedObjectReference('HostSystem', 'host1') mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}} - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 2, 'memory_mb': 96} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) - def test_get_host_reservations_with_some_default(self): + expected = self._create_reserved(2, 96) + self.assertSubset(expected, result) + + def test_set_host_reservations_with_some_default(self): host = fake.ManagedObjectReference('HostSystem', 'host1') mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}, 'host1': {'vcpus': 1}} - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 1, 'memory_mb': 96} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(1, 96) + self.assertSubset(expected, result) mapping = {'__default__': {'vcpus': 2, 'memory_mb': 96}, 'host1': {'memory_mb': 2048}} - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 2, 'memory_mb': 2048} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(2, 2048) + self.assertSubset(expected, result) - def test_get_host_reservations_priority(self): + def test_set_host_reservations_priority(self): host = fake.ManagedObjectReference('HostSystem', 'host1') mapping = {'__default__': {'vcpus_percent': 10, 'memory_percent': 50}, 'host1': {'vcpus': 1}} - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 1, 'memory_mb': 2048} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(1, 2048) + self.assertSubset(expected, result) mapping['host1']['memory_mb'] = 96 - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 1, 'memory_mb': 96} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(1, 96) + self.assertSubset(expected, result) - def test_get_host_reservations_override(self): + def test_set_host_reservations_override(self): host = fake.ManagedObjectReference('HostSystem', 'host1') mapping = {'__default__': {'vcpus': 5}, 'host1': {'vcpus_percent': 10}} - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 5, 'memory_mb': 0} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(5, 0) + self.assertSubset(expected, result) mapping['host1']['vcpus'] = None - result = vm_util._get_host_reservations(mapping, host, 32, 4096) - expected = {'vcpus': 3, 'memory_mb': 0} - self.assertEqual(expected, result) + result = self._create_stats() + vm_util._set_host_reservations(result, mapping, host) + expected = self._create_reserved(3, 0) + self.assertSubset(expected, result) def test_get_host_ref_no_hosts_in_cluster(self): self.assertRaises(exception.NoValidHost, diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 25b95acd680..e8394d5db5a 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -52,7 +52,6 @@ from nova.virt import driver from nova.virt.vmwareapi import cluster_util from nova.virt.vmwareapi import constants -from nova.virt.vmwareapi import ds_util from nova.virt.vmwareapi import error_util from nova.virt.vmwareapi import host from nova.virt.vmwareapi.rpc import VmwareRpcService @@ -372,29 +371,17 @@ def _create_nodename(self, mo_id): return '%s.%s' % (mo_id, self._vcenter_uuid) - def _get_available_resources(self, host_stats): - return {'vcpus': host_stats['vcpus'], - 'memory_mb': host_stats['host_memory_total'], - 'local_gb': host_stats['disk_total'], - 'vcpus_used': 0, - 'memory_mb_used': host_stats['host_memory_total'] - - host_stats['host_memory_free'], - 'local_gb_used': host_stats['disk_used'], - 'vcpus_reserved': CONF.reserved_host_cpus + - host_stats['vcpus_reserved'], - 'memory_mb_reserved': CONF.reserved_host_memory_mb + - host_stats['host_memory_reserved'], - 'hypervisor_type': host_stats['hypervisor_type'], - 'hypervisor_version': host_stats['hypervisor_version'], - 'hypervisor_hostname': host_stats['hypervisor_hostname'], - # The VMWare driver manages multiple hosts, so there are - # likely many different CPU models in use. As such it is - # impossible to provide any meaningful info on the CPU - # model of the "host" - 'cpu_info': None, - 'supported_instances': host_stats['supported_instances'], - 'numa_topology': None, - } + def _get_available_resources(self, host_stats, nodename): + stats = host_stats[nodename].copy() + stats["memory_mb_reserved"] = min(stats["memory_mb"], + stats["memory_mb_reserved"] + + CONF.reserved_host_memory_mb) + stats["vcpus_reserved"] = min(stats["vcpus"], + stats["vcpus_reserved"] + + CONF.reserved_host_cpus) + stats["cpu_info"] = jsonutils.dumps(stats['cpu_info'], sort_keys=True) + + return stats def get_available_resource(self, nodename): """Retrieve resource info. @@ -405,8 +392,8 @@ def get_available_resource(self, nodename): :returns: dictionary describing resources """ - host_stats = self._vc_state.get_host_stats(refresh=True) - stats_dict = self._get_available_resources(host_stats) + host_stats = self._vc_state.get_host_stats() + stats_dict = self._get_available_resources(host_stats, nodename) return stats_dict def get_available_nodes(self, refresh=False): @@ -414,6 +401,15 @@ def get_available_nodes(self, refresh=False): This driver supports only one compute node. """ + # get_available_nodes is called at the beginning of polling + # the resources of all the nodes via + # get_inventory & get_available_resource + # We follow here the same pattern as in the ironic driver and use this + # function call as an indicator of a new polling cycle and refresh + # the host stats cached in _vc_state by calling... + self._vc_state.get_host_stats(refresh=True) + # In the following calls to get_inventory and get_available_resource + # for each node, we then return the cached data return [self._nodename] def update_provider_tree(self, provider_tree, nodename, allocations=None): @@ -463,78 +459,67 @@ def update_provider_tree(self, provider_tree, nodename, allocations=None): :raises: ReshapeFailed if the requested tree reshape fails for whatever reason. """ - # NOTE(cdent): This is a side-effecty method, we are changing the - # the provider tree in place (on purpose). + stats = self.get_available_resource(nodename) + result = {} + + # NOTE(yikun): If the inv record does not exists, the allocation_ratio + # will use the CONF.xxx_allocation_ratio value if xxx_allocation_ratio + # is set, and fallback to use the initial_xxx_allocation_ratio + # otherwise. inv = provider_tree.data(nodename).inventory ratios = self._get_allocation_ratios(inv) - stats = vm_util.get_stats_from_cluster(self._session, - self._cluster_ref) - datastores = ds_util.get_available_datastores(self._session, - self._cluster_ref, - self._datastore_regex) - total_disk_capacity = sum([ds.capacity for ds in datastores]) - max_free_space = max([ds.freespace for ds in datastores]) - reserved_disk_gb = compute_utils.convert_mb_to_ceil_gb( - CONF.reserved_host_disk_mb) - result = { - orc.DISK_GB: { - 'total': total_disk_capacity // units.Gi, + + local_gb = stats["local_gb"] + local_gb_max_free = stats.get("local_gb_max_free", "local_gb") + if local_gb > 0 and local_gb_max_free > 0: + reserved_disk_gb = compute_utils.convert_mb_to_ceil_gb( + CONF.reserved_host_disk_mb) + result[orc.DISK_GB] = { + 'total': local_gb, 'reserved': reserved_disk_gb, 'min_unit': 1, - 'max_unit': max_free_space // units.Gi, + 'max_unit': local_gb_max_free, 'step_size': 1, + 'allocation_ratio': ratios[orc.DISK_GB], } - } - if stats['cpu']['max_vcpus_per_host'] > 0: - reserved_vcpus = stats['cpu'].get('reserved_vcpus', 0) - reserved_vcpus += CONF.reserved_host_cpus - result.update({orc.VCPU: { - 'total': stats['cpu']['vcpus'], + + vcpus = stats["vcpus"] + max_vcpus = stats.get("max_vcpus_per_host", vcpus) + if vcpus > 0 and max_vcpus > 0: + reserved_vcpus = stats["vcpus_reserved"] + result[orc.VCPU] = { + 'total': vcpus, 'reserved': reserved_vcpus, 'min_unit': 1, - 'max_unit': stats['cpu']['max_vcpus_per_host'], + 'max_unit': max_vcpus, 'step_size': 1, 'allocation_ratio': ratios[orc.VCPU], - }}) - if stats['mem']['max_mem_mb_per_host'] > 0: - reserved_memory_mb = stats['mem'].get('reserved_memory_mb', 0) - reserved_memory_mb += CONF.reserved_host_memory_mb - result.update({orc.MEMORY_MB: { - 'total': stats['mem']['total'], + } + + memory_mb = stats["memory_mb"] + reserved_memory_mb = stats["memory_mb_reserved"] + max_memory_mb = stats.get("max_mem_mb_per_host", memory_mb) + if memory_mb > 0 and max_memory_mb > 0: + result[orc.MEMORY_MB] = { + 'total': memory_mb, 'reserved': reserved_memory_mb, 'min_unit': 1, - 'max_unit': stats['mem']['max_mem_mb_per_host'], + 'max_unit': max_memory_mb, 'step_size': 1, 'allocation_ratio': ratios[orc.MEMORY_MB], - }}) - available_memory_mb = stats['mem']['total'] - reserved_memory_mb - result.update({ - utils.MEMORY_RESERVABLE_MB_RESOURCE: { - 'total': available_memory_mb, - 'reserved': int(available_memory_mb * - (1 - stats['mem']['vm_reservable_memory_ratio'])), - 'min_unit': 1, - 'max_unit': stats['mem']['max_mem_mb_per_host'], - 'step_size': 1, - }}) - - # If a sharing DISK_GB provider exists in the provider tree, then our - # storage is shared, and we should not report the DISK_GB inventory in - # the compute node provider. - # TODO(cdent): We don't do this yet, in part because of the issues - # in bug #1784020, but also because we can represent all datastores - # as shared providers and should do once update_provider_tree is - # working well. - if provider_tree.has_sharing_provider(orc.DISK_GB): - LOG.debug('Ignoring sharing provider - see bug #1784020') - result[orc.DISK_GB] = { - 'total': total_disk_capacity // units.Gi, - 'reserved': reserved_disk_gb, - 'min_unit': 1, - 'max_unit': max_free_space // units.Gi, - 'step_size': 1, - 'allocation_ratio': ratios[orc.DISK_GB], - } + } + + available_memory_mb = memory_mb - reserved_memory_mb + reserved_reservable_memory = int(available_memory_mb * + (1 - stats.get("vm_reservable_memory_ratio", 0))) + if available_memory_mb > 0: + result[utils.MEMORY_RESERVABLE_MB_RESOURCE] = { + 'total': available_memory_mb, + 'reserved': reserved_reservable_memory, + 'min_unit': 1, + 'max_unit': max_memory_mb, + 'step_size': 1, + } provider_tree.update_inventory(nodename, result) diff --git a/nova/virt/vmwareapi/host.py b/nova/virt/vmwareapi/host.py index bbc88700c33..fb3fb65a73c 100644 --- a/nova/virt/vmwareapi/host.py +++ b/nova/virt/vmwareapi/host.py @@ -16,6 +16,7 @@ """ Management class for host-related functions (start, reboot, etc). """ +from copy import deepcopy from oslo_log import log as logging from oslo_utils import units @@ -41,24 +42,26 @@ def _get_ds_capacity_and_freespace(session, cluster=None, datastore_regex=None): capacity = 0 freespace = 0 + max_freespace = 0 try: for ds in ds_util.get_available_datastores(session, cluster, datastore_regex): capacity += ds.capacity freespace += ds.freespace + max_freespace = max(max_freespace, ds.freespace) except exception.DatastoreNotFound: pass - return capacity, freespace + return capacity, freespace, max_freespace class VCState(object): """Manages information about the vCenter cluster""" - def __init__(self, session, host_name, cluster, datastore_regex): + def __init__(self, session, cluster_node_name, cluster, datastore_regex): super(VCState, self).__init__() self._session = session - self._host_name = host_name + self._cluster_node_name = cluster_node_name self._cluster = cluster self._datastore_regex = datastore_regex self._stats = {} @@ -88,44 +91,58 @@ def update_status(self): """Update the current state of the cluster.""" data = {} try: - capacity, freespace = _get_ds_capacity_and_freespace(self._session, - self._cluster, self._datastore_regex) + capacity, free, max_free = _get_ds_capacity_and_freespace( + self._session, self._cluster, self._datastore_regex) # Get cpu, memory stats from the cluster - stats = vm_util.get_stats_from_cluster(self._session, - self._cluster) + per_host_stats = vm_util.get_stats_from_cluster_per_host( + self._session, self._cluster) except (vexc.VimConnectionException, vexc.VimAttributeException) as ex: # VimAttributeException is thrown when vpxd service is down LOG.warning("Failed to connect with %(node)s. " "Error: %(error)s", - {'node': self._host_name, 'error': ex}) + {'node': self._cluster_node_name, 'error': ex}) self._set_host_enabled(False) return data - data["vcpus"] = stats['cpu']['vcpus'] - data["vcpus_reserved"] = stats['cpu']['reserved_vcpus'] - data["disk_total"] = capacity // units.Gi - data["disk_available"] = freespace // units.Gi - data["disk_used"] = data["disk_total"] - data["disk_available"] - data["host_memory_total"] = stats['mem']['total'] - data["host_memory_free"] = stats['mem']['free'] - data['host_memory_reserved'] = stats['mem']['reserved_memory_mb'] - data["hypervisor_type"] = self._hypervisor_type - data["hypervisor_version"] = self._hypervisor_version - data["hypervisor_hostname"] = self._host_name - data["supported_instances"] = [ - (obj_fields.Architecture.I686, - obj_fields.HVType.VMWARE, - obj_fields.VMMode.HVM), - (obj_fields.Architecture.X86_64, - obj_fields.HVType.VMWARE, - obj_fields.VMMode.HVM)] + local_gb = capacity / units.Gi + local_gb_used = (capacity - free) / units.Gi + local_gb_max_free = max_free / units.Gi + + defaults = { + "local_gb": local_gb, + "local_gb_used": local_gb_used, + "local_gb_max_free": local_gb_max_free, + "supported_instances": [ + (obj_fields.Architecture.I686, + obj_fields.HVType.VMWARE, + obj_fields.VMMode.HVM), + (obj_fields.Architecture.X86_64, + obj_fields.HVType.VMWARE, + obj_fields.VMMode.HVM)], + "numa_topology": None, + } + + for host, stats in per_host_stats.items(): + data[host] = self._merge_stats(host, stats, defaults) + + cluster_stats = vm_util.aggregate_stats_from_cluster(per_host_stats) + cluster_stats["hypervisor_type"] = self._hypervisor_type + cluster_stats["hypervisor_version"] = self._hypervisor_version + data[self._cluster_node_name] = self._merge_stats( + self._cluster_node_name, cluster_stats, defaults) self._stats = data if self._auto_service_disabled: self._set_host_enabled(True) return data + def _merge_stats(self, host, stats, defaults): + result = deepcopy(defaults) + result["hypervisor_hostname"] = host + result.update(stats) + return result + def _set_host_enabled(self, enabled): """Sets the compute host's ability to accept new instances.""" ctx = context.get_admin_context() diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 8c64b3d09e8..60679df24eb 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -31,6 +31,7 @@ from oslo_service import loopingcall from oslo_utils import excutils from oslo_utils import units +from oslo_utils.versionutils import convert_version_to_int from oslo_vmware import exceptions as vexc from oslo_vmware import pbm from oslo_vmware import vim_util as vutil @@ -1338,18 +1339,17 @@ def get_vm_state(session, instance): return constants.POWER_STATES[vm_state] -def _get_host_reservations(host_reservations_map, host_moref, host_vcpus, - host_memory_mb): +def _set_host_reservations(stats, host_reservations_map, host_moref): """Compute the number of vcpus and memory in MB reserved for the host from the configured reservations or the default. For every host, both vcpus and memory can be given as static number or in percent, with static number taking priority. """ - reservations = { - 'vcpus': 0, - 'memory_mb': 0 - } + host_vcpus = stats["vcpus"] + host_memory_mb = stats["memory_mb"] + stats["vcpus_reserved"] = 0 + stats["memory_mb_reserved"] = 0 default_key = _HOST_RESERVATIONS_DEFAULT_KEY host_reservations = host_reservations_map.get(default_key, {}) @@ -1357,28 +1357,27 @@ def _get_host_reservations(host_reservations_map, host_moref, host_vcpus, for key in ['vcpus', 'vcpus_percent', 'memory_mb', 'memory_percent']: if key in group_reservations: host_reservations[key] = group_reservations[key] + if not host_reservations: - return reservations + return # compute the number of vcpus if host_reservations.get('vcpus') is not None: vcpus = max(0, host_reservations['vcpus']) - reservations['vcpus'] = min(host_vcpus, vcpus) + stats['vcpus_reserved'] = min(host_vcpus, vcpus) elif host_reservations.get('vcpus_percent') is not None: percent = max(0, min(100, host_reservations['vcpus_percent'])) # This will round down. - reservations['vcpus'] = host_vcpus * percent // 100 + stats['vcpus_reserved'] = host_vcpus * percent // 100 # compute the memory in MB if host_reservations.get('memory_mb') is not None: memory_mb = max(0, host_reservations['memory_mb']) - reservations['memory_mb'] = min(host_memory_mb, memory_mb) + stats['memory_mb_reserved'] = min(host_memory_mb, memory_mb) elif host_reservations.get('memory_percent') is not None: percent = max(0, min(100, host_reservations['memory_percent'])) # This will round down. - reservations['memory_mb'] = host_memory_mb * percent // 100 - - return reservations + stats['memory_mb_reserved'] = host_memory_mb * percent // 100 def _get_host_reservations_map(groups=None): @@ -1417,79 +1416,176 @@ def _get_host_reservations_map(groups=None): return hrm +# Only for satisfying the tests, and to ensure it is producing the same results def get_stats_from_cluster(session, cluster): + return aggregate_stats_from_cluster( + get_stats_from_cluster_per_host(session, cluster)) + + +def aggregate_stats_from_cluster(host_stats): """Get the aggregate resource stats of a cluster.""" - vcpus = 0 + total_vcpus = 0 + total_vcpus_used = 0 + total_vcpus_reserved = 0 max_vcpus_per_host = 0 - reserved_vcpus = 0 - used_mem_mb = 0 - total_mem_mb = 0 + + total_memory_mb = 0 + total_memory_mb_used = 0 + total_memory_mb_reserved = 0 max_mem_mb_per_host = 0 - reserved_memory_mb = 0 + # NOTE (jakobk): For the total amount of hosts it doesn't matter + # whether the host is in MM or unreachable, because the count is + # used to calculate safety margins for resource allocations, and MM + # or otherwise unreachable hosts is precisely what that is supposed + # to guard against. + total_hypervisor_count = len(host_stats) + previous_cpu_info = None + cpu_infos_equal = True + + for stats in host_stats.values(): + if not stats["available"]: + continue + # Total vcpus is the sum of all pCPUs of individual hosts + # The overcommitment ratio is factored in by the scheduler + vcpus = stats["vcpus"] + total_vcpus += vcpus + vcpus_used = stats["vcpus_used"] + total_vcpus_used += vcpus_used + vcpus_reserved = stats["vcpus_reserved"] + total_vcpus_reserved += vcpus_reserved + max_vcpus_per_host = max(max_vcpus_per_host, + vcpus - vcpus_reserved) + + memory_mb = stats["memory_mb"] + total_memory_mb += memory_mb + memory_mb_used = stats["memory_mb_used"] + total_memory_mb_used += memory_mb_used + memory_mb_reserved = stats["memory_mb_reserved"] + total_memory_mb_reserved += memory_mb_reserved + max_mem_mb_per_host = max(max_mem_mb_per_host, + memory_mb - memory_mb_reserved) + + if cpu_infos_equal and previous_cpu_info: + cpu_infos_equal = (previous_cpu_info == stats["cpu_info"]) + + previous_cpu_info = stats["cpu_info"] + + # Calculate VM-reservable memory as a ratio of total available + # memory, depending on either the configured tolerance for failed + # hypervisors or a single configurable ratio. + max_fail_hvs = \ + CONF.vmware.memory_reservation_cluster_hosts_max_fail + if max_fail_hvs and total_hypervisor_count: + vm_reservable_memory_ratio = \ + (1 - max_fail_hvs / total_hypervisor_count) + else: + vm_reservable_memory_ratio = \ + CONF.vmware.memory_reservation_max_ratio_fallback + + return { + "vcpus": total_vcpus, + "vcpus_used": total_vcpus_used, + "vcpus_reserved": total_vcpus_reserved, + "max_vcpus_per_host": max_vcpus_per_host, + "memory_mb": total_memory_mb, + "memory_mb_used": total_memory_mb_used, + "memory_mb_reserved": total_memory_mb_reserved, + "max_mem_mb_per_host": max_mem_mb_per_host, + "vm_reservable_memory_ratio": vm_reservable_memory_ratio, + "cpu_info": (previous_cpu_info if cpu_infos_equal + else "CPUs for this cluster have different values!") + } + + +def _host_props_to_cpu_info(host_props): + processor_type = None + cpu_vendor = None + hardware_cpu_pkg = host_props.get("hardware.cpuPkg") + if hardware_cpu_pkg and hardware_cpu_pkg.HostCpuPackage: + t = hardware_cpu_pkg.HostCpuPackage[0] + processor_type = t.description + cpu_vendor = t.vendor.title() + + features = [] + if "config.featureCapability" in host_props: + feature_capability = host_props["config.featureCapability"] + for feature in feature_capability.HostFeatureCapability: + if not feature.featureName.startswith("cpuid."): + continue + if feature.value != "1": + continue + + name = feature.featureName + features.append(name.split(".", 1)[1].lower()) + cpu_info = { + "model": processor_type, + "vendor": cpu_vendor, + "features": sorted(features) + } + hardware_cpu_info = host_props.get("hardware.cpuInfo") + if hardware_cpu_info: + cpu_info["topology"] = { + "cores": hardware_cpu_info.numCpuCores, + "sockets": hardware_cpu_info.numCpuPackages, + "threads": hardware_cpu_info.numCpuThreads + } + return cpu_info + + +def _set_hypervisor_type_and_version(stats, host_props): + product = host_props.get("summary.config.product") + if not product: + return + + stats["hypervisor_type"] = product.name + stats["hypervisor_version"] = convert_version_to_int(product.version) + + +def _process_host_stats(obj, host_reservations_map): + host_props = propset_dict(obj.propSet) + runtime_summary = host_props["summary.runtime"] + hardware_summary = host_props.get("summary.hardware") + stats_summary = host_props.get("summary.quickStats") + # Total vcpus is the sum of all pCPUs of individual hosts + # The overcommitment ratio is factored in by the scheduler + threads = getattr(hardware_summary, "numCpuThreads", 0) + mem_mb = getattr(hardware_summary, "memorySize", 0) // units.Mi + + stats = { + "available": (not runtime_summary.inMaintenanceMode and + runtime_summary.connectionState == "connected"), + "vcpus": threads, + "vcpus_used": 0, + "memory_mb": mem_mb, + "memory_mb_used": getattr(stats_summary, "overallMemoryUsage", 0), + "cpu_info": _host_props_to_cpu_info(host_props), + } + + _set_hypervisor_type_and_version(stats, host_props) + _set_host_reservations(stats, host_reservations_map, obj.obj) + return host_props["name"], stats + + +def get_stats_from_cluster_per_host(session, cluster): + """Get the resource stats per host of a cluster.""" host_mors, host_reservations_map = \ get_hosts_and_reservations_for_cluster(session, cluster) - if host_mors: - result = session._call_method(vim_util, - "get_properties_for_a_collection_of_objects", - "HostSystem", host_mors, - ["summary.hardware", "summary.runtime", - "summary.quickStats"]) - total_hypervisor_count = 0 - # NOTE (jakobk): For the total amount of hosts it doesn't matter - # whether the host is in MM or unreachable, because the count is - # used to calculate safety margins for resource allocations, and MM - # or otherwise unreachable hosts is precisely what that is supposed - # to guard against. - with vutil.WithRetrieval(session.vim, result) as objects: - for obj in objects: - total_hypervisor_count += 1 - host_props = propset_dict(obj.propSet) - runtime_summary = host_props['summary.runtime'] - if (runtime_summary.inMaintenanceMode or - runtime_summary.connectionState != "connected"): - continue - hardware_summary = host_props['summary.hardware'] - stats_summary = host_props['summary.quickStats'] - # Total vcpus is the sum of all pCPUs of individual hosts - # The overcommitment ratio is factored in by the scheduler - threads = hardware_summary.numCpuThreads - vcpus += threads - used_mem_mb += stats_summary.overallMemoryUsage - mem_mb = hardware_summary.memorySize // units.Mi - total_mem_mb += mem_mb - reserved = _get_host_reservations( - host_reservations_map, obj.obj, - threads, mem_mb) - reserved_vcpus += reserved['vcpus'] - reserved_memory_mb += reserved['memory_mb'] - max_vcpus_per_host = max(max_vcpus_per_host, - threads - reserved['vcpus']) - max_mem_mb_per_host = max(max_mem_mb_per_host, - mem_mb - reserved['memory_mb']) - - # Calculate VM-reservable memory as a ratio of total available - # memory, depending on either the configured tolerance for failed - # hypervisors or a single configurable ratio. - max_fail_hvs = \ - CONF.vmware.memory_reservation_cluster_hosts_max_fail - if max_fail_hvs and total_hypervisor_count: - vm_reservable_memory_ratio = \ - (1 - max_fail_hvs / total_hypervisor_count) - else: - vm_reservable_memory_ratio = \ - CONF.vmware.memory_reservation_max_ratio_fallback - - stats = {'cpu': {'vcpus': vcpus, - 'max_vcpus_per_host': max_vcpus_per_host, - 'reserved_vcpus': reserved_vcpus}, - 'mem': {'total': total_mem_mb, - 'free': total_mem_mb - used_mem_mb, - 'max_mem_mb_per_host': max_mem_mb_per_host, - 'reserved_memory_mb': reserved_memory_mb, - 'vm_reservable_memory_ratio': vm_reservable_memory_ratio}} - return stats + if not host_mors: + return {} + + result = session._call_method(vim_util, + "get_properties_for_a_collection_of_objects", + "HostSystem", host_mors, + ["name", "summary.hardware", "summary.runtime", + "summary.quickStats", "summary.config.product", + "hardware.cpuPkg", "hardware.cpuInfo", + "config.featureCapability", + ]) + with vutil.WithRetrieval(session.vim, result) as objects: + return dict(_process_host_stats(obj, host_reservations_map) + for obj in objects if hasattr(obj, "propSet")) def get_hosts_and_reservations_for_cluster(session, cluster): From 7ea149e95ba8be338c24aca76769ddf0fbb3d3cb Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 4 Oct 2021 10:55:11 +0200 Subject: [PATCH 91/92] ResourceTracker: Fetch Bdms up front The resource tracker iterates over all instances in _update_usage_from_instances, where it will the fetch for each instance the block-device-mappings in an rpc call. By fetching the block-device-mappings up-front, we get a single rpc/db call, instead of several small ones, where the latencies then add up If a block-device-mapping list cannot be retrieved for an instance, it will fall back to the old behaviour and fetch it individually. Change-Id: Ice9ab0a9c1b783c059687e1d992eea1f97cb3193 (cherry picked from commit 6caf72a16df07da55c5df3c892b5d7c0a0ae83da) (cherry picked from commit ae68565b8cdac03efd22dbba84b524db858ac311) --- nova/compute/resource_tracker.py | 21 ++++++--- .../unit/compute/test_resource_tracker.py | 45 ++++++++++++++----- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/nova/compute/resource_tracker.py b/nova/compute/resource_tracker.py index 698049fe389..53b7d760046 100644 --- a/nova/compute/resource_tracker.py +++ b/nova/compute/resource_tracker.py @@ -1621,7 +1621,7 @@ def _update_usage_from_migrations(self, context, migrations, nodename): continue def _update_usage_from_instance(self, context, instance, nodename, - is_removed=False): + is_removed=False, bdms=None): """Update usage for a single instance.""" uuid = instance['uuid'] @@ -1653,7 +1653,8 @@ def _update_usage_from_instance(self, context, instance, nodename, instance, sign=sign) # new instance, update compute node resource usage: - self._update_usage(self._get_usage_dict(instance, instance), + self._update_usage(self._get_usage_dict(instance, instance, + bdms=bdms), nodename, sign=sign) # Stop tracking removed instances in the is_bfv cache. This needs to @@ -1688,11 +1689,20 @@ def _update_usage_from_instances(self, context, instances, nodename): cn.running_vms = 0 instance_by_uuid = {} + for instance in instances: + instance_by_uuid[instance.uuid] = instance + + bdms_by_instance_uuid = ( + objects.BlockDeviceMappingList.bdms_by_instance_uuid( + context, instance_by_uuid.keys())) + for instance in instances: if not vm_states.allow_resource_removal( vm_state=instance['vm_state'], task_state=instance.task_state): - self._update_usage_from_instance(context, instance, nodename) + bdms = bdms_by_instance_uuid.get(instance.uuid) + self._update_usage_from_instance(context, instance, nodename, + bdms=bdms) instance_by_uuid[instance.uuid] = instance return instance_by_uuid @@ -1868,7 +1878,7 @@ def _get_flavor(self, instance, prefix, migration): # them. In that case - just get the instance flavor. return instance.flavor - def _get_usage_dict(self, object_or_dict, instance, **updates): + def _get_usage_dict(self, object_or_dict, instance, bdms=None, **updates): """Make a usage dict _update methods expect. Accepts a dict or an Instance or Flavor object, and a set of updates. @@ -1878,6 +1888,7 @@ def _get_usage_dict(self, object_or_dict, instance, **updates): :param instance: nova.objects.Instance for the related operation; this is needed to determine if the instance is volume-backed + :param bdms: Optional a BlockDeviceMappingList for the instance :param updates: key-value pairs to update the passed object. Currently only considers 'numa_topology', all other keys are ignored. @@ -1892,7 +1903,7 @@ def _is_bfv(): is_bfv = self.is_bfv[instance.uuid] else: is_bfv = compute_utils.is_volume_backed_instance( - instance._context, instance) + instance._context, instance, bdms=bdms) self.is_bfv[instance.uuid] = is_bfv return is_bfv diff --git a/nova/tests/unit/compute/test_resource_tracker.py b/nova/tests/unit/compute/test_resource_tracker.py index beefce9764f..d88b90dc835 100644 --- a/nova/tests/unit/compute/test_resource_tracker.py +++ b/nova/tests/unit/compute/test_resource_tracker.py @@ -829,6 +829,8 @@ def test_no_instances_no_migrations_reserved_disk_ram_and_cpu_driver( self.assertTrue(obj_base.obj_equal_prims(expected_resources, actual_resources)) + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', return_value=objects.InstancePCIRequests(requests=[])) @@ -839,7 +841,8 @@ def test_no_instances_no_migrations_reserved_disk_ram_and_cpu_driver( @mock.patch('nova.objects.InstanceList.get_by_host_and_node') def test_some_instances_no_migrations(self, get_mock, migr_mock, get_cn_mock, pci_mock, - instance_pci_mock, bfv_check_mock): + instance_pci_mock, bfv_check_mock, + bdms_by_uuid_mock): # Setup virt resources to match used resources to number # of defined instances on the hypervisor # Note that the usage numbers here correspond to only the first @@ -1068,6 +1071,8 @@ def test_no_instances_dest_evacuation(self, get_mock, get_inst_mock, actual_resources)) update_mock.assert_called_once() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', return_value=objects.InstancePCIRequests(requests=[])) @@ -1085,7 +1090,8 @@ def test_some_instances_source_and_dest_migration(self, get_mock, get_mig_ctxt_mock, pci_mock, instance_pci_mock, - bfv_check_mock): + bfv_check_mock, + bdm_by_instance_mock): # We test the behavior of update_available_resource() when # there is an active migration that involves this compute node # as the destination host AND the source host, and the resource @@ -1204,6 +1210,8 @@ def test_no_instances_err_migration( obj_base.obj_equal_prims(expected_resources, actual_resources)) update_mock.assert_called_once() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', new=mock.Mock(return_value=False)) @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', @@ -1217,7 +1225,8 @@ def test_no_instances_err_migration( def test_populate_assigned_resources(self, mock_get_instances, mock_get_instance, mock_get_migrations, - mock_get_cn): + mock_get_cn, + bdm_by_instance_mock): # when update_available_resources, rt.assigned_resources # will be populated, resources assigned to tracked migrations # and instances will be tracked in rt.assigned_resources. @@ -1257,6 +1266,8 @@ def test_populate_assigned_resources(self, mock_get_instances, self.assertEqual(expected_assigned_resources, self.rt.assigned_resources) + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', new=mock.Mock(return_value=False)) @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', @@ -1270,7 +1281,8 @@ def test_populate_assigned_resources(self, mock_get_instances, def test_check_resources_startup_success(self, mock_get_instances, mock_get_instance, mock_get_migrations, - mock_get_cn): + mock_get_cn, + bdms_by_instance_mock): # When update_available_resources is running on startup, # it will trigger this function to check if there are # assigned resources not in provider tree. If so, the reason @@ -1301,6 +1313,8 @@ def test_check_resources_startup_success(self, mock_get_instances, update_mock = self._update_available_resources(startup=True) update_mock.assert_called_once() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', new=mock.Mock(return_value=False)) @mock.patch('nova.objects.PciDeviceList.get_by_compute_node', @@ -1310,7 +1324,8 @@ def test_check_resources_startup_success(self, mock_get_instances, @mock.patch('nova.objects.InstanceList.get_by_host_and_node') def test_check_resources_startup_fail(self, mock_get_instances, mock_get_migrations, - mock_get_cn): + mock_get_cn, + bdms_by_uuid_mock): # Similar to testcase test_check_resources_startup_success, # and this one is for check_resources failed resource = objects.Resource(provider_uuid=self.compute.uuid, @@ -2774,6 +2789,8 @@ def test_claim_numa(self, save_mock, migr_mock, check_bfv_mock): class TestResize(BaseTestCase): @mock.patch('nova.compute.resource_tracker.ResourceTracker.' '_sync_compute_service_disabled_trait', new=mock.Mock()) + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance', return_value=False) @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', @@ -2786,7 +2803,7 @@ class TestResize(BaseTestCase): @mock.patch('nova.objects.ComputeNode.save') def test_resize_claim_same_host(self, save_mock, get_mock, migr_mock, get_cn_mock, pci_mock, instance_pci_mock, - is_bfv_mock): + is_bfv_mock, bdms_by_uuid_mock): # Resize an existing instance from its current flavor (instance type # 1) to a new flavor (instance type 2) and verify that the compute # node's resources are appropriately updated to account for the new @@ -3353,6 +3370,8 @@ def test_resize_claim_two_instances(self, save_mock, get_mock, migr_mock, class TestRebuild(BaseTestCase): @mock.patch('nova.compute.resource_tracker.ResourceTracker.' '_sync_compute_service_disabled_trait', new=mock.Mock()) + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') @mock.patch('nova.objects.InstancePCIRequests.get_by_instance', return_value=objects.InstancePCIRequests(requests=[])) @@ -3363,7 +3382,8 @@ class TestRebuild(BaseTestCase): @mock.patch('nova.objects.InstanceList.get_by_host_and_node') @mock.patch('nova.objects.ComputeNode.save') def test_rebuild_claim(self, save_mock, get_mock, migr_mock, get_cn_mock, - pci_mock, instance_pci_mock, bfv_check_mock): + pci_mock, instance_pci_mock, bfv_check_mock, + bdms_by_instance_mock): # Rebuild an instance, emulating an evacuate command issued against the # original instance. The rebuild operation uses the resource tracker's # _move_claim() method, but unlike with resize_claim(), rebuild_claim() @@ -3686,16 +3706,18 @@ def setUp(self): self.rt.compute_nodes[_NODENAME] = cn self.instance = _INSTANCE_FIXTURES[0].obj_clone() + @mock.patch('nova.objects.BlockDeviceMappingList.bdms_by_instance_uuid', + return_value={}) @mock.patch('nova.compute.utils.is_volume_backed_instance') def test_get_usage_dict_return_0_root_gb_for_bfv_instance( - self, mock_check_bfv): + self, mock_check_bfv, mock_batch_bdms): mock_check_bfv.return_value = True # Make sure the cache is empty. self.assertNotIn(self.instance.uuid, self.rt.is_bfv) result = self.rt._get_usage_dict(self.instance, self.instance) self.assertEqual(0, result['root_gb']) mock_check_bfv.assert_called_once_with( - self.instance._context, self.instance) + self.instance._context, self.instance, bdms=None) # Make sure we updated the cache. self.assertIn(self.instance.uuid, self.rt.is_bfv) self.assertTrue(self.rt.is_bfv[self.instance.uuid]) @@ -4088,10 +4110,13 @@ def test_update_usage_from_instances_goes_negative(self): self.rt._copy_resources(cn, resources) self.rt.compute_nodes['foo'] = cn + @mock.patch('nova.objects.BlockDeviceMappingList' + '.bdms_by_instance_uuid', + return_value={}) @mock.patch.object(self.rt, '_update_usage_from_instance') @mock.patch('nova.objects.Service.get_minimum_version', return_value=22) - def test(version_mock, uufi): + def test(version_mock, uufi, bdms): def _update_usage(*args, **kwargs): # simulate an instance with 512 MB memory and 1 GB disk using # resources From ed5da0fc726263c475b2be83b383a033d622bc87 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Wed, 12 Jan 2022 14:51:02 +0100 Subject: [PATCH 92/92] Vmware: Aggregate cpu_info on item level Clients such as the cli expect cpu_info to be a json dict, so returning a string is breaking them. By aggregating the individual attributes, we also work around having different cpu models with the same flags in a cluster not yielding any information. Now we can see that the model mismatch, but the cpu feature flags are matching. Change-Id: Id552121b642ec90f6e06be09825ab7339531f9d6 (cherry picked from commit a84a7853dc20ae922055b34210e234fbe4ef8454) (cherry picked from commit 273e2e7f47372b774f4237463e77f795b815ab9e) --- .../tests/unit/virt/vmwareapi/test_vm_util.py | 27 +++++++++++++++++++ nova/virt/vmwareapi/vm_util.py | 18 ++++++------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index 8ced16a0857..24caab66718 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -68,6 +68,33 @@ def _expected_cpu_info(): 'vendor': 'Intel', } + @staticmethod + def _host_stats(): + return { + 'available': True, + 'vcpus': 4, + 'vcpus_used': 2, + 'vcpus_reserved': 1, + 'memory_mb': 4, + 'memory_mb_used': 2, + 'memory_mb_reserved': 1, + 'cpu_info': VMwareVMUtilTestCase._expected_cpu_info() + } + + def test_aggregate_stats_from_cluster(self): + a = self._host_stats() + b = self._host_stats() + b["cpu_info"]["model"] = "Different" + host_stats = {'a': a, 'b': b} + aggregated = vm_util.aggregate_stats_from_cluster(host_stats) + for k in a: + if k in ('available', 'cpu_info'): + continue + self.assertEqual(aggregated[k], sum(item[k] + for item in host_stats.values() + )) + self.assertEqual(aggregated["cpu_info"]["model"], "Mismatching values") + def _test_get_stats_from_cluster(self, connection_state="connected", maintenance_mode=False, failover_policy=None, diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index 60679df24eb..e6dc9ac22a1 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -1440,8 +1440,7 @@ def aggregate_stats_from_cluster(host_stats): # or otherwise unreachable hosts is precisely what that is supposed # to guard against. total_hypervisor_count = len(host_stats) - previous_cpu_info = None - cpu_infos_equal = True + aggregated_cpu_info = {} for stats in host_stats.values(): if not stats["available"]: @@ -1465,11 +1464,13 @@ def aggregate_stats_from_cluster(host_stats): total_memory_mb_reserved += memory_mb_reserved max_mem_mb_per_host = max(max_mem_mb_per_host, memory_mb - memory_mb_reserved) - - if cpu_infos_equal and previous_cpu_info: - cpu_infos_equal = (previous_cpu_info == stats["cpu_info"]) - - previous_cpu_info = stats["cpu_info"] + if not aggregated_cpu_info: + aggregated_cpu_info = stats["cpu_info"].copy() + else: + cpu_info = stats["cpu_info"] + for key, value in cpu_info.items(): + if aggregated_cpu_info.get(key) != value: + aggregated_cpu_info[key] = "Mismatching values" # Calculate VM-reservable memory as a ratio of total available # memory, depending on either the configured tolerance for failed @@ -1493,8 +1494,7 @@ def aggregate_stats_from_cluster(host_stats): "memory_mb_reserved": total_memory_mb_reserved, "max_mem_mb_per_host": max_mem_mb_per_host, "vm_reservable_memory_ratio": vm_reservable_memory_ratio, - "cpu_info": (previous_cpu_info if cpu_infos_equal - else "CPUs for this cluster have different values!") + "cpu_info": aggregated_cpu_info }