diff --git a/.zuul.yaml b/.zuul.yaml index 846e3e23bac..ea488401cf2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -197,24 +197,11 @@ parent: devstack-tempest description: | Run tempest compute API tests using LVM image backend. This only runs - against nova/virt/libvirt/* changes. - # Copy irrelevant-files from nova-dsvm-multinode-base and then exclude - # anything that is not in nova/virt/libvirt/* or nova/privsep/*. - irrelevant-files: - - ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/privsep/).*$ - - ^api-.*$ - - ^(test-|)requirements.txt$ - - ^.*\.rst$ - - ^.git.*$ - - ^doc/.*$ - - ^nova/hacking/.*$ - - ^nova/locale/.*$ - - ^nova/tests/.*$ - - ^nova/test.py$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^tools/.*$ - - ^tox.ini$ + against nova/virt/libvirt/*, nova/privsep/* and .zuul.yaml changes. + files: + - ^nova/virt/libvirt/.*$ + - ^nova/privsep/.*$ + - .zuul.yaml vars: # We use the "all" environment for tempest_test_regex and # tempest_exclude_regex. @@ -253,22 +240,11 @@ # NOTE(chateaulav): due to constraints with no IDE support for aarch64, # tests have been limited to eliminate any items that are incompatible. # This is to be re-evaluated as greater support is added and defined. - irrelevant-files: - - ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/objects/)(?!nova/scheduler/).*$ - - ^api-.*$ - - ^(test-|)requirements.txt$ - - ^.*\.rst$ - - ^.git.*$ - - ^doc/.*$ - - ^nova/hacking/.*$ - - ^nova/locale/.*$ - - ^nova/policies/.*$ - - ^nova/tests/.*$ - - ^nova/test.py$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^tools/.*$ - - ^tox.ini$ + files: + - ^nova/virt/libvirt/.*$ + - ^nova/objects/.*$ + - ^nova/scheduler/.*$ + - .zuul.yaml vars: tox_envlist: all tempest_test_regex: ^tempest\.(api\.compute\.servers|scenario\.test_network_basic_ops) @@ -580,6 +556,16 @@ GLANCE_STANDALONE: True GLANCE_USE_IMPORT_WORKFLOW: True DEVSTACK_PARALLEL: True + MYSQL_REDUCE_MEMORY: True + # NOTE(danms): This job is pretty heavy as it is, so we disable some + # services that are not relevant to the nova-glance-ceph scenario + # that this job is intended to validate. + devstack_services: + c-bak: false + s-account: false + s-container: false + s-object: false + s-proxy: false devstack_local_conf: post-config: $NOVA_CONF: @@ -632,11 +618,12 @@ - nova-ceph-multistore: irrelevant-files: *nova-base-irrelevant-files - neutron-linuxbridge-tempest: - irrelevant-files: + files: # NOTE(mriedem): This job has its own irrelevant-files section # so that we only run it on changes to networking and libvirt/vif # code; we don't need to run this on all changes. - - ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$ + - ^nova/network/.*$ + - nova/virt/libvirt/vif.py - nova-live-migration - nova-live-migration-ceph - nova-lvm @@ -680,11 +667,6 @@ - barbican-tempest-plugin-simple-crypto: irrelevant-files: *nova-base-irrelevant-files voting: false - - tempest-integrated-compute-centos-8-stream: - irrelevant-files: *nova-base-irrelevant-files - - tempest-centos8-stream-fips: - irrelevant-files: *nova-base-irrelevant-files - voting: false gate: jobs: - nova-live-migration @@ -697,11 +679,12 @@ - nova-ceph-multistore: irrelevant-files: *nova-base-irrelevant-files - neutron-linuxbridge-tempest: - irrelevant-files: + files: # NOTE(mriedem): This job has its own irrelevant-files section # so that we only run it on changes to networking and libvirt/vif # code; we don't need to run this on all changes. - - ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$ + - ^nova/network/.*$ + - nova/virt/libvirt/vif.py - tempest-integrated-compute: irrelevant-files: *policies-irrelevant-files - nova-grenade-multinode: @@ -710,8 +693,6 @@ irrelevant-files: *nova-base-irrelevant-files - openstacksdk-functional-devstack: irrelevant-files: *nova-base-irrelevant-files - - tempest-integrated-compute-centos-8-stream: - irrelevant-files: *nova-base-irrelevant-files experimental: jobs: - ironic-tempest-bfv: diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 44185ef667a..427cb3aac35 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -6646,9 +6646,9 @@ def _shelve_offload_instance(self, context, instance, clean_shutdown, instance.power_state = current_power_state # NOTE(mriedem): The vm_state has to be set before updating the - # resource tracker, see vm_states.ALLOW_RESOURCE_REMOVAL. The host/node - # values cannot be nulled out until after updating the resource tracker - # though. + # resource tracker, see vm_states.allow_resource_removal(). The + # host/node values cannot be nulled out until after updating the + # resource tracker though. instance.vm_state = vm_states.SHELVED_OFFLOADED instance.task_state = None instance.save(expected_task_state=[task_states.SHELVING, diff --git a/nova/compute/resource_tracker.py b/nova/compute/resource_tracker.py index 058777d1ed0..d195044a57c 100644 --- a/nova/compute/resource_tracker.py +++ b/nova/compute/resource_tracker.py @@ -1495,7 +1495,8 @@ def _update_usage_from_instance(self, context, instance, nodename, # NOTE(sfinucan): Both brand new instances as well as instances that # are being unshelved will have is_new_instance == True is_removed_instance = not is_new_instance and (is_removed or - instance['vm_state'] in vm_states.ALLOW_RESOURCE_REMOVAL) + vm_states.allow_resource_removal( + vm_state=instance['vm_state'], task_state=instance.task_state)) if is_new_instance: self.tracked_instances.add(uuid) @@ -1554,7 +1555,9 @@ def _update_usage_from_instances(self, context, instances, nodename): instance_by_uuid = {} for instance in instances: - if instance.vm_state not in vm_states.ALLOW_RESOURCE_REMOVAL: + if not vm_states.allow_resource_removal( + vm_state=instance['vm_state'], + task_state=instance.task_state): self._update_usage_from_instance(context, instance, nodename) instance_by_uuid[instance.uuid] = instance return instance_by_uuid diff --git a/nova/compute/stats.py b/nova/compute/stats.py index cfbee2e6bc1..e9180ec6d6d 100644 --- a/nova/compute/stats.py +++ b/nova/compute/stats.py @@ -105,7 +105,8 @@ def update_stats_for_instance(self, instance, is_removed=False): (vm_state, task_state, os_type, project_id) = \ self._extract_state_from_instance(instance) - if is_removed or vm_state in vm_states.ALLOW_RESOURCE_REMOVAL: + if is_removed or vm_states.allow_resource_removal( + vm_state=vm_state, task_state=task_state): self._decrement("num_instances") self.states.pop(uuid) else: diff --git a/nova/compute/vm_states.py b/nova/compute/vm_states.py index 633894c1ea4..1c4da06d155 100644 --- a/nova/compute/vm_states.py +++ b/nova/compute/vm_states.py @@ -27,6 +27,7 @@ See http://wiki.openstack.org/VMState """ +from nova.compute import task_states from nova.objects import fields @@ -74,5 +75,11 @@ # states we allow to trigger crash dump ALLOW_TRIGGER_CRASH_DUMP = [ACTIVE, PAUSED, RESCUED, RESIZED, ERROR] -# states we allow resources to be freed in -ALLOW_RESOURCE_REMOVAL = [DELETED, SHELVED_OFFLOADED] + +def allow_resource_removal(vm_state, task_state=None): + """(vm_state, task_state) combinations we allow resources to be freed in""" + + return ( + vm_state == DELETED or + vm_state == SHELVED_OFFLOADED and task_state != task_states.SPAWNING + ) diff --git a/nova/conf/workarounds.py b/nova/conf/workarounds.py index 3be204f29a3..c8eedcb0e5c 100644 --- a/nova/conf/workarounds.py +++ b/nova/conf/workarounds.py @@ -416,6 +416,16 @@ help=""" When this is enabled, it will skip version-checking of hypervisors during live migration. +"""), + cfg.BoolOpt( + 'disable_deep_image_inspection', + default=False, + help=""" +This disables the additional deep image inspection that the compute node does +when downloading from glance. This includes backing-file, data-file, and +known-features detection *before* passing the image to qemu-img. Generally, +this inspection should be enabled for maximum safety, but this workaround +option allows disabling it if there is a compatibility concern. """), cfg.BoolOpt( 'skip_reserve_in_use_ironic_nodes', @@ -431,16 +441,6 @@ Howerver, if you don't use automatic cleaning, it can cause an extra delay before and Ironic node is available for building a new Nova instance. -"""), - cfg.BoolOpt( - 'disable_deep_image_inspection', - default=False, - help=""" -This disables the additional deep image inspection that the compute node does -when downloading from glance. This includes backing-file, data-file, and -known-features detection *before* passing the image to qemu-img. Generally, -this inspection should be enabled for maximum safety, but this workaround -option allows disabling it if there is a compatibility concern. """), ] diff --git a/nova/tests/functional/regressions/test_bug_2025480.py b/nova/tests/functional/regressions/test_bug_2025480.py new file mode 100644 index 00000000000..c707a40a846 --- /dev/null +++ b/nova/tests/functional/regressions/test_bug_2025480.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from unittest import mock + +from nova import context +from nova.objects import compute_node +from nova import test +from nova.tests import fixtures as nova_fixtures +from nova.tests.functional import fixtures as func_fixtures +from nova.tests.functional import integrated_helpers + + +class UnshelveUpdateAvailableResourcesPeriodicRace( + test.TestCase, integrated_helpers.InstanceHelperMixin): + def setUp(self): + super(UnshelveUpdateAvailableResourcesPeriodicRace, self).setUp() + + placement = func_fixtures.PlacementFixture() + self.useFixture(placement) + self.placement = placement.api + self.neutron = nova_fixtures.NeutronFixture(self) + self.useFixture(self.neutron) + self.useFixture(nova_fixtures.GlanceFixture(self)) + # Start nova services. + self.api = self.useFixture(nova_fixtures.OSAPIFixture( + api_version='v2.1')).admin_api + self.api.microversion = 'latest' + self.notifier = self.useFixture( + nova_fixtures.NotificationFixture(self)) + + self.start_service('conductor') + self.start_service('scheduler') + + def test_unshelve_spawning_update_available_resources(self): + compute = self._start_compute('compute1') + + server = self._create_server( + networks=[{'port': self.neutron.port_1['id']}]) + + node = compute_node.ComputeNode.get_by_nodename( + context.get_admin_context(), 'compute1') + self.assertEqual(1, node.vcpus_used) + + # with default config shelve means immediate offload as well + req = { + 'shelve': {} + } + self.api.post_server_action(server['id'], req) + self._wait_for_server_parameter( + server, {'status': 'SHELVED_OFFLOADED', + 'OS-EXT-SRV-ATTR:host': None}) + + node = compute_node.ComputeNode.get_by_nodename( + context.get_admin_context(), 'compute1') + self.assertEqual(0, node.vcpus_used) + + def fake_spawn(*args, **kwargs): + self._run_periodics() + + with mock.patch.object( + compute.driver, 'spawn', side_effect=fake_spawn): + req = {'unshelve': None} + self.api.post_server_action(server['id'], req) + self.notifier.wait_for_versioned_notifications( + 'instance.unshelve.start') + self._wait_for_server_parameter( + server, + { + 'status': 'ACTIVE', + 'OS-EXT-STS:task_state': None, + 'OS-EXT-SRV-ATTR:host': 'compute1', + }) + + node = compute_node.ComputeNode.get_by_nodename( + context.get_admin_context(), 'compute1') + # After the fix, the instance should have resources claimed + self.assertEqual(1, node.vcpus_used) diff --git a/nova/tests/unit/compute/test_stats.py b/nova/tests/unit/compute/test_stats.py index e713794a19a..b95475f09db 100644 --- a/nova/tests/unit/compute/test_stats.py +++ b/nova/tests/unit/compute/test_stats.py @@ -208,6 +208,22 @@ def test_update_stats_for_instance_offloaded(self): self.assertEqual(0, self.stats.num_os_type("Linux")) self.assertEqual(0, self.stats["num_vm_" + vm_states.BUILDING]) + def test_update_stats_for_instance_being_unshelved(self): + instance = self._create_instance() + self.stats.update_stats_for_instance(instance) + self.assertEqual(1, self.stats.num_instances_for_project("1234")) + + instance["vm_state"] = vm_states.SHELVED_OFFLOADED + instance["task_state"] = task_states.SPAWNING + self.stats.update_stats_for_instance(instance) + + self.assertEqual(1, self.stats.num_instances) + self.assertEqual(1, self.stats.num_instances_for_project(1234)) + self.assertEqual(1, self.stats["num_os_type_Linux"]) + self.assertEqual(1, self.stats["num_vm_%s" % + vm_states.SHELVED_OFFLOADED]) + self.assertEqual(1, self.stats["num_task_%s" % task_states.SPAWNING]) + def test_io_workload(self): vms = [vm_states.ACTIVE, vm_states.BUILDING, vm_states.PAUSED] tasks = [task_states.RESIZE_MIGRATING, task_states.REBUILDING, diff --git a/nova/tests/unit/image/test_format_inspector.py b/nova/tests/unit/image/test_format_inspector.py index 8406dfca378..f74731f22df 100644 --- a/nova/tests/unit/image/test_format_inspector.py +++ b/nova/tests/unit/image/test_format_inspector.py @@ -426,12 +426,14 @@ def test_qcow2_safety_checks(self): inspector = format_inspector.QcowInspector.from_file(fn) self.assertFalse(inspector.safety_check()) + # Note(lajoskatona): This image create fails on bionic due to + # old qemu-img utilities, let's skip this only test from yoga # A data-file makes it unsafe - fn = self._create_img('qcow2', 5 * units.Mi, - options={'data_file': data_fn, - 'data_file_raw': 'on'}) - inspector = format_inspector.QcowInspector.from_file(fn) - self.assertFalse(inspector.safety_check()) + # fn = self._create_img('qcow2', 5 * units.Mi, + # options={'data_file': data_fn, + # 'data_file_raw': 'on'}) + # inspector = format_inspector.QcowInspector.from_file(fn) + # self.assertFalse(inspector.safety_check()) # Trying to load a non-QCOW file is an error self.assertRaises(format_inspector.ImageFormatError, diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index 0e053f7d531..bdb8bc18aa8 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -8848,7 +8848,7 @@ def test_unquiesce(self, mock_has_min_version): def test_create_snapshot_metadata(self): base = objects.ImageMeta.from_dict( - {'disk_format': 'raw'}) + {'disk_format': 'qcow2'}) instance_data = {'kernel_id': 'kernel', 'project_id': 'prj_id', 'ramdisk_id': 'ram_id', @@ -8880,10 +8880,12 @@ def test_create_snapshot_metadata(self): {'disk_format': 'ami', 'container_format': 'test_container'}) expected['properties']['os_type'] = instance['os_type'] - expected['disk_format'] = base.disk_format + # The disk_format of the snapshot should be the *actual* format of the + # thing we upload, regardless of what type of image we booted from. + expected['disk_format'] = img_fmt expected['container_format'] = base.container_format ret = drvr._create_snapshot_metadata(base, instance, img_fmt, snp_name) - self.assertEqual(ret, expected) + self.assertEqual(expected, ret) def test_get_volume_driver(self): conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) @@ -28225,7 +28227,8 @@ def test_ami(self): utils.get_system_metadata_from_image( {'disk_format': 'ami'}) - self._test_snapshot(disk_format='ami') + # If we're uploading a qcow2, we must set the disk_format as such + self._test_snapshot(disk_format='qcow2') @mock.patch('nova.virt.libvirt.utils.get_disk_type_from_path', new=mock.Mock(return_value=None)) diff --git a/nova/tests/unit/virt/test_virt_drivers.py b/nova/tests/unit/virt/test_virt_drivers.py index e275cd3e3aa..3ca1f5dcb5e 100644 --- a/nova/tests/unit/virt/test_virt_drivers.py +++ b/nova/tests/unit/virt/test_virt_drivers.py @@ -838,6 +838,11 @@ def setUp(self): # since we don't care about it. self.stub_out('os_vif.unplug', lambda a, kw: None) self.stub_out('nova.compute.utils.get_machine_ips', lambda: []) + self.stub_out('nova.virt.libvirt.utils.get_disk_size', + lambda *a, **k: 123456) + self.stub_out('nova.virt.libvirt.utils.get_disk_backing_file', + lambda *a, **k: None) + self.stub_out('nova.privsep.path.chown', lambda *a, **k: None) def test_init_host_image_type_rbd_force_raw_images_true(self): CONF.set_override('images_type', 'rbd', group='libvirt') diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 4de51ce8e31..3353d712b39 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -2929,11 +2929,7 @@ def _create_snapshot_metadata(self, image_meta, instance, if instance.os_type: metadata['properties']['os_type'] = instance.os_type - # NOTE(vish): glance forces ami disk format to be ami - if image_meta.disk_format == 'ami': - metadata['disk_format'] = 'ami' - else: - metadata['disk_format'] = img_fmt + metadata['disk_format'] = img_fmt if image_meta.obj_attr_is_set("container_format"): metadata['container_format'] = image_meta.container_format diff --git a/tox.ini b/tox.ini index 92edcd7b469..8df61ef43bb 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,11 @@ envlist = py39,functional,pep8 # env and ignore basepython inherited from [testenv] if we set # ignore_basepython_conflict. ignore_basepython_conflict = True +# Cap setuptools via virtualenv to prevent compatibility issue with yoga +# branch's upper constraint of 'packaging' package (21.3). +requires = + virtualenv<20.26.4 + tox<4 [testenv] basepython = python3