Skip to content

Commit

Permalink
Merge pull request #116 from stackhpc/upstream/yoga-2024-10-07
Browse files Browse the repository at this point in the history
Synchronise yoga with upstream
  • Loading branch information
priteau authored Oct 7, 2024
2 parents 651fed4 + 084b4d3 commit 36746a3
Show file tree
Hide file tree
Showing 13 changed files with 182 additions and 77 deletions.
71 changes: 26 additions & 45 deletions .zuul.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -197,24 +197,11 @@
parent: devstack-tempest
description: |
Run tempest compute API tests using LVM image backend. This only runs
against nova/virt/libvirt/* changes.
# Copy irrelevant-files from nova-dsvm-multinode-base and then exclude
# anything that is not in nova/virt/libvirt/* or nova/privsep/*.
irrelevant-files:
- ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/privsep/).*$
- ^api-.*$
- ^(test-|)requirements.txt$
- ^.*\.rst$
- ^.git.*$
- ^doc/.*$
- ^nova/hacking/.*$
- ^nova/locale/.*$
- ^nova/tests/.*$
- ^nova/test.py$
- ^releasenotes/.*$
- ^setup.cfg$
- ^tools/.*$
- ^tox.ini$
against nova/virt/libvirt/*, nova/privsep/* and .zuul.yaml changes.
files:
- ^nova/virt/libvirt/.*$
- ^nova/privsep/.*$
- .zuul.yaml
vars:
# We use the "all" environment for tempest_test_regex and
# tempest_exclude_regex.
Expand Down Expand Up @@ -253,22 +240,11 @@
# NOTE(chateaulav): due to constraints with no IDE support for aarch64,
# tests have been limited to eliminate any items that are incompatible.
# This is to be re-evaluated as greater support is added and defined.
irrelevant-files:
- ^(?!.zuul.yaml)(?!nova/virt/libvirt/)(?!nova/objects/)(?!nova/scheduler/).*$
- ^api-.*$
- ^(test-|)requirements.txt$
- ^.*\.rst$
- ^.git.*$
- ^doc/.*$
- ^nova/hacking/.*$
- ^nova/locale/.*$
- ^nova/policies/.*$
- ^nova/tests/.*$
- ^nova/test.py$
- ^releasenotes/.*$
- ^setup.cfg$
- ^tools/.*$
- ^tox.ini$
files:
- ^nova/virt/libvirt/.*$
- ^nova/objects/.*$
- ^nova/scheduler/.*$
- .zuul.yaml
vars:
tox_envlist: all
tempest_test_regex: ^tempest\.(api\.compute\.servers|scenario\.test_network_basic_ops)
Expand Down Expand Up @@ -580,6 +556,16 @@
GLANCE_STANDALONE: True
GLANCE_USE_IMPORT_WORKFLOW: True
DEVSTACK_PARALLEL: True
MYSQL_REDUCE_MEMORY: True
# NOTE(danms): This job is pretty heavy as it is, so we disable some
# services that are not relevant to the nova-glance-ceph scenario
# that this job is intended to validate.
devstack_services:
c-bak: false
s-account: false
s-container: false
s-object: false
s-proxy: false
devstack_local_conf:
post-config:
$NOVA_CONF:
Expand Down Expand Up @@ -632,11 +618,12 @@
- nova-ceph-multistore:
irrelevant-files: *nova-base-irrelevant-files
- neutron-linuxbridge-tempest:
irrelevant-files:
files:
# NOTE(mriedem): This job has its own irrelevant-files section
# so that we only run it on changes to networking and libvirt/vif
# code; we don't need to run this on all changes.
- ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$
- ^nova/network/.*$
- nova/virt/libvirt/vif.py
- nova-live-migration
- nova-live-migration-ceph
- nova-lvm
Expand Down Expand Up @@ -680,11 +667,6 @@
- barbican-tempest-plugin-simple-crypto:
irrelevant-files: *nova-base-irrelevant-files
voting: false
- tempest-integrated-compute-centos-8-stream:
irrelevant-files: *nova-base-irrelevant-files
- tempest-centos8-stream-fips:
irrelevant-files: *nova-base-irrelevant-files
voting: false
gate:
jobs:
- nova-live-migration
Expand All @@ -697,11 +679,12 @@
- nova-ceph-multistore:
irrelevant-files: *nova-base-irrelevant-files
- neutron-linuxbridge-tempest:
irrelevant-files:
files:
# NOTE(mriedem): This job has its own irrelevant-files section
# so that we only run it on changes to networking and libvirt/vif
# code; we don't need to run this on all changes.
- ^(?!nova/network/.*)(?!nova/virt/libvirt/vif.py).*$
- ^nova/network/.*$
- nova/virt/libvirt/vif.py
- tempest-integrated-compute:
irrelevant-files: *policies-irrelevant-files
- nova-grenade-multinode:
Expand All @@ -710,8 +693,6 @@
irrelevant-files: *nova-base-irrelevant-files
- openstacksdk-functional-devstack:
irrelevant-files: *nova-base-irrelevant-files
- tempest-integrated-compute-centos-8-stream:
irrelevant-files: *nova-base-irrelevant-files
experimental:
jobs:
- ironic-tempest-bfv:
Expand Down
6 changes: 3 additions & 3 deletions nova/compute/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -6646,9 +6646,9 @@ def _shelve_offload_instance(self, context, instance, clean_shutdown,

instance.power_state = current_power_state
# NOTE(mriedem): The vm_state has to be set before updating the
# resource tracker, see vm_states.ALLOW_RESOURCE_REMOVAL. The host/node
# values cannot be nulled out until after updating the resource tracker
# though.
# resource tracker, see vm_states.allow_resource_removal(). The
# host/node values cannot be nulled out until after updating the
# resource tracker though.
instance.vm_state = vm_states.SHELVED_OFFLOADED
instance.task_state = None
instance.save(expected_task_state=[task_states.SHELVING,
Expand Down
7 changes: 5 additions & 2 deletions nova/compute/resource_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -1495,7 +1495,8 @@ def _update_usage_from_instance(self, context, instance, nodename,
# NOTE(sfinucan): Both brand new instances as well as instances that
# are being unshelved will have is_new_instance == True
is_removed_instance = not is_new_instance and (is_removed or
instance['vm_state'] in vm_states.ALLOW_RESOURCE_REMOVAL)
vm_states.allow_resource_removal(
vm_state=instance['vm_state'], task_state=instance.task_state))

if is_new_instance:
self.tracked_instances.add(uuid)
Expand Down Expand Up @@ -1554,7 +1555,9 @@ def _update_usage_from_instances(self, context, instances, nodename):

instance_by_uuid = {}
for instance in instances:
if instance.vm_state not in vm_states.ALLOW_RESOURCE_REMOVAL:
if not vm_states.allow_resource_removal(
vm_state=instance['vm_state'],
task_state=instance.task_state):
self._update_usage_from_instance(context, instance, nodename)
instance_by_uuid[instance.uuid] = instance
return instance_by_uuid
Expand Down
3 changes: 2 additions & 1 deletion nova/compute/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ def update_stats_for_instance(self, instance, is_removed=False):
(vm_state, task_state, os_type, project_id) = \
self._extract_state_from_instance(instance)

if is_removed or vm_state in vm_states.ALLOW_RESOURCE_REMOVAL:
if is_removed or vm_states.allow_resource_removal(
vm_state=vm_state, task_state=task_state):
self._decrement("num_instances")
self.states.pop(uuid)
else:
Expand Down
11 changes: 9 additions & 2 deletions nova/compute/vm_states.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
See http://wiki.openstack.org/VMState
"""

from nova.compute import task_states
from nova.objects import fields


Expand Down Expand Up @@ -74,5 +75,11 @@
# states we allow to trigger crash dump
ALLOW_TRIGGER_CRASH_DUMP = [ACTIVE, PAUSED, RESCUED, RESIZED, ERROR]

# states we allow resources to be freed in
ALLOW_RESOURCE_REMOVAL = [DELETED, SHELVED_OFFLOADED]

def allow_resource_removal(vm_state, task_state=None):
"""(vm_state, task_state) combinations we allow resources to be freed in"""

return (
vm_state == DELETED or
vm_state == SHELVED_OFFLOADED and task_state != task_states.SPAWNING
)
20 changes: 10 additions & 10 deletions nova/conf/workarounds.py
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,16 @@
help="""
When this is enabled, it will skip version-checking of hypervisors
during live migration.
"""),
cfg.BoolOpt(
'disable_deep_image_inspection',
default=False,
help="""
This disables the additional deep image inspection that the compute node does
when downloading from glance. This includes backing-file, data-file, and
known-features detection *before* passing the image to qemu-img. Generally,
this inspection should be enabled for maximum safety, but this workaround
option allows disabling it if there is a compatibility concern.
"""),
cfg.BoolOpt(
'skip_reserve_in_use_ironic_nodes',
Expand All @@ -431,16 +441,6 @@
Howerver, if you don't use automatic cleaning, it can cause an
extra delay before and Ironic node is available for building a
new Nova instance.
"""),
cfg.BoolOpt(
'disable_deep_image_inspection',
default=False,
help="""
This disables the additional deep image inspection that the compute node does
when downloading from glance. This includes backing-file, data-file, and
known-features detection *before* passing the image to qemu-img. Generally,
this inspection should be enabled for maximum safety, but this workaround
option allows disabling it if there is a compatibility concern.
"""),
]

Expand Down
86 changes: 86 additions & 0 deletions nova/tests/functional/regressions/test_bug_2025480.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock

from nova import context
from nova.objects import compute_node
from nova import test
from nova.tests import fixtures as nova_fixtures
from nova.tests.functional import fixtures as func_fixtures
from nova.tests.functional import integrated_helpers


class UnshelveUpdateAvailableResourcesPeriodicRace(
test.TestCase, integrated_helpers.InstanceHelperMixin):
def setUp(self):
super(UnshelveUpdateAvailableResourcesPeriodicRace, self).setUp()

placement = func_fixtures.PlacementFixture()
self.useFixture(placement)
self.placement = placement.api
self.neutron = nova_fixtures.NeutronFixture(self)
self.useFixture(self.neutron)
self.useFixture(nova_fixtures.GlanceFixture(self))
# Start nova services.
self.api = self.useFixture(nova_fixtures.OSAPIFixture(
api_version='v2.1')).admin_api
self.api.microversion = 'latest'
self.notifier = self.useFixture(
nova_fixtures.NotificationFixture(self))

self.start_service('conductor')
self.start_service('scheduler')

def test_unshelve_spawning_update_available_resources(self):
compute = self._start_compute('compute1')

server = self._create_server(
networks=[{'port': self.neutron.port_1['id']}])

node = compute_node.ComputeNode.get_by_nodename(
context.get_admin_context(), 'compute1')
self.assertEqual(1, node.vcpus_used)

# with default config shelve means immediate offload as well
req = {
'shelve': {}
}
self.api.post_server_action(server['id'], req)
self._wait_for_server_parameter(
server, {'status': 'SHELVED_OFFLOADED',
'OS-EXT-SRV-ATTR:host': None})

node = compute_node.ComputeNode.get_by_nodename(
context.get_admin_context(), 'compute1')
self.assertEqual(0, node.vcpus_used)

def fake_spawn(*args, **kwargs):
self._run_periodics()

with mock.patch.object(
compute.driver, 'spawn', side_effect=fake_spawn):
req = {'unshelve': None}
self.api.post_server_action(server['id'], req)
self.notifier.wait_for_versioned_notifications(
'instance.unshelve.start')
self._wait_for_server_parameter(
server,
{
'status': 'ACTIVE',
'OS-EXT-STS:task_state': None,
'OS-EXT-SRV-ATTR:host': 'compute1',
})

node = compute_node.ComputeNode.get_by_nodename(
context.get_admin_context(), 'compute1')
# After the fix, the instance should have resources claimed
self.assertEqual(1, node.vcpus_used)
16 changes: 16 additions & 0 deletions nova/tests/unit/compute/test_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,22 @@ def test_update_stats_for_instance_offloaded(self):
self.assertEqual(0, self.stats.num_os_type("Linux"))
self.assertEqual(0, self.stats["num_vm_" + vm_states.BUILDING])

def test_update_stats_for_instance_being_unshelved(self):
instance = self._create_instance()
self.stats.update_stats_for_instance(instance)
self.assertEqual(1, self.stats.num_instances_for_project("1234"))

instance["vm_state"] = vm_states.SHELVED_OFFLOADED
instance["task_state"] = task_states.SPAWNING
self.stats.update_stats_for_instance(instance)

self.assertEqual(1, self.stats.num_instances)
self.assertEqual(1, self.stats.num_instances_for_project(1234))
self.assertEqual(1, self.stats["num_os_type_Linux"])
self.assertEqual(1, self.stats["num_vm_%s" %
vm_states.SHELVED_OFFLOADED])
self.assertEqual(1, self.stats["num_task_%s" % task_states.SPAWNING])

def test_io_workload(self):
vms = [vm_states.ACTIVE, vm_states.BUILDING, vm_states.PAUSED]
tasks = [task_states.RESIZE_MIGRATING, task_states.REBUILDING,
Expand Down
12 changes: 7 additions & 5 deletions nova/tests/unit/image/test_format_inspector.py
Original file line number Diff line number Diff line change
Expand Up @@ -426,12 +426,14 @@ def test_qcow2_safety_checks(self):
inspector = format_inspector.QcowInspector.from_file(fn)
self.assertFalse(inspector.safety_check())

# Note(lajoskatona): This image create fails on bionic due to
# old qemu-img utilities, let's skip this only test from yoga
# A data-file makes it unsafe
fn = self._create_img('qcow2', 5 * units.Mi,
options={'data_file': data_fn,
'data_file_raw': 'on'})
inspector = format_inspector.QcowInspector.from_file(fn)
self.assertFalse(inspector.safety_check())
# fn = self._create_img('qcow2', 5 * units.Mi,
# options={'data_file': data_fn,
# 'data_file_raw': 'on'})
# inspector = format_inspector.QcowInspector.from_file(fn)
# self.assertFalse(inspector.safety_check())

# Trying to load a non-QCOW file is an error
self.assertRaises(format_inspector.ImageFormatError,
Expand Down
11 changes: 7 additions & 4 deletions nova/tests/unit/virt/libvirt/test_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -8848,7 +8848,7 @@ def test_unquiesce(self, mock_has_min_version):

def test_create_snapshot_metadata(self):
base = objects.ImageMeta.from_dict(
{'disk_format': 'raw'})
{'disk_format': 'qcow2'})
instance_data = {'kernel_id': 'kernel',
'project_id': 'prj_id',
'ramdisk_id': 'ram_id',
Expand Down Expand Up @@ -8880,10 +8880,12 @@ def test_create_snapshot_metadata(self):
{'disk_format': 'ami',
'container_format': 'test_container'})
expected['properties']['os_type'] = instance['os_type']
expected['disk_format'] = base.disk_format
# The disk_format of the snapshot should be the *actual* format of the
# thing we upload, regardless of what type of image we booted from.
expected['disk_format'] = img_fmt
expected['container_format'] = base.container_format
ret = drvr._create_snapshot_metadata(base, instance, img_fmt, snp_name)
self.assertEqual(ret, expected)
self.assertEqual(expected, ret)

def test_get_volume_driver(self):
conn = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
Expand Down Expand Up @@ -28225,7 +28227,8 @@ def test_ami(self):
utils.get_system_metadata_from_image(
{'disk_format': 'ami'})

self._test_snapshot(disk_format='ami')
# If we're uploading a qcow2, we must set the disk_format as such
self._test_snapshot(disk_format='qcow2')

@mock.patch('nova.virt.libvirt.utils.get_disk_type_from_path',
new=mock.Mock(return_value=None))
Expand Down
Loading

0 comments on commit 36746a3

Please sign in to comment.