diff --git a/.zuul.yaml b/.zuul.yaml index f04d00b387..601bfb6acf 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -206,6 +206,8 @@ The regular tempest-integrated-storage job but with glance metadata injection post-run: playbooks/post-check-metadata-injection.yaml vars: + configure_swap_size: 8192 + tempest_concurrency: 3 zuul_copy_output: /etc/glance-remote: logs devstack_localrc: @@ -314,7 +316,6 @@ - release-notes-jobs-python3 check: jobs: - - openstack-tox-functional-py38-fips - openstack-tox-functional-py39 - glance-tox-functional-py39-rbac-defaults - glance-ceph-thin-provisioning: diff --git a/glance/cmd/cache_prefetcher.py b/glance/cmd/cache_prefetcher.py index 9a6c521d3c..f6e35e8086 100644 --- a/glance/cmd/cache_prefetcher.py +++ b/glance/cmd/cache_prefetcher.py @@ -36,6 +36,7 @@ import glance_store from oslo_log import log as logging +import glance.async_ from glance.common import config from glance.image_cache import prefetcher @@ -49,6 +50,7 @@ def main(): config.parse_cache_args() logging.setup(CONF, 'glance') CONF.import_opt('enabled_backends', 'glance.common.wsgi') + glance.async_.set_threadpool_model('eventlet') if CONF.enabled_backends: glance_store.register_store_opts(CONF) diff --git a/glance/common/format_inspector.py b/glance/common/format_inspector.py index 550cceadbb..d9576f1f8d 100755 --- a/glance/common/format_inspector.py +++ b/glance/common/format_inspector.py @@ -512,7 +512,7 @@ def __str__(self): # # https://www.vmware.com/app/vmdk/?src=vmdk class VMDKInspector(FileInspector): - """vmware VMDK format (monolithicSparse variant only) + """vmware VMDK format (monolithicSparse and streamOptimized variants only) This needs to store the 512 byte header and the descriptor region which should be just after that. The descriptor region is some @@ -582,7 +582,7 @@ def virtual_size(self): vmdktype = descriptor[type_idx:type_end] else: vmdktype = b'formatnotfound' - if vmdktype != b'monolithicSparse': + if vmdktype not in (b'monolithicSparse', b'streamOptimized'): LOG.warning('Unsupported VMDK format %s', vmdktype) return 0 diff --git a/glance/tests/unit/common/test_format_inspector.py b/glance/tests/unit/common/test_format_inspector.py index db6a9830bd..38f8caeb41 100644 --- a/glance/tests/unit/common/test_format_inspector.py +++ b/glance/tests/unit/common/test_format_inspector.py @@ -51,38 +51,51 @@ def tearDown(self): except Exception: pass - def _create_img(self, fmt, size): + def _create_img(self, fmt, size, subformat=None): if fmt == 'vhd': # QEMU calls the vhd format vpc fmt = 'vpc' - fn = tempfile.mktemp(prefix='glance-unittest-formatinspector-', + opt = '' + prefix = 'glance-unittest-formatinspector-' + + if subformat: + opt = ' -o subformat=%s' % subformat + prefix += subformat + '-' + + fn = tempfile.mktemp(prefix=prefix, suffix='.%s' % fmt) self._created_files.append(fn) subprocess.check_output( - 'qemu-img create -f %s %s %i' % (fmt, fn, size), + 'qemu-img create -f %s %s %s %i' % (fmt, opt, fn, size), shell=True) return fn - def _create_allocated_vmdk(self, size_mb): + def _create_allocated_vmdk(self, size_mb, subformat=None): # We need a "big" VMDK file to exercise some parts of the code of the # format_inspector. A way to create one is to first create an empty # file, and then to convert it with the -S 0 option. - fn = tempfile.mktemp(prefix='glance-unittest-formatinspector-', - suffix='.vmdk') + + if subformat is None: + # Matches qemu-img default, see `qemu-img convert -O vmdk -o help` + subformat = 'monolithicSparse' + + prefix = 'glance-unittest-formatinspector-%s-' % subformat + fn = tempfile.mktemp(prefix=prefix, suffix='.vmdk') self._created_files.append(fn) - zeroes = tempfile.mktemp(prefix='glance-unittest-formatinspector-', - suffix='.zero') - self._created_files.append(zeroes) + raw = tempfile.mktemp(prefix=prefix, suffix='.raw') + self._created_files.append(raw) - # Create an empty file + # Create a file with pseudo-random data, otherwise it will get + # compressed in the streamOptimized format subprocess.check_output( - 'dd if=/dev/zero of=%s bs=1M count=%i' % (zeroes, size_mb), + 'dd if=/dev/urandom of=%s bs=1M count=%i' % (raw, size_mb), shell=True) # Convert it to VMDK subprocess.check_output( - 'qemu-img convert -f raw -O vmdk -S 0 %s %s' % (zeroes, fn), + 'qemu-img convert -f raw -O vmdk -o subformat=%s -S 0 %s %s' % ( + subformat, raw, fn), shell=True) return fn @@ -101,8 +114,9 @@ def _test_format_at_block_size(self, format_name, img, block_size): wrapper.close() return fmt - def _test_format_at_image_size(self, format_name, image_size): - img = self._create_img(format_name, image_size) + def _test_format_at_image_size(self, format_name, image_size, + subformat=None): + img = self._create_img(format_name, image_size, subformat=subformat) # Some formats have internal alignment restrictions making this not # always exactly like image_size, so get the real value for comparison @@ -124,11 +138,12 @@ def _test_format_at_image_size(self, format_name, image_size): 'Format used more than 512KiB of memory: %s' % ( fmt.context_info)) - def _test_format(self, format_name): + def _test_format(self, format_name, subformat=None): # Try a few different image sizes, including some odd and very small # sizes for image_size in (512, 513, 2057, 7): - self._test_format_at_image_size(format_name, image_size * units.Mi) + self._test_format_at_image_size(format_name, image_size * units.Mi, + subformat=subformat) def test_qcow2(self): self._test_format('qcow2') @@ -142,12 +157,15 @@ def test_vhdx(self): def test_vmdk(self): self._test_format('vmdk') - def test_vmdk_bad_descriptor_offset(self): + def test_vmdk_stream_optimized(self): + self._test_format('vmdk', 'streamOptimized') + + def _test_vmdk_bad_descriptor_offset(self, subformat=None): format_name = 'vmdk' image_size = 10 * units.Mi descriptorOffsetAddr = 0x1c BAD_ADDRESS = 0x400 - img = self._create_img(format_name, image_size) + img = self._create_img(format_name, image_size, subformat=subformat) # Corrupt the header fd = open(img, 'r+b') @@ -167,7 +185,13 @@ def test_vmdk_bad_descriptor_offset(self): 'size %i block %i') % (format_name, image_size, block_size)) - def test_vmdk_bad_descriptor_mem_limit(self): + def test_vmdk_bad_descriptor_offset(self): + self._test_vmdk_bad_descriptor_offset() + + def test_vmdk_bad_descriptor_offset_stream_optimized(self): + self._test_vmdk_bad_descriptor_offset(subformat='streamOptimized') + + def _test_vmdk_bad_descriptor_mem_limit(self, subformat=None): format_name = 'vmdk' image_size = 5 * units.Mi virtual_size = 5 * units.Mi @@ -176,7 +200,8 @@ def test_vmdk_bad_descriptor_mem_limit(self): twoMBInSectors = (2 << 20) // 512 # We need a big VMDK because otherwise we will not have enough data to # fill-up the CaptureRegion. - img = self._create_allocated_vmdk(image_size // units.Mi) + img = self._create_allocated_vmdk(image_size // units.Mi, + subformat=subformat) # Corrupt the end of descriptor address so it "ends" at 2MB fd = open(img, 'r+b') @@ -200,6 +225,12 @@ def test_vmdk_bad_descriptor_mem_limit(self): 'Format used more than 1.5MiB of memory: %s' % ( fmt.context_info)) + def test_vmdk_bad_descriptor_mem_limit(self): + self._test_vmdk_bad_descriptor_mem_limit() + + def test_vmdk_bad_descriptor_mem_limit_stream_optimized(self): + self._test_vmdk_bad_descriptor_mem_limit(subformat='streamOptimized') + def test_vdi(self): self._test_format('vdi') diff --git a/playbooks/post-check-metadata-injection.yaml b/playbooks/post-check-metadata-injection.yaml index 178866ba1b..2c273693a5 100644 --- a/playbooks/post-check-metadata-injection.yaml +++ b/playbooks/post-check-metadata-injection.yaml @@ -9,12 +9,16 @@ set -xe cirrosimg=$(glance image-list | grep cirros | cut -d" " -f 2) - echo "Dumping the cirros image for debugging..." - glance image-show $cirrosimg + # There could be more than one cirros image so traverse through the list + for image in $cirrosimg + do + echo "Dumping the cirros image for debugging..." + glance image-show $image - echo "Checking that the cirros image was decorated with metdata on import..." - glance image-list --property-filter 'glance_devstack_test=doyouseeme?' | grep cirros + echo "Checking that the cirros image was decorated with metdata on import..." + glance image-list --property-filter 'glance_devstack_test=doyouseeme?' | grep $image - echo "Checking that the cirros image was converted to raw on import..." - glance image-show $cirrosimg | egrep -e 'disk_format.*raw' + echo "Checking that the cirros image was converted to raw on import..." + glance image-show $image | egrep -e 'disk_format.*raw' + done environment: '{{ zuul | zuul_legacy_vars }}'