Skip to content

Commit

Permalink
Improve compilation speed for android build
Browse files Browse the repository at this point in the history 
FFTF python script used for flashfiles.zip in IVI. It uses
ZipWrite python api to create it. This method is time consuming
and adds several unwanted build steps that adds overhead. Like,
flash.json generation, unpacking of target files, installer.cmd
generation.

This fix, changes FFTF python to a normal FFTF bash script, which
uses tar + pigz combo for faster flashfiles generation.

NOTE:
We deprecate flash.json as its used only by PFT & installer.cmd
will be maintained in product config repo.

Tracked-On: OAM-112083
Signed-off-by: sgnanase <[email protected]>
  • Loading branch information
@sgnanase @swaroopbalan
sgnanase authored and swaroopbalan committed Oct 12, 2023
1 parent 9489d80 commit 1870d5f
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 24 deletions.
39 changes: 39 additions & 0 deletions releasetools/flashfiles_from_target_files.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/bash

echo "Preparing flashfiles - New Method"
echo "No more Zip. We use tar + pigz"

echo "========================"
echo "Images / Files to be packed"
echo "========================"
echo "acpio.img"
echo "boot.img"
echo "bootloader.img"
echo "config.img"
echo "flash.json"
echo "gpt.bin"
echo "installer.cmd"
echo "installer.efi "
echo "odm.img "
echo "startup.nsh "
echo "super_empty.img "
echo "super.img "
echo "vbmeta.img "
echo "vendor_boot.img "
echo "========================"


#tar
flashfile=`basename $1`
flashfile_dir=`echo $flashfile | sed 's/\.tar\.gz//g'`
PRODUCT_OUT=`dirname $1`

cd $PRODUCT_OUT
rm -rf $flashfile_dir
mkdir $flashfile_dir
cp acpio.img boot.img bootloader.img config.img gpt.bin installer.cmd efi/installer.efi odm.img efi/startup.nsh super_empty.img ./obj/PACKAGING/super.img_intermediates/super.img vbmeta.img vendor_boot.img $flashfile_dir/.
tar -cvf - $flashfile_dir/ | /usr/bin/pigz > $flashfile

echo "========================"
echo "Flashfiles Tar created"
echo "========================"
44 changes: 20 additions & 24 deletions tasks/flashfiles.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ endif
endif

ifeq ($(USE_INTEL_FLASHFILES),true)
fftf := $(INTEL_PATH_BUILD)/releasetools/flashfiles_from_target_files
fftf := $(INTEL_PATH_BUILD)/releasetools/flashfiles_from_target_files.sh
#fftf := $(INTEL_PATH_BUILD)/releasetools/flashfiles_from_target_files
odf := $(INTEL_PATH_BUILD)/releasetools/ota_deployment_fixup

ifneq ($(FLASHFILE_VARIANTS),)
Expand All @@ -92,7 +93,7 @@ ifneq ($(FLASHFILE_VARIANTS),)
$(info Adding $(var)) \
$(eval fn_prefix := $(PRODUCT_OUT)/$(TARGET_PRODUCT)) \
$(eval fn_suffix := $(var)-$(FILE_NAME_TAG)) \
$(eval ff_zip := $(fn_prefix)-flashfiles-$(fn_suffix).zip) \
$(eval ff_zip := $(fn_prefix)-flashfiles-$(fn_suffix).tar.gz) \
$(eval INTEL_FACTORY_FLASHFILES_TARGET += $(ff_zip)) \
$(call dist-for-goals,droidcore,$(ff_zip):$(notdir $(ff_zip))))

Expand Down Expand Up @@ -134,20 +135,20 @@ ifneq ($(FLASHFILE_VARIANTS),)
FLASHFILES_ADD_ARGS := '--unified-variants'
endif

INTEL_FACTORY_FLASHFILES_TARGET := $(PRODUCT_OUT)/$(name).zip
INTEL_FACTORY_FLASHFILES_TARGET := $(PRODUCT_OUT)/$(name).tar.gz

ifneq ($(SOFIA_FIRMWARE_VARIANTS),)
mvcfg_default_arg = $(MV_CONFIG_DEFAULT_TYPE.$(firstword $(SOFIA_FIRMWARE_VARIANTS)))
else
mvcfg_default_arg = $(MV_CONFIG_DEFAULT_TYPE)
endif

##########################################
##########################################
ifeq ($(SUPER_IMG_IN_FLASHZIP),true)
$(INTEL_FACTORY_FLASHFILES_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(fftf) $(UEFI_ADDITIONAL_TOOLS) $(INTERNAL_SUPERIMAGE_DIST_TARGET)
$(hide) mkdir -p $(dir $@)
$(fftf) $(FLASHFILES_ADD_ARGS) --mv_config_default=$(notdir $(mvcfg_default_arg)) --add_image=$(INTERNAL_SUPERIMAGE_DIST_TARGET) $(BUILT_TARGET_FILES_PACKAGE) $@
#remove system.img vendor.img product.img from flashfiles.zip
$(hide)zip -d $@ "system.img" "product.img" "vendor.img";
$(fftf) $@
else
$(INTEL_FACTORY_FLASHFILES_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(fftf) $(UEFI_ADDITIONAL_TOOLS)
$(hide) mkdir -p $(dir $@)
Expand Down Expand Up @@ -286,7 +287,7 @@ ISO_INSTALL_IMG = $(PRODUCT_OUT)/$(TARGET_PRODUCT)-sign-flashfile-$(FILE_NAME_TA
else
ISO_INSTALL_IMG = $(PRODUCT_OUT)/$(TARGET_PRODUCT)-flashfile-$(FILE_NAME_TAG).iso
endif
ISO_INSTALL_IMG_ZIP = $(ISO_INSTALL_IMG).zip
ISO_INSTALL_IMG_ZIP = $(ISO_INSTALL_IMG).tar.gz
ISO_RELEASE_TAR = $(PRODUCT_OUT)/$(TARGET_PRODUCT)-releasefile-$(TARGET_BUILD_VARIANT).iso.tar.gz
ISO_EFI = $(PRODUCT_OUT)/iso_tmp.efi

Expand All @@ -300,7 +301,7 @@ flashfiles: $(INTEL_FACTORY_FLASHFILES_TARGET) $(BUILT_RELEASE_FLASH_FILES_PACKA
ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT)))
@echo "Publishing Release files started ..."
$(hide) mkdir -p $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.zip $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.tar.gz $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r $(PRODUCT_OUT)/scripts $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r vendor/intel/utils/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) mv $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/patches
Expand Down Expand Up @@ -349,20 +350,20 @@ flashfiles: $(INTEL_FACTORY_FLASHFILES_TARGET) publish_gptimage_var publish_mkdi
ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT)))
@echo "Publishing Release files started"
$(hide) mkdir -p $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.zip $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.tar.gz $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r $(PRODUCT_OUT)/scripts $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) cp -r vendor/intel/utils/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files
$(hide) mv $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/patches
$(hide) cp -r $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/* $(TOP)
ifneq (,$(wildcard vendor/intel/utils_vertical))
ifneq (,$(wildcard vendor/intel/fw/keybox_provisioning))
@echo "vertical_keybox_provisioning included"
$(hide) tar --exclude=*.git -czf $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip *provisioning
$(hide) tar --exclude=*.git -cvf - scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip *provisioning | /usr/bin/pigz > $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz
else
$(hide) tar --exclude=*.git -czf $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip
$(hide) tar --exclude=*.git -cvf - scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip | /usr/bin/pigz > $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz
endif
else
$(hide) tar --exclude=*.git -czf $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip
$(hide) tar --exclude=*.git -cvf - scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip | /usr/bin/pigz > $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz
endif
$(hide) cp -r $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)
$(hide) cp -r $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz $(PRODUCT_OUT)
Expand Down Expand Up @@ -405,13 +406,8 @@ ifneq ($(iso_image),false)
$(hide)rm -rf $(ISO_INSTALL_IMG) $(ISO_INSTALL_IMG_ZIP)
$(hide)mkdir -p $(PRODUCT_OUT)/efi_images_tmp;

ifeq ($(RELEASE_BUILD),true)
$(hide)unzip $(BUILT_RELEASE_FLASH_FILES_PACKAGE) -d $(PRODUCT_OUT)/efi_images_tmp/ > /dev/null;
else

$(hide)unzip $(INTEL_FACTORY_FLASHFILES_TARGET) -d $(PRODUCT_OUT)/efi_images_tmp/ > /dev/null;
endif
G_size=`echo "$$((1 << 32))"`; \
cp -r $(PRODUCT_OUT)/$(TARGET_PRODUCT)-flashfiles-*/* $(PRODUCT_OUT)/efi_images_tmp/. ; \
for img in `ls $(PRODUCT_OUT)/efi_images_tmp/`;do \
size=`stat -c %s $(PRODUCT_OUT)/efi_images_tmp/$${img}`; \
if [[ $${size} -gt $${G_size} ]]; then \
Expand Down Expand Up @@ -439,21 +435,21 @@ endif
$(hide)xorriso -as mkisofs -iso-level 3 -r -V "Civ ISO" -J -joliet-long -append_partition 2 0xef $(ISO_EFI) \
-partition_cyl_align all -o $(ISO_INSTALL_IMG) $(PRODUCT_OUT)/iso/

@echo "Zipping ISO image $(ISO_INSTALL_IMG_ZIP) ..."
$(hide)zip -r -j $(ISO_INSTALL_IMG_ZIP) $(ISO_INSTALL_IMG)
@echo "Tar ISO image $(ISO_INSTALL_IMG_ZIP) ..."
$(hide) tar -cvf - $(ISO_INSTALL_IMG) | /usr/bin/pigz > $(ISO_INSTALL_IMG_ZIP)
ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT)))
@echo "Zipping ISO release image $(ISO_RELEASE_TAR) ..."
$(hide)rm -rf $(ISO_RELEASE_TAR)
$(hide)cp $(ISO_INSTALL_IMG) $(TOP)/
ifneq (,$(wildcard vendor/intel/utils_vertical))
ifneq (,$(wildcard vendor/intel/fw/keybox_provisioning))
@echo "vertical_keybox_provisioning included"
$(hide) tar --exclude=*.git -czf $(ISO_RELEASE_TAR) scripts *patches *-flashfile-*.iso *provisioning
$(hide) tar --exclude=*.git -cvf - scripts *patches *-flashfile-*.iso *provisioning | /usr/bin/pigz > $(ISO_RELEASE_TAR)
else
$(hide) tar --exclude=*.git -czf $(ISO_RELEASE_TAR) scripts *patches *-flashfile-*.iso
$(hide) tar --exclude=*.git -cvf - scripts *patches *-flashfile-*.iso | /usr/bin/pigz > $(ISO_RELEASE_TAR)
endif
else
$(hide) tar --exclude=*.git -czf $(ISO_RELEASE_TAR) scripts *patches *-flashfile-*.iso
$(hide) tar --exclude=*.git -cvf - scripts *patches *-flashfile-*.iso | /usr/bin/pigz > $(ISO_RELEASE_TAR)
endif
endif
@echo "make ISO image done ---"
Expand All @@ -466,4 +462,4 @@ endif

@echo "ISO Release files are published"
endif
$(hide)rm -rf $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz && rm -rf $(TOP)/Release_Files && rm -rf $(TOP)/$(TARGET_PRODUCT)-flashfiles-*.zip && rm -rf $(TOP)/scripts && rm -rf $(TOP)/*patches && rm -rf $(TOP)/*provisioning && rm -rf $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(TOP)/*-flashfile-*.iso
$(hide)rm -rf $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz && rm -rf $(TOP)/Release_Files && rm -rf $(TOP)/$(TARGET_PRODUCT)-flashfiles-*.tar.gz && rm -rf $(TOP)/scripts && rm -rf $(TOP)/*patches && rm -rf $(TOP)/*provisioning && rm -rf $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(TOP)/*-flashfile-*.iso

0 comments on commit 1870d5f

Please sign in to comment.