Merge bitcoin/bitcoin#28432: build: Produce a .zip for macOS distri…

…bution

b5790c3 build: remove dmg dependencies (fanquake)
33ae0bd macdeploy: remove DMG generation from deploy script (fanquake)
a128111 build: produce a .zip for macOS distribution (Hennadii Stepanov)
c38561d build: add -zip option to macdeployqtplus (fanquake)

Pull request description:

  It is bitcoin/bitcoin#27099 revived with addressed [comments](bitcoin/bitcoin#27099 (comment)).

  From bitcoin/bitcoin#27099 (comment):
  > Reviving the discussion around using a `.zip` for the distributed macOS binaries, as opposed to a `.dmg`.
  >
  > Given we only had a single report of the "no finder window" issue (#26176), I wonder if that means macOS users were able to figure it out, they gave up/didn't report, or, we just have very few macOS users.
  >
  > Related to #18128.

  That's how it looks on macOS:

  ![image](https://github.com/bitcoin/bitcoin/assets/32963518/baa637bb-256b-4b24-8645-8c2754c2ae64)

ACKs for top commit:
  Sjors:
    tACK b5790c3
  jarolrod:
    ACK b5790c3
  TheCharlatan:
    utACK b5790c3

Tree-SHA512: 6e9cb3ab0f60f8a92bfec50577e8d096c5b23ec09ebbb334826415609140ddc96d470aea37379495c1c6bb1beec0d306b09460f62e1543bb0f4396c10a1dfbe2

.gitignore

@@ -74,7 +74,7 @@ src/qt/bitcoin-qt.includes
 
 *.log
 *.trs
-*.dmg
+*.zip
 
 *.json.h
 *.raw.h

Makefile.am

@@ -37,7 +37,7 @@ space := $(empty) $(empty)
 
 OSX_APP=Bitcoin-Qt.app
 OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME))
-OSX_DMG = $(OSX_VOLNAME).dmg
+OSX_ZIP = $(OSX_VOLNAME).zip
 OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus
 OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns
 OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed
@@ -124,23 +124,24 @@ osx_volname:
 	echo $(OSX_VOLNAME) >$@
 
 if BUILD_DARWIN
-$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING)
-	$(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg
+$(OSX_ZIP): $(OSX_APP_BUILT) $(OSX_PACKAGING)
+	$(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -zip
 
-deploydir: $(OSX_DMG)
+deploydir: $(OSX_ZIP)
 else !BUILD_DARWIN
 APP_DIST_DIR=$(top_builddir)/dist
 
-$(OSX_DMG): deploydir
-	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH))
+$(OSX_ZIP): deploydir
+	if [ -n "$(SOURCE_DATE_EPOCH)" ]; then find $(APP_DIST_DIR) -exec touch -d @$(SOURCE_DATE_EPOCH) {} +; fi
+	cd $(APP_DIST_DIR) && find . | sort | $(ZIP) -X@ $@
 
 $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING)
 	INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR)
 
 deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt
 endif !BUILD_DARWIN
 
-deploy: $(OSX_DMG)
+deploy: $(OSX_ZIP)
 endif
 
 $(BITCOIN_QT_BIN): FORCE
@@ -313,7 +314,7 @@ EXTRA_DIST += \
     test/util/data/txcreatesignv2.hex \
     test/util/rpcauth-test.py
 
-CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER)
+CLEANFILES = $(OSX_ZIP) $(BITCOIN_WIN_INSTALLER)
 
 DISTCHECK_CONFIGURE_FLAGS = --enable-man
 

ci/test/00_setup_env_mac.sh

@@ -11,7 +11,7 @@ export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export CONTAINER_NAME=ci_macos_cross
 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
 export HOST=x86_64-apple-darwin
-export PACKAGES="cmake libz-dev python3-setuptools xorriso"
+export PACKAGES="cmake libz-dev python3-setuptools zip"
 export XCODE_VERSION=12.2
 export XCODE_BUILD_ID=12B45b
 export RUN_UNIT_TESTS=false

configure.ac

@@ -799,7 +799,7 @@ case $host in
            AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil])
            AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool])
            AC_PATH_TOOL([OTOOL], [otool], [otool])
-           AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs])
+           AC_PATH_PROG([ZIP], [zip], [zip])
 
            dnl libtool will try to strip the static lib, which is a problem for
            dnl cross-builds because strip attempts to call a hard-coded ld,
@@ -1936,7 +1936,6 @@ AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/spl
 AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])])
 AC_CONFIG_LINKS([contrib/devtools/iwyu/bitcoin.core.imp:contrib/devtools/iwyu/bitcoin.core.imp])
 AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py])
-AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff])
 AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config])
 AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy])
 AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py])

contrib/guix/libexec/build.sh

@@ -315,7 +315,7 @@ mkdir -p "$DISTSRC"
                     | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \
                     || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 )
             )
-            make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg"
+            make deploy ${V:+V=1} OSX_ZIP="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip"
             ;;
     esac
     (

contrib/guix/libexec/codesign.sh

@@ -85,11 +85,8 @@ mkdir -p "$DISTSRC"
             # Apply detached codesignatures to dist/ (in-place)
             signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist
 
-            # Make a DMG from dist/
-            xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \
-                      -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \
-                      dist \
-                      -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH"
+            # Make a .zip from dist/
+            zip "${OUTDIR}/${DISTNAME}-${HOST}.zip" dist/*
             ;;
         *)
             exit 1

contrib/guix/manifest.scm

@@ -3,7 +3,6 @@
              ((gnu packages bash) #:select (bash-minimal))
              (gnu packages bison)
              ((gnu packages certs) #:select (nss-certs))
-             ((gnu packages cdrom) #:select (xorriso))
              ((gnu packages cmake) #:select (cmake-minimal))
              (gnu packages commencement)
              (gnu packages compression)
@@ -606,5 +605,5 @@ inspecting signatures in Mach-O binaries.")
           ((string-contains target "-linux-")
            (list (make-bitcoin-cross-toolchain target)))
           ((string-contains target "darwin")
-           (list clang-toolchain-15 binutils cmake-minimal xorriso python-signapple))
+           (list clang-toolchain-15 binutils cmake-minimal python-signapple zip))
           (else '())))))

contrib/macdeploy/README.md

@@ -6,7 +6,7 @@ The `macdeployqtplus` script should not be run manually. Instead, after building
 make deploy
 ```
 
-When complete, it will have produced `Bitcoin-Core.dmg`.
+When complete, it will have produced `Bitcoin-Core.zip`.
 
 ## SDK Extraction
 
@@ -60,10 +60,10 @@ previous stage) as the first argument.
 
 The `sha256sum` of the generated TAR.GZ archive should be `df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619`.
 
-## Deterministic macOS DMG Notes
+## Deterministic macOS App Notes
 
-Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple
-`binutils` (`ld`, `ar`, etc) and DMG authoring tools.
+macOS Applications are created in Linux by combining a recent `clang` and the Apple
+`binutils` (`ld`, `ar`, etc).
 
 Apple uses `clang` extensively for development and has upstreamed the necessary
 functionality so that a vanilla clang can take advantage. It supports the use of `-F`,
@@ -93,20 +93,15 @@ created using these tools. The build process has been designed to avoid includin
 SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely
 redistributed.
 
-[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG.
-
-A background image is added to DMG files by inserting a `.DS_Store` during creation.
-
 As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in
 order to satisfy the new Gatekeeper requirements. Because this private key cannot be
 shared, we'll have to be a bit creative in order for the build process to remain somewhat
 deterministic. Here's how it works:
 
-- Builders use Guix to create an unsigned release. This outputs an unsigned DMG which
+- Builders use Guix to create an unsigned release. This outputs an unsigned ZIP which
   users may choose to bless and run. It also outputs an unsigned app structure in the form
-  of a tarball, which also contains all of the tools that have been previously (deterministically)
-  built in order to create a final DMG.
+  of a tarball.
 - The Apple keyholder uses this unsigned app to create a detached signature, using the
   script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs).
 - Builders feed the unsigned app + detached signature back into Guix. It uses the
-  pre-built tools to recombine the pieces into a deterministic DMG.
+  pre-built tools to recombine the pieces into a deterministic ZIP.

contrib/macdeploy/macdeployqtplus

@@ -18,8 +18,6 @@
 
 import sys, re, os, platform, shutil, stat, subprocess, os.path
 from argparse import ArgumentParser
-from ds_store import DSStore
-from mac_alias import Alias
 from pathlib import Path
 from subprocess import PIPE, run
 from typing import List, Optional
@@ -385,7 +383,7 @@ def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: Deployme
 
 ap = ArgumentParser(description="""Improved version of macdeployqt.
 
-Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .dmg file.
+Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .zip file.
 Note, that the "dist" folder will be deleted before deploying on each run.
 
 Optionally, Qt translation files (.qm) can be added to the bundle.""")
@@ -395,8 +393,8 @@ ap.add_argument("appname", nargs=1, metavar="appname", help="name of the app bei
 ap.add_argument("-verbose", nargs="?", const=True, help="Output additional debugging information")
 ap.add_argument("-no-plugins", dest="plugins", action="store_false", default=True, help="skip plugin deployment")
 ap.add_argument("-no-strip", dest="strip", action="store_false", default=True, help="don't run 'strip' on the binaries")
-ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image")
 ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translations. Base translations will automatically be added to the bundle's resources.")
+ap.add_argument("-zip", nargs="?", const="", metavar="zip", help="create a .zip containing the app bundle")
 
 config = ap.parse_args()
 
@@ -417,12 +415,9 @@ if os.path.exists("dist"):
     print("+ Removing existing dist folder +")
     shutil.rmtree("dist")
 
-if os.path.exists(appname + ".dmg"):
-    print("+ Removing existing DMG +")
-    os.unlink(appname + ".dmg")
-
-if os.path.exists(appname + ".temp.dmg"):
-    os.unlink(appname + ".temp.dmg")
+if os.path.exists(appname + ".zip"):
+    print("+ Removing existing .zip +")
+    os.unlink(appname + ".zip")
 
 # ------------------------------------------------
 
@@ -497,99 +492,13 @@ with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f:
 
 # ------------------------------------------------
 
-print("+ Generating .DS_Store +")
-
-output_file = os.path.join("dist", ".DS_Store")
-
-ds = DSStore.open(output_file, 'w+')
-
-ds['.']['bwsp'] = {
-    'WindowBounds': '{{300, 280}, {500, 343}}',
-    'PreviewPaneVisibility': False,
-}
-
-icvp = {
-    'gridOffsetX': 0.0,
-    'textSize': 12.0,
-    'viewOptionsVersion': 1,
-    'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00',
-    'backgroundColorBlue': 1.0,
-    'iconSize': 96.0,
-    'backgroundColorGreen': 1.0,
-    'arrangeBy': 'none',
-    'showIconPreview': True,
-    'gridSpacing': 100.0,
-    'gridOffsetY': 0.0,
-    'showItemInfo': False,
-    'labelOnBottom': True,
-    'backgroundType': 2,
-    'backgroundColorRed': 1.0
-}
-alias = Alias().from_bytes(icvp['backgroundImageAlias'])
-alias.volume.name = appname
-alias.volume.posix_path = '/Volumes/' + appname
-icvp['backgroundImageAlias'] = alias.to_bytes()
-ds['.']['icvp'] = icvp
-
-ds['.']['vSrn'] = ('long', 1)
-
-ds['Applications']['Iloc'] = (370, 156)
-ds['Bitcoin-Qt.app']['Iloc'] = (128, 156)
-
-ds.flush()
-ds.close()
-
-# ------------------------------------------------
-
 if platform.system() == "Darwin":
     subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True)
 
-print("+ Installing background.tiff +")
-
-bg_path = os.path.join('dist', '.background', 'background.tiff')
-os.mkdir(os.path.dirname(bg_path))
-
-tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff')
-shutil.copy2(tiff_path, bg_path)
-
-# ------------------------------------------------
-
-print("+ Generating symlink for /Applications +")
-
-os.symlink("/Applications", os.path.join('dist', "Applications"))
-
 # ------------------------------------------------
 
-if config.dmg is not None:
-
-    print("+ Preparing .dmg disk image +")
-
-    if verbose:
-        print("Determining size of \"dist\"...")
-    size = 0
-    for path, dirs, files in os.walk("dist"):
-        for file in files:
-            size += os.path.getsize(os.path.join(path, file))
-    size += int(size * 0.15)
-
-    if verbose:
-        print("Creating temp image for modification...")
-
-    tempname: str = appname + ".temp.dmg"
-
-    run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, text=True)
-
-    if verbose:
-        print("Attaching temp image...")
-    output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, text=True, stdout=PIPE).stdout
-
-    print("+ Finalizing .dmg disk image +")
-
-    run(["hdiutil", "detach", f"/Volumes/{appname}"], text=True)
-
-    run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, text=True)
-
-    os.unlink(tempname)
+if config.zip is not None:
+    shutil.make_archive('{}'.format(appname), format='zip', root_dir='dist', base_dir='Bitcoin-Qt.app')
 
 # ------------------------------------------------
 

depends/README.md

@@ -48,7 +48,7 @@ The paths are automatically configured and no other options are needed unless ta
 
 #### For macOS cross compilation
 
-    sudo apt-get install curl bsdmainutils cmake libz-dev python3-setuptools xorriso
+    sudo apt-get install curl bsdmainutils cmake libz-dev python3-setuptools zip
 
 Note: You must obtain the macOS SDK before proceeding with a cross-compile.
 Under the depends directory, create a subdirectory named `SDKs`.

depends/packages/packages.mk

@@ -27,7 +27,7 @@ multiprocess_native_packages = native_libmultiprocess native_capnp
 
 usdt_linux_packages=systemtap
 
-darwin_native_packages = native_ds_store native_mac_alias
+darwin_native_packages =
 
 ifneq ($(build_os),darwin)
 darwin_native_packages += native_cctools native_libtapi

doc/build-osx.md

@@ -163,14 +163,8 @@ brew install python
 
 #### Deploy Dependencies
 
-You can deploy a `.dmg` containing the Bitcoin Core application using `make deploy`.
-This command depends on a couple of python packages, so it is required that you have `python` installed.
-
-Ensuring that `python` is installed, you can install the deploy dependencies by running the following commands in your terminal:
-
-``` bash
-pip3 install ds_store mac_alias
-```
+You can deploy a `.zip` containing the Bitcoin Core application using `make deploy`.
+It is required that you have `python` installed.
 
 ## Building Bitcoin Core
 
@@ -230,7 +224,7 @@ make check  # Run tests if Python 3 is available
 
 ### 3. Deploy (optional)
 
-You can also create a  `.dmg` containing the `.app` bundle by running the following command:
+You can also create a  `.zip` containing the `.app` bundle by running the following command:
 
 ``` bash
 make deploy

doc/release-process.md

@@ -123,7 +123,7 @@ git -C ./guix.sigs pull
 ### Create the macOS SDK tarball (first time, or when SDK version changes)
 
 Create the macOS SDK tarball, see the [macdeploy
-instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for
+instructions](/contrib/macdeploy/README.md#deterministic-macos-app-notes) for
 details.
 
 ### Build and attest to build outputs